nexus-queue 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nexus_queue/__init__.py +86 -0
- nexus_queue/app.py +47 -0
- nexus_queue/broker.py +29 -0
- nexus_queue/config.py +98 -0
- nexus_queue/delayed.py +91 -0
- nexus_queue/envelope.py +56 -0
- nexus_queue/exceptions.py +15 -0
- nexus_queue/handlers.py +80 -0
- nexus_queue/kicker.py +98 -0
- nexus_queue/lifecycle.py +104 -0
- nexus_queue/middleware/__init__.py +8 -0
- nexus_queue/middleware/metrics.py +59 -0
- nexus_queue/middleware/retry_dlq.py +125 -0
- nexus_queue/naming.py +72 -0
- nexus_queue/pipeline.py +49 -0
- nexus_queue/ports.py +81 -0
- nexus_queue/publisher.py +78 -0
- nexus_queue/py.typed +0 -0
- nexus_queue/tracing.py +38 -0
- nexus_queue-0.1.1.dist-info/METADATA +61 -0
- nexus_queue-0.1.1.dist-info/RECORD +22 -0
- nexus_queue-0.1.1.dist-info/WHEEL +4 -0
nexus_queue/__init__.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""nexus-queue — portable taskiq + Redis Streams worker runtime.
|
|
2
|
+
|
|
3
|
+
Public API is intentionally small and stable: the config, the ports, the
|
|
4
|
+
envelope, and the error taxonomy. The runtime pieces (broker, middleware,
|
|
5
|
+
publisher, kicker) are added on top of these and re-exported as they land.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from nexus_queue.app import WorkerApp, create_worker
|
|
11
|
+
from nexus_queue.broker import create_broker
|
|
12
|
+
from nexus_queue.config import RuntimeConfig
|
|
13
|
+
from nexus_queue.delayed import DelayedRetryPoller
|
|
14
|
+
from nexus_queue.envelope import (
|
|
15
|
+
Envelope,
|
|
16
|
+
missing_required_labels,
|
|
17
|
+
require_supported_version,
|
|
18
|
+
)
|
|
19
|
+
from nexus_queue.exceptions import (
|
|
20
|
+
NexusPermanentError,
|
|
21
|
+
NexusQueueError,
|
|
22
|
+
NexusRetryableError,
|
|
23
|
+
)
|
|
24
|
+
from nexus_queue.handlers import HandlerSpec, register
|
|
25
|
+
from nexus_queue.kicker import create_kicker
|
|
26
|
+
from nexus_queue.lifecycle import (
|
|
27
|
+
IdempotencyStore,
|
|
28
|
+
configure_logging,
|
|
29
|
+
register_lifecycle,
|
|
30
|
+
)
|
|
31
|
+
from nexus_queue.naming import (
|
|
32
|
+
NQ_VERSION,
|
|
33
|
+
SINGLE_TENANT,
|
|
34
|
+
consumer_group,
|
|
35
|
+
delayed_set,
|
|
36
|
+
dlq_stream,
|
|
37
|
+
status_stream,
|
|
38
|
+
work_stream,
|
|
39
|
+
)
|
|
40
|
+
from nexus_queue.pipeline import PipelineRouter
|
|
41
|
+
from nexus_queue.ports import (
|
|
42
|
+
BlobStorePort,
|
|
43
|
+
IndexPort,
|
|
44
|
+
JobStatePort,
|
|
45
|
+
JobStatus,
|
|
46
|
+
StatusEvent,
|
|
47
|
+
StatusEventPort,
|
|
48
|
+
)
|
|
49
|
+
from nexus_queue.publisher import Publisher
|
|
50
|
+
from nexus_queue.tracing import configure_tracing
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"NQ_VERSION",
|
|
54
|
+
"SINGLE_TENANT",
|
|
55
|
+
"BlobStorePort",
|
|
56
|
+
"DelayedRetryPoller",
|
|
57
|
+
"Envelope",
|
|
58
|
+
"HandlerSpec",
|
|
59
|
+
"IdempotencyStore",
|
|
60
|
+
"IndexPort",
|
|
61
|
+
"JobStatePort",
|
|
62
|
+
"JobStatus",
|
|
63
|
+
"NexusPermanentError",
|
|
64
|
+
"NexusQueueError",
|
|
65
|
+
"NexusRetryableError",
|
|
66
|
+
"PipelineRouter",
|
|
67
|
+
"Publisher",
|
|
68
|
+
"RuntimeConfig",
|
|
69
|
+
"StatusEvent",
|
|
70
|
+
"StatusEventPort",
|
|
71
|
+
"WorkerApp",
|
|
72
|
+
"configure_logging",
|
|
73
|
+
"configure_tracing",
|
|
74
|
+
"consumer_group",
|
|
75
|
+
"create_broker",
|
|
76
|
+
"create_kicker",
|
|
77
|
+
"create_worker",
|
|
78
|
+
"delayed_set",
|
|
79
|
+
"dlq_stream",
|
|
80
|
+
"missing_required_labels",
|
|
81
|
+
"register",
|
|
82
|
+
"register_lifecycle",
|
|
83
|
+
"require_supported_version",
|
|
84
|
+
"status_stream",
|
|
85
|
+
"work_stream",
|
|
86
|
+
]
|
nexus_queue/app.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Top-level factory. Consumers call ``create_worker(config, adapters, handlers)``
|
|
2
|
+
and expose the returned ``broker`` to the taskiq CLI and ``app`` to uvicorn.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from collections.abc import Iterator, Sequence
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
|
|
10
|
+
from fastapi import FastAPI
|
|
11
|
+
from taskiq import AsyncBroker
|
|
12
|
+
|
|
13
|
+
from nexus_queue.broker import create_broker
|
|
14
|
+
from nexus_queue.config import RuntimeConfig
|
|
15
|
+
from nexus_queue.handlers import HandlerSpec, register
|
|
16
|
+
from nexus_queue.kicker import create_kicker
|
|
17
|
+
from nexus_queue.lifecycle import configure_logging, register_lifecycle
|
|
18
|
+
from nexus_queue.tracing import configure_tracing
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(slots=True, frozen=True)
|
|
22
|
+
class WorkerApp:
|
|
23
|
+
"""Bundle returned by :func:`create_worker`: ``app, broker = create_worker(...)``."""
|
|
24
|
+
|
|
25
|
+
app: FastAPI
|
|
26
|
+
broker: AsyncBroker
|
|
27
|
+
|
|
28
|
+
def __iter__(self) -> Iterator[FastAPI | AsyncBroker]:
|
|
29
|
+
yield self.app
|
|
30
|
+
yield self.broker
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def create_worker(
|
|
34
|
+
config: RuntimeConfig,
|
|
35
|
+
adapters: object,
|
|
36
|
+
handlers: Sequence[HandlerSpec],
|
|
37
|
+
) -> WorkerApp:
|
|
38
|
+
"""Build the broker (namespaced + middleware), register lifecycle + handlers,
|
|
39
|
+
and wrap a FastAPI kicker."""
|
|
40
|
+
configure_logging(config)
|
|
41
|
+
configure_tracing(config)
|
|
42
|
+
broker = create_broker(config)
|
|
43
|
+
register_lifecycle(broker, config, adapters)
|
|
44
|
+
for spec in handlers:
|
|
45
|
+
register(broker, spec, config)
|
|
46
|
+
app = create_kicker(broker, config)
|
|
47
|
+
return WorkerApp(app=app, broker=broker)
|
nexus_queue/broker.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Broker factory — the one place that builds a namespaced, middleware-wrapped broker.
|
|
2
|
+
|
|
3
|
+
Consumers never instantiate ``RedisStreamBroker`` directly: that is how ZP and
|
|
4
|
+
nixon ended up on the default global ``"taskiq"`` stream (which collides across
|
|
5
|
+
projects). Here the stream and consumer group are always
|
|
6
|
+
``nq:{project}:{queue}`` / ``…:cg``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from taskiq import AsyncBroker
|
|
12
|
+
from taskiq_redis import RedisStreamBroker
|
|
13
|
+
|
|
14
|
+
from nexus_queue.config import RuntimeConfig
|
|
15
|
+
from nexus_queue.middleware.metrics import MetricsMiddleware
|
|
16
|
+
from nexus_queue.middleware.retry_dlq import RetryDlqMiddleware
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def create_broker(config: RuntimeConfig) -> AsyncBroker:
|
|
20
|
+
"""Build the standard broker: namespaced streams + the Nexus-Queue middleware stack."""
|
|
21
|
+
broker: AsyncBroker = RedisStreamBroker(
|
|
22
|
+
url=config.redis_url,
|
|
23
|
+
queue_name=config.work_stream,
|
|
24
|
+
consumer_group_name=config.consumer_group,
|
|
25
|
+
)
|
|
26
|
+
return broker.with_middlewares(
|
|
27
|
+
MetricsMiddleware(config),
|
|
28
|
+
RetryDlqMiddleware(config),
|
|
29
|
+
)
|
nexus_queue/config.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Runtime configuration consumed by the worker runtime.
|
|
2
|
+
|
|
3
|
+
A single pydantic model filled in by the consumer. No env loading inside the
|
|
4
|
+
library (mirrors `agno-agent-builder` / `payload-documents-worker-builder`) so a
|
|
5
|
+
multi-tenant deploy can build several `RuntimeConfig` instances from one env.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field, SecretStr, ValidationInfo, field_validator
|
|
11
|
+
|
|
12
|
+
from nexus_queue import naming
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class RuntimeConfig(BaseModel):
|
|
16
|
+
"""Everything the runtime needs that is not a port adapter or a handler."""
|
|
17
|
+
|
|
18
|
+
app_name: str = Field(
|
|
19
|
+
description="FastAPI title and structlog identity; shows up in logs and /health.",
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
# ── Identity / namespacing ─────────────────────────────────────────────
|
|
23
|
+
project: str = Field(
|
|
24
|
+
description="Stable short project slug used to namespace streams, e.g. 'zp', 'nixon'.",
|
|
25
|
+
)
|
|
26
|
+
queue: str = Field(
|
|
27
|
+
description="Logical queue/pipeline name, e.g. 'documents', 'jobs'.",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# ── Broker ─────────────────────────────────────────────────────────────
|
|
31
|
+
redis_url: str = Field(
|
|
32
|
+
description="Redis URL for the taskiq-redis stream broker (e.g. redis://redis:6379).",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# ── HTTP kicker ────────────────────────────────────────────────────────
|
|
36
|
+
internal_secret: SecretStr = Field(
|
|
37
|
+
default=SecretStr(""),
|
|
38
|
+
description="Shared secret required by the kicker (X-Nexus-Secret). Empty disables the kicker auth gate.",
|
|
39
|
+
)
|
|
40
|
+
public_paths: tuple[str, ...] = Field(
|
|
41
|
+
default=("/health", "/ready", "/metrics", "/docs", "/openapi.json"),
|
|
42
|
+
description="Kicker paths served without the secret.",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
# ── Retry / DLQ / idempotency ──────────────────────────────────────────
|
|
46
|
+
max_retries: int = Field(
|
|
47
|
+
default=3,
|
|
48
|
+
ge=0,
|
|
49
|
+
description="Retry attempts before a message is dead-lettered.",
|
|
50
|
+
)
|
|
51
|
+
retry_base_delay_s: float = Field(
|
|
52
|
+
default=2.0,
|
|
53
|
+
gt=0,
|
|
54
|
+
description="Base delay for the exponential backoff applied between retries.",
|
|
55
|
+
)
|
|
56
|
+
retry_poll_interval_s: float = Field(
|
|
57
|
+
default=1.0,
|
|
58
|
+
gt=0,
|
|
59
|
+
description="How often the delayed-retry poller drains due retries back onto the work stream.",
|
|
60
|
+
)
|
|
61
|
+
idempotency_ttl_s: int = Field(
|
|
62
|
+
default=86_400,
|
|
63
|
+
ge=0,
|
|
64
|
+
description="TTL of the dedup key; 0 disables the idempotency middleware.",
|
|
65
|
+
)
|
|
66
|
+
dlq_maxlen: int = Field(
|
|
67
|
+
default=100_000,
|
|
68
|
+
ge=0,
|
|
69
|
+
description="Approx MAXLEN for the dead-letter stream; 0 = unbounded.",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# ── Logging ────────────────────────────────────────────────────────────
|
|
73
|
+
log_level: str = Field(default="INFO")
|
|
74
|
+
|
|
75
|
+
@field_validator("project", "queue")
|
|
76
|
+
@classmethod
|
|
77
|
+
def _validate_slug(cls, value: str, info: ValidationInfo) -> str:
|
|
78
|
+
return naming.validate_slug(value, kind=info.field_name or "slug")
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def work_stream(self) -> str:
|
|
82
|
+
return naming.work_stream(self.project, self.queue)
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def consumer_group(self) -> str:
|
|
86
|
+
return naming.consumer_group(self.project, self.queue)
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def dlq_stream(self) -> str:
|
|
90
|
+
return naming.dlq_stream(self.project, self.queue)
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def delayed_set(self) -> str:
|
|
94
|
+
return naming.delayed_set(self.project, self.queue)
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def status_stream(self) -> str:
|
|
98
|
+
return naming.status_stream(self.project)
|
nexus_queue/delayed.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Delayed-retry queue: hold a failed message until its backoff elapses, then
|
|
2
|
+
re-enqueue it.
|
|
3
|
+
|
|
4
|
+
Redis Streams deliver immediately, so a backed-off retry can't ride the work
|
|
5
|
+
stream. :class:`~nexus_queue.middleware.retry_dlq.RetryDlqMiddleware` parks the
|
|
6
|
+
message in a sorted set scored by its due time; the :class:`DelayedRetryPoller`
|
|
7
|
+
(started by the worker lifecycle) drains due messages back onto the broker. The
|
|
8
|
+
atomic ``ZREM`` makes the move safe across worker replicas — only the one that
|
|
9
|
+
removes a member re-enqueues it.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
import contextlib
|
|
16
|
+
import json
|
|
17
|
+
import time
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import redis.asyncio as aioredis
|
|
21
|
+
import structlog
|
|
22
|
+
from taskiq import AsyncBroker
|
|
23
|
+
from taskiq.kicker import AsyncKicker
|
|
24
|
+
from taskiq.message import TaskiqMessage
|
|
25
|
+
|
|
26
|
+
from nexus_queue.config import RuntimeConfig
|
|
27
|
+
|
|
28
|
+
logger = structlog.get_logger("nexus_queue.delayed")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def pack_retry(message: TaskiqMessage, labels: dict[str, Any]) -> str:
|
|
32
|
+
"""Serialize a message (with its updated labels) for the delayed-retry set."""
|
|
33
|
+
return json.dumps(
|
|
34
|
+
{
|
|
35
|
+
"task_name": message.task_name,
|
|
36
|
+
"task_id": message.task_id,
|
|
37
|
+
"labels": labels,
|
|
38
|
+
"args": message.args,
|
|
39
|
+
"kwargs": message.kwargs,
|
|
40
|
+
}
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class DelayedRetryPoller:
|
|
45
|
+
"""Moves due retries from the delayed sorted set back onto the broker."""
|
|
46
|
+
|
|
47
|
+
def __init__(self, broker: AsyncBroker, config: RuntimeConfig) -> None:
|
|
48
|
+
self._broker = broker
|
|
49
|
+
self._config = config
|
|
50
|
+
self._redis: aioredis.Redis | None = None
|
|
51
|
+
self._task: asyncio.Task[None] | None = None
|
|
52
|
+
|
|
53
|
+
async def startup(self) -> None:
|
|
54
|
+
self._redis = aioredis.from_url(self._config.redis_url)
|
|
55
|
+
self._task = asyncio.create_task(self._run())
|
|
56
|
+
|
|
57
|
+
async def shutdown(self) -> None:
|
|
58
|
+
if self._task is not None:
|
|
59
|
+
self._task.cancel()
|
|
60
|
+
with contextlib.suppress(asyncio.CancelledError):
|
|
61
|
+
await self._task
|
|
62
|
+
if self._redis is not None:
|
|
63
|
+
await self._redis.aclose()
|
|
64
|
+
|
|
65
|
+
async def _run(self) -> None:
|
|
66
|
+
while True:
|
|
67
|
+
try:
|
|
68
|
+
await self._drain_due()
|
|
69
|
+
except asyncio.CancelledError:
|
|
70
|
+
raise
|
|
71
|
+
except Exception:
|
|
72
|
+
logger.exception("delayed-poll-failed")
|
|
73
|
+
await asyncio.sleep(self._config.retry_poll_interval_s)
|
|
74
|
+
|
|
75
|
+
async def _drain_due(self) -> None:
|
|
76
|
+
if self._redis is None:
|
|
77
|
+
return
|
|
78
|
+
due = await self._redis.zrangebyscore(self._config.delayed_set, "-inf", time.time())
|
|
79
|
+
for record in due:
|
|
80
|
+
# Atomic claim: only the replica that removes the member re-enqueues it.
|
|
81
|
+
if await self._redis.zrem(self._config.delayed_set, record) == 1:
|
|
82
|
+
await self._reenqueue(json.loads(record))
|
|
83
|
+
|
|
84
|
+
async def _reenqueue(self, data: dict[str, Any]) -> None:
|
|
85
|
+
kicker: AsyncKicker[Any, Any] = AsyncKicker(
|
|
86
|
+
task_name=data["task_name"],
|
|
87
|
+
broker=self._broker,
|
|
88
|
+
labels=data["labels"],
|
|
89
|
+
).with_task_id(data["task_id"])
|
|
90
|
+
await kicker.kiq(*data["args"], **data["kwargs"])
|
|
91
|
+
logger.info("retry-reenqueued", task=data["task_name"])
|
nexus_queue/envelope.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""The Nexus-Queue envelope: standard labels stamped on every message.
|
|
2
|
+
|
|
3
|
+
Producers build an :class:`Envelope` and turn it into the ``labels`` dict that
|
|
4
|
+
rides on the taskiq message; consumers validate the version on receipt.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import UTC, datetime
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from nexus_queue import naming
|
|
15
|
+
from nexus_queue.exceptions import NexusPermanentError
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(slots=True)
|
|
19
|
+
class Envelope:
|
|
20
|
+
"""Routing/metadata for a single enqueue. Carried in message labels, not args."""
|
|
21
|
+
|
|
22
|
+
task: str
|
|
23
|
+
tenant: str = naming.SINGLE_TENANT
|
|
24
|
+
idempotency_key: str | None = None
|
|
25
|
+
trace: str | None = None
|
|
26
|
+
priority: str = "default"
|
|
27
|
+
|
|
28
|
+
def to_labels(self) -> dict[str, str]:
|
|
29
|
+
"""Render the standard labels. ``nq_enqueued_at`` is stamped now (UTC)."""
|
|
30
|
+
labels: dict[str, str] = {
|
|
31
|
+
naming.LABEL_VERSION: naming.NQ_VERSION,
|
|
32
|
+
naming.LABEL_TASK: self.task,
|
|
33
|
+
naming.LABEL_TENANT: self.tenant,
|
|
34
|
+
naming.LABEL_ENQUEUED_AT: datetime.now(UTC).isoformat(),
|
|
35
|
+
naming.LABEL_PRIORITY: self.priority,
|
|
36
|
+
}
|
|
37
|
+
if self.idempotency_key:
|
|
38
|
+
labels[naming.LABEL_IDEM] = self.idempotency_key
|
|
39
|
+
if self.trace:
|
|
40
|
+
labels[naming.LABEL_TRACE] = self.trace
|
|
41
|
+
return labels
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def require_supported_version(labels: Mapping[str, Any]) -> None:
|
|
45
|
+
"""Raise :class:`NexusPermanent` (→ DLQ) if the message is a version we
|
|
46
|
+
don't speak. Keeps a future ``nq_v=2`` from being silently mishandled."""
|
|
47
|
+
version = str(labels.get(naming.LABEL_VERSION, ""))
|
|
48
|
+
if version != naming.NQ_VERSION:
|
|
49
|
+
raise NexusPermanentError(
|
|
50
|
+
f"Unsupported nq_v={version!r}; this worker speaks {naming.NQ_VERSION!r}"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def missing_required_labels(labels: Mapping[str, Any]) -> tuple[str, ...]:
|
|
55
|
+
"""Return the required labels absent from ``labels`` (empty tuple if valid)."""
|
|
56
|
+
return tuple(key for key in naming.REQUIRED_LABELS if not labels.get(key))
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Error taxonomy that drives the runtime's retry vs dead-letter decision."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class NexusQueueError(Exception):
|
|
7
|
+
"""Base class for every error raised by nexus-queue."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class NexusRetryableError(NexusQueueError):
|
|
11
|
+
"""Transient failure — retry until attempts are exhausted, then dead-letter."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class NexusPermanentError(NexusQueueError):
|
|
15
|
+
"""Non-retryable failure — route straight to the DLQ, skip remaining retries."""
|
nexus_queue/handlers.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Handler registration with the cross-cutting concerns that need an ``around``
|
|
2
|
+
scope (which taskiq middleware hooks can't give): idempotency, tracing, latency.
|
|
3
|
+
|
|
4
|
+
A handler is ``async def h(payload, deps) -> None`` — it depends only on its
|
|
5
|
+
typed payload and the project's :class:`NexusAdapters`. ``register`` wraps it
|
|
6
|
+
so the same handler runs unchanged in any project.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Awaitable, Callable
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import structlog
|
|
16
|
+
from opentelemetry import trace
|
|
17
|
+
from opentelemetry.propagate import extract
|
|
18
|
+
from pydantic import BaseModel
|
|
19
|
+
from taskiq import AsyncBroker, Context, TaskiqDepends
|
|
20
|
+
|
|
21
|
+
from nexus_queue.config import RuntimeConfig
|
|
22
|
+
from nexus_queue.envelope import require_supported_version
|
|
23
|
+
from nexus_queue.lifecycle import IdempotencyStore
|
|
24
|
+
from nexus_queue.middleware.metrics import CONSUME_SECONDS
|
|
25
|
+
from nexus_queue.naming import LABEL_IDEM, LABEL_TENANT, LABEL_TRACE, SINGLE_TENANT
|
|
26
|
+
|
|
27
|
+
logger = structlog.get_logger("nexus_queue.handlers")
|
|
28
|
+
_tracer = trace.get_tracer("nexus_queue")
|
|
29
|
+
|
|
30
|
+
# deps is the project's adapter container; the handler types it to a Protocol
|
|
31
|
+
# of the ports it actually uses.
|
|
32
|
+
HandlerFn = Callable[[Any, Any], Awaitable[None]]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(slots=True)
|
|
36
|
+
class HandlerSpec:
|
|
37
|
+
"""Binds a wire task name to a handler and its payload model."""
|
|
38
|
+
|
|
39
|
+
task_name: str
|
|
40
|
+
handler: HandlerFn
|
|
41
|
+
payload_model: type[BaseModel]
|
|
42
|
+
idempotent: bool = True
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def register(broker: AsyncBroker, spec: HandlerSpec, config: RuntimeConfig) -> None:
|
|
46
|
+
"""Register ``spec.handler`` under its wire task name, wrapped with version
|
|
47
|
+
check, idempotency dedup, an OTel consume span, and a latency histogram."""
|
|
48
|
+
|
|
49
|
+
async def _run(context: Context = TaskiqDepends(), **kwargs: Any) -> None:
|
|
50
|
+
labels = context.message.labels
|
|
51
|
+
require_supported_version(labels)
|
|
52
|
+
state = context.state
|
|
53
|
+
|
|
54
|
+
raw_idem = labels.get(LABEL_IDEM) if spec.idempotent else None
|
|
55
|
+
idem = str(raw_idem) if raw_idem else None
|
|
56
|
+
store: IdempotencyStore | None = state.nexus_idempotency if idem else None
|
|
57
|
+
if store is not None and idem and await store.already_processed(idem):
|
|
58
|
+
logger.info("duplicate-skipped", task=spec.task_name, idem=idem)
|
|
59
|
+
return
|
|
60
|
+
|
|
61
|
+
adapters = state.nexus_adapters
|
|
62
|
+
payload = spec.payload_model(**kwargs)
|
|
63
|
+
|
|
64
|
+
traceparent = labels.get(LABEL_TRACE)
|
|
65
|
+
parent = extract({"traceparent": str(traceparent)}) if traceparent else None
|
|
66
|
+
with _tracer.start_as_current_span(
|
|
67
|
+
f"nexus_queue.consume {spec.task_name}",
|
|
68
|
+
context=parent,
|
|
69
|
+
) as span:
|
|
70
|
+
span.set_attribute("nq.task", spec.task_name)
|
|
71
|
+
span.set_attribute("nq.tenant", str(labels.get(LABEL_TENANT, SINGLE_TENANT)))
|
|
72
|
+
with CONSUME_SECONDS.labels(config.project, config.queue, spec.task_name).time():
|
|
73
|
+
await spec.handler(payload, adapters)
|
|
74
|
+
|
|
75
|
+
# Record dedup only after success: a failed attempt must stay retryable.
|
|
76
|
+
if store is not None and idem:
|
|
77
|
+
await store.mark_processed(idem)
|
|
78
|
+
|
|
79
|
+
_run.__name__ = "nexus_run_" + spec.task_name.replace(".", "_").replace(":", "_")
|
|
80
|
+
broker.task(task_name=spec.task_name)(_run)
|
nexus_queue/kicker.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Generic HTTP kicker for producers that can't speak Redis directly
|
|
2
|
+
(e.g. a TypeScript/Payload caller). One standard contract for every project:
|
|
3
|
+
``POST /enqueue/{task}`` gated by ``X-Nexus-Secret``, plus ``/health``/``/ready``.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hmac
|
|
9
|
+
from collections.abc import AsyncIterator
|
|
10
|
+
from contextlib import asynccontextmanager
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import structlog
|
|
14
|
+
from fastapi import FastAPI, Request, Response, status
|
|
15
|
+
from fastapi.responses import JSONResponse
|
|
16
|
+
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
|
|
17
|
+
from pydantic import BaseModel, Field
|
|
18
|
+
from taskiq import AsyncBroker
|
|
19
|
+
|
|
20
|
+
from nexus_queue.config import RuntimeConfig
|
|
21
|
+
from nexus_queue.naming import SINGLE_TENANT
|
|
22
|
+
from nexus_queue.publisher import Publisher
|
|
23
|
+
|
|
24
|
+
logger = structlog.get_logger("nexus_queue.kicker")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class EnqueueRequest(BaseModel):
|
|
28
|
+
payload: dict[str, Any] = Field(default_factory=dict)
|
|
29
|
+
tenant: str = SINGLE_TENANT
|
|
30
|
+
idempotency_key: str | None = None
|
|
31
|
+
priority: str = "default"
|
|
32
|
+
trace: str | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def create_kicker(broker: AsyncBroker, config: RuntimeConfig) -> FastAPI:
|
|
36
|
+
"""Build the FastAPI kicker the consumer hands to uvicorn."""
|
|
37
|
+
|
|
38
|
+
@asynccontextmanager
|
|
39
|
+
async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
|
40
|
+
# Fail-open is allowed (some deploys front the kicker with a mesh), but it
|
|
41
|
+
# must never be silent: an empty secret leaves /enqueue unauthenticated.
|
|
42
|
+
if not config.internal_secret.get_secret_value():
|
|
43
|
+
logger.warning(
|
|
44
|
+
"kicker-auth-disabled",
|
|
45
|
+
detail="internal_secret is empty; /enqueue is unauthenticated",
|
|
46
|
+
)
|
|
47
|
+
# The kicker connects the broker; the worker process owns its own lifecycle.
|
|
48
|
+
if not broker.is_worker_process:
|
|
49
|
+
await broker.startup()
|
|
50
|
+
yield
|
|
51
|
+
if not broker.is_worker_process:
|
|
52
|
+
await broker.shutdown()
|
|
53
|
+
|
|
54
|
+
app = FastAPI(title=config.app_name, lifespan=lifespan)
|
|
55
|
+
publisher = Publisher(broker, config)
|
|
56
|
+
|
|
57
|
+
@app.middleware("http")
|
|
58
|
+
async def _auth(request: Request, call_next: Any) -> Any: # pyright: ignore[reportUnusedFunction]
|
|
59
|
+
if request.url.path in config.public_paths:
|
|
60
|
+
return await call_next(request)
|
|
61
|
+
secret = config.internal_secret.get_secret_value()
|
|
62
|
+
if secret:
|
|
63
|
+
provided = request.headers.get("x-nexus-secret", "")
|
|
64
|
+
if not hmac.compare_digest(provided, secret):
|
|
65
|
+
return JSONResponse(
|
|
66
|
+
{"error": "Forbidden"},
|
|
67
|
+
status_code=status.HTTP_403_FORBIDDEN,
|
|
68
|
+
)
|
|
69
|
+
return await call_next(request)
|
|
70
|
+
|
|
71
|
+
@app.get("/health")
|
|
72
|
+
async def health() -> dict[str, str]: # pyright: ignore[reportUnusedFunction]
|
|
73
|
+
return {"status": "ok"}
|
|
74
|
+
|
|
75
|
+
@app.get("/ready")
|
|
76
|
+
async def ready() -> dict[str, str]: # pyright: ignore[reportUnusedFunction]
|
|
77
|
+
return {"status": "ok"}
|
|
78
|
+
|
|
79
|
+
@app.get("/metrics")
|
|
80
|
+
async def metrics() -> Response: # pyright: ignore[reportUnusedFunction]
|
|
81
|
+
return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)
|
|
82
|
+
|
|
83
|
+
@app.post("/enqueue/{task}", status_code=status.HTTP_202_ACCEPTED)
|
|
84
|
+
async def enqueue( # pyright: ignore[reportUnusedFunction]
|
|
85
|
+
task: str,
|
|
86
|
+
body: EnqueueRequest,
|
|
87
|
+
) -> dict[str, str]:
|
|
88
|
+
task_id = await publisher.enqueue_raw(
|
|
89
|
+
task,
|
|
90
|
+
body.payload,
|
|
91
|
+
tenant=body.tenant,
|
|
92
|
+
idempotency_key=body.idempotency_key,
|
|
93
|
+
priority=body.priority,
|
|
94
|
+
trace=body.trace,
|
|
95
|
+
)
|
|
96
|
+
return {"status": "queued", "task": task, "task_id": task_id}
|
|
97
|
+
|
|
98
|
+
return app
|
nexus_queue/lifecycle.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Worker lifecycle: logging, dependency injection, and the idempotency store.
|
|
2
|
+
|
|
3
|
+
``register_lifecycle`` wires the project's adapters and a Redis-backed
|
|
4
|
+
idempotency store into the worker ``state`` on startup, where the handler
|
|
5
|
+
wrapper (see :mod:`nexus_queue.handlers`) reads them.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
import redis.asyncio as aioredis
|
|
13
|
+
import structlog
|
|
14
|
+
from taskiq import AsyncBroker, TaskiqEvents, TaskiqState
|
|
15
|
+
|
|
16
|
+
from nexus_queue.config import RuntimeConfig
|
|
17
|
+
from nexus_queue.delayed import DelayedRetryPoller
|
|
18
|
+
from nexus_queue.naming import idempotency_redis_key
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class IdempotencyStore:
|
|
22
|
+
"""Redis mark-done dedup keyed by the message's ``nq_idem`` label.
|
|
23
|
+
|
|
24
|
+
The key is recorded only *after* a handler succeeds, so a failed attempt
|
|
25
|
+
leaves no marker and stays eligible for retry/DLQ. Claiming the key up
|
|
26
|
+
front would make a re-enqueued retry skip itself as a phantom duplicate."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, config: RuntimeConfig) -> None:
|
|
29
|
+
self._config = config
|
|
30
|
+
self._redis: aioredis.Redis | None = None
|
|
31
|
+
|
|
32
|
+
async def startup(self) -> None:
|
|
33
|
+
self._redis = aioredis.from_url(self._config.redis_url)
|
|
34
|
+
|
|
35
|
+
async def shutdown(self) -> None:
|
|
36
|
+
if self._redis is not None:
|
|
37
|
+
await self._redis.aclose()
|
|
38
|
+
|
|
39
|
+
async def already_processed(self, idem: str) -> bool:
|
|
40
|
+
"""True if a message with this key already completed within the TTL."""
|
|
41
|
+
if self._config.idempotency_ttl_s <= 0 or self._redis is None:
|
|
42
|
+
return False
|
|
43
|
+
return bool(await self._redis.exists(idempotency_redis_key(idem)))
|
|
44
|
+
|
|
45
|
+
async def mark_processed(self, idem: str) -> None:
|
|
46
|
+
"""Record completion so later duplicates are skipped within the TTL."""
|
|
47
|
+
if self._config.idempotency_ttl_s <= 0 or self._redis is None:
|
|
48
|
+
return
|
|
49
|
+
await self._redis.set(
|
|
50
|
+
idempotency_redis_key(idem),
|
|
51
|
+
"1",
|
|
52
|
+
ex=self._config.idempotency_ttl_s,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def configure_logging(config: RuntimeConfig) -> None:
|
|
57
|
+
"""Idempotent structlog setup so taskiq + FastAPI share one JSON sink."""
|
|
58
|
+
level = getattr(logging, config.log_level.upper(), logging.INFO)
|
|
59
|
+
logging.basicConfig(level=level, format="%(message)s")
|
|
60
|
+
structlog.configure(
|
|
61
|
+
processors=[
|
|
62
|
+
structlog.contextvars.merge_contextvars,
|
|
63
|
+
structlog.processors.add_log_level,
|
|
64
|
+
structlog.processors.TimeStamper(fmt="iso"),
|
|
65
|
+
structlog.processors.StackInfoRenderer(),
|
|
66
|
+
structlog.processors.format_exc_info,
|
|
67
|
+
structlog.processors.JSONRenderer(),
|
|
68
|
+
],
|
|
69
|
+
wrapper_class=structlog.make_filtering_bound_logger(level),
|
|
70
|
+
cache_logger_on_first_use=True,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def register_lifecycle(
|
|
75
|
+
broker: AsyncBroker,
|
|
76
|
+
config: RuntimeConfig,
|
|
77
|
+
adapters: object,
|
|
78
|
+
) -> None:
|
|
79
|
+
"""Register startup/shutdown hooks that expose config, adapters and the
|
|
80
|
+
idempotency store on the worker ``state``.
|
|
81
|
+
|
|
82
|
+
``adapters`` is whatever container the project chooses — the runtime only
|
|
83
|
+
stashes it on ``state`` for the handler wrapper to hand back as ``deps``.
|
|
84
|
+
The package contributes the *ports* (see :mod:`nexus_queue.ports`); the
|
|
85
|
+
*container* is the project's, so a handler can depend on exactly the ports
|
|
86
|
+
it needs."""
|
|
87
|
+
|
|
88
|
+
@broker.on_event(TaskiqEvents.WORKER_STARTUP)
|
|
89
|
+
async def _startup(state: TaskiqState) -> None: # pyright: ignore[reportUnusedFunction]
|
|
90
|
+
state.nexus_config = config
|
|
91
|
+
state.nexus_adapters = adapters
|
|
92
|
+
store = IdempotencyStore(config)
|
|
93
|
+
await store.startup()
|
|
94
|
+
state.nexus_idempotency = store
|
|
95
|
+
poller = DelayedRetryPoller(broker, config)
|
|
96
|
+
await poller.startup()
|
|
97
|
+
state.nexus_delayed = poller
|
|
98
|
+
|
|
99
|
+
@broker.on_event(TaskiqEvents.WORKER_SHUTDOWN)
|
|
100
|
+
async def _shutdown(state: TaskiqState) -> None: # pyright: ignore[reportUnusedFunction]
|
|
101
|
+
store: IdempotencyStore = state.nexus_idempotency
|
|
102
|
+
await store.shutdown()
|
|
103
|
+
poller: DelayedRetryPoller = state.nexus_delayed
|
|
104
|
+
await poller.shutdown()
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Broker-level middleware for the Nexus-Queue runtime."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from nexus_queue.middleware.metrics import MetricsMiddleware
|
|
6
|
+
from nexus_queue.middleware.retry_dlq import RetryDlqMiddleware
|
|
7
|
+
|
|
8
|
+
__all__ = ["MetricsMiddleware", "RetryDlqMiddleware"]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Prometheus counters for queue throughput.
|
|
2
|
+
|
|
3
|
+
Latency is measured in the handler wrapper (it needs an ``around`` scope that
|
|
4
|
+
middleware hooks can't give); here we keep the stateless counters.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from prometheus_client import Counter, Histogram
|
|
12
|
+
from taskiq.abc.middleware import TaskiqMiddleware
|
|
13
|
+
from taskiq.message import TaskiqMessage
|
|
14
|
+
from taskiq.result import TaskiqResult
|
|
15
|
+
|
|
16
|
+
from nexus_queue.config import RuntimeConfig
|
|
17
|
+
from nexus_queue.naming import LABEL_TASK
|
|
18
|
+
|
|
19
|
+
_LABELNAMES = ("project", "queue", "task")
|
|
20
|
+
|
|
21
|
+
RECEIVED = Counter(
|
|
22
|
+
"nexus_queue_received_total",
|
|
23
|
+
"Messages pulled from the stream for execution.",
|
|
24
|
+
_LABELNAMES,
|
|
25
|
+
)
|
|
26
|
+
COMPLETED = Counter(
|
|
27
|
+
"nexus_queue_completed_total",
|
|
28
|
+
"Messages whose handler returned successfully.",
|
|
29
|
+
_LABELNAMES,
|
|
30
|
+
)
|
|
31
|
+
FAILED = Counter(
|
|
32
|
+
"nexus_queue_failed_total",
|
|
33
|
+
"Messages whose handler raised (before retry/DLQ resolution).",
|
|
34
|
+
_LABELNAMES,
|
|
35
|
+
)
|
|
36
|
+
CONSUME_SECONDS = Histogram(
|
|
37
|
+
"nexus_queue_consume_seconds",
|
|
38
|
+
"Handler execution wall-time (measured in the handler wrapper).",
|
|
39
|
+
_LABELNAMES,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class MetricsMiddleware(TaskiqMiddleware):
|
|
44
|
+
"""Increment throughput counters around execution."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, config: RuntimeConfig) -> None:
|
|
47
|
+
super().__init__()
|
|
48
|
+
self._config = config
|
|
49
|
+
|
|
50
|
+
def _task(self, message: TaskiqMessage) -> str:
|
|
51
|
+
return str(message.labels.get(LABEL_TASK, message.task_name))
|
|
52
|
+
|
|
53
|
+
def pre_execute(self, message: TaskiqMessage) -> TaskiqMessage:
|
|
54
|
+
RECEIVED.labels(self._config.project, self._config.queue, self._task(message)).inc()
|
|
55
|
+
return message
|
|
56
|
+
|
|
57
|
+
def post_execute(self, message: TaskiqMessage, result: TaskiqResult[Any]) -> None:
|
|
58
|
+
counter = FAILED if result.is_err else COMPLETED
|
|
59
|
+
counter.labels(self._config.project, self._config.queue, self._task(message)).inc()
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Retry + dead-letter middleware.
|
|
2
|
+
|
|
3
|
+
Unifies retry and DLQ in one place so the two decisions stay consistent:
|
|
4
|
+
|
|
5
|
+
* transient error and attempts remain → park the message in the delayed-retry
|
|
6
|
+
sorted set scored by its due time (``now + exponential backoff``). Redis
|
|
7
|
+
Streams deliver immediately, so the backoff can't ride the work stream; the
|
|
8
|
+
:class:`~nexus_queue.delayed.DelayedRetryPoller` moves the message back onto
|
|
9
|
+
the broker once its delay elapses.
|
|
10
|
+
* attempts exhausted, or a :class:`NexusPermanentError` → ``XADD`` to the
|
|
11
|
+
``nq:{project}:{queue}:dlq`` stream with failure metadata, instead of the
|
|
12
|
+
silent ack-and-drop that both reference projects do today.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import random
|
|
19
|
+
import time
|
|
20
|
+
from datetime import UTC, datetime
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import redis.asyncio as aioredis
|
|
24
|
+
import structlog
|
|
25
|
+
from taskiq.abc.middleware import TaskiqMiddleware
|
|
26
|
+
from taskiq.exceptions import NoResultError
|
|
27
|
+
from taskiq.message import TaskiqMessage
|
|
28
|
+
from taskiq.result import TaskiqResult
|
|
29
|
+
|
|
30
|
+
from nexus_queue.config import RuntimeConfig
|
|
31
|
+
from nexus_queue.delayed import pack_retry
|
|
32
|
+
from nexus_queue.exceptions import NexusPermanentError
|
|
33
|
+
|
|
34
|
+
logger = structlog.get_logger("nexus_queue.retry_dlq")
|
|
35
|
+
|
|
36
|
+
# SystemRandom keeps ruff's bandit check (S311) happy without a noqa: jitter is
|
|
37
|
+
# not security-sensitive, but a CSPRNG is a fine source for it.
|
|
38
|
+
_jitter = random.SystemRandom()
|
|
39
|
+
|
|
40
|
+
_MAX_BACKOFF_S = 60.0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class RetryDlqMiddleware(TaskiqMiddleware):
|
|
44
|
+
"""Re-enqueue transient failures with backoff; dead-letter the rest."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, config: RuntimeConfig) -> None:
|
|
47
|
+
super().__init__()
|
|
48
|
+
self._config = config
|
|
49
|
+
self._redis: aioredis.Redis | None = None
|
|
50
|
+
|
|
51
|
+
async def startup(self) -> None:
|
|
52
|
+
self._redis = aioredis.from_url(self._config.redis_url)
|
|
53
|
+
|
|
54
|
+
async def shutdown(self) -> None:
|
|
55
|
+
if self._redis is not None:
|
|
56
|
+
await self._redis.aclose()
|
|
57
|
+
|
|
58
|
+
def _backoff_delay(self, attempt: int) -> float:
|
|
59
|
+
base = self._config.retry_base_delay_s * (2.0 ** max(attempt - 1, 0))
|
|
60
|
+
return min(base, _MAX_BACKOFF_S) + _jitter.random()
|
|
61
|
+
|
|
62
|
+
async def on_error(
|
|
63
|
+
self,
|
|
64
|
+
message: TaskiqMessage,
|
|
65
|
+
result: TaskiqResult[Any],
|
|
66
|
+
exception: BaseException,
|
|
67
|
+
) -> None:
|
|
68
|
+
if isinstance(exception, NoResultError):
|
|
69
|
+
return
|
|
70
|
+
|
|
71
|
+
permanent = isinstance(exception, NexusPermanentError)
|
|
72
|
+
attempt = int(message.labels.get("_retries", 0)) + 1
|
|
73
|
+
max_retries = int(message.labels.get("max_retries", self._config.max_retries))
|
|
74
|
+
|
|
75
|
+
if not permanent and attempt < max_retries:
|
|
76
|
+
delay = self._backoff_delay(attempt)
|
|
77
|
+
labels = {**message.labels, "_retries": attempt}
|
|
78
|
+
record = pack_retry(message, labels)
|
|
79
|
+
if self._redis is not None:
|
|
80
|
+
await self._redis.zadd(self._config.delayed_set, {record: time.time() + delay})
|
|
81
|
+
result.error = NoResultError()
|
|
82
|
+
logger.info(
|
|
83
|
+
"retry-scheduled",
|
|
84
|
+
task=message.task_name,
|
|
85
|
+
attempt=attempt,
|
|
86
|
+
max_retries=max_retries,
|
|
87
|
+
delay_s=round(delay, 2),
|
|
88
|
+
)
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
await self._dead_letter(message, exception, attempts=attempt, permanent=permanent)
|
|
92
|
+
|
|
93
|
+
async def _dead_letter(
|
|
94
|
+
self,
|
|
95
|
+
message: TaskiqMessage,
|
|
96
|
+
exception: BaseException,
|
|
97
|
+
*,
|
|
98
|
+
attempts: int,
|
|
99
|
+
permanent: bool,
|
|
100
|
+
) -> None:
|
|
101
|
+
record: dict[str, Any] = {
|
|
102
|
+
"task_id": message.task_id,
|
|
103
|
+
"task_name": message.task_name,
|
|
104
|
+
"labels": message.labels,
|
|
105
|
+
"args": message.args,
|
|
106
|
+
"kwargs": message.kwargs,
|
|
107
|
+
"error": repr(exception),
|
|
108
|
+
"permanent": permanent,
|
|
109
|
+
"attempts": attempts,
|
|
110
|
+
"failed_at": datetime.now(UTC).isoformat(),
|
|
111
|
+
}
|
|
112
|
+
if self._redis is not None:
|
|
113
|
+
await self._redis.xadd(
|
|
114
|
+
self._config.dlq_stream,
|
|
115
|
+
{"data": json.dumps(record, default=str)},
|
|
116
|
+
maxlen=self._config.dlq_maxlen or None,
|
|
117
|
+
approximate=True,
|
|
118
|
+
)
|
|
119
|
+
logger.warning(
|
|
120
|
+
"dead-letter",
|
|
121
|
+
task=message.task_name,
|
|
122
|
+
stream=self._config.dlq_stream,
|
|
123
|
+
permanent=permanent,
|
|
124
|
+
attempts=attempts,
|
|
125
|
+
)
|
nexus_queue/naming.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Stream/group naming and envelope label keys for the Nexus-Queue wire contract.
|
|
2
|
+
|
|
3
|
+
Single source of truth for the strings that travel on the wire. Producers and
|
|
4
|
+
consumers in any language must agree on these; the TypeScript client mirrors
|
|
5
|
+
the same constants.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
#: Wire-contract version. Bumped only on incompatible envelope changes.
|
|
13
|
+
NQ_VERSION = "1"
|
|
14
|
+
|
|
15
|
+
# ── Envelope label keys (carried in TaskiqMessage.labels) ──────────────────
|
|
16
|
+
LABEL_VERSION = "nq_v"
|
|
17
|
+
LABEL_TASK = "nq_task"
|
|
18
|
+
LABEL_TENANT = "nq_tenant"
|
|
19
|
+
LABEL_IDEM = "nq_idem"
|
|
20
|
+
LABEL_TRACE = "nq_trace"
|
|
21
|
+
LABEL_ENQUEUED_AT = "nq_enqueued_at"
|
|
22
|
+
LABEL_PRIORITY = "nq_priority"
|
|
23
|
+
|
|
24
|
+
#: Labels a conformant message MUST carry.
|
|
25
|
+
REQUIRED_LABELS: tuple[str, ...] = (
|
|
26
|
+
LABEL_VERSION,
|
|
27
|
+
LABEL_TASK,
|
|
28
|
+
LABEL_TENANT,
|
|
29
|
+
LABEL_ENQUEUED_AT,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
#: Sentinel tenant for single-tenant deployments.
|
|
33
|
+
SINGLE_TENANT = "_"
|
|
34
|
+
|
|
35
|
+
_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def validate_slug(part: str, *, kind: str) -> str:
|
|
39
|
+
"""Reject slugs that would break the ``nq:{project}:{queue}`` scheme."""
|
|
40
|
+
if not _SLUG.match(part):
|
|
41
|
+
raise ValueError(f"Invalid {kind} {part!r}: must match [a-z0-9][a-z0-9-]*")
|
|
42
|
+
return part
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def work_stream(project: str, queue: str) -> str:
|
|
46
|
+
"""Redis stream key that carries work for a queue."""
|
|
47
|
+
return f"nq:{project}:{queue}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def consumer_group(project: str, queue: str) -> str:
|
|
51
|
+
"""Consumer group name for a queue's workers."""
|
|
52
|
+
return f"nq:{project}:{queue}:cg"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def dlq_stream(project: str, queue: str) -> str:
|
|
56
|
+
"""Dead-letter stream key for a queue."""
|
|
57
|
+
return f"nq:{project}:{queue}:dlq"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def delayed_set(project: str, queue: str) -> str:
|
|
61
|
+
"""Sorted-set key holding retries until their backoff elapses."""
|
|
62
|
+
return f"nq:{project}:{queue}:delayed"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def status_stream(project: str) -> str:
|
|
66
|
+
"""Per-project status-event stream key."""
|
|
67
|
+
return f"nq:{project}:status"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def idempotency_redis_key(idem: str) -> str:
|
|
71
|
+
"""Redis key used by the idempotency middleware to dedup a message."""
|
|
72
|
+
return f"nq:idem:{idem}"
|
nexus_queue/pipeline.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Pipeline routing: enqueue the next stage of a multi-step job, propagating
|
|
2
|
+
tenant + trace from the current message so the whole pipeline is one trace.
|
|
3
|
+
|
|
4
|
+
Single-stage workers (e.g. ZP documents) don't need this; nixon's split
|
|
5
|
+
pipeline does.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from pydantic import BaseModel
|
|
13
|
+
from taskiq import AsyncBroker, TaskiqMessage
|
|
14
|
+
from taskiq.kicker import AsyncKicker
|
|
15
|
+
|
|
16
|
+
from nexus_queue.config import RuntimeConfig
|
|
17
|
+
from nexus_queue.envelope import Envelope
|
|
18
|
+
from nexus_queue.naming import LABEL_PRIORITY, LABEL_TENANT, LABEL_TRACE, SINGLE_TENANT
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PipelineRouter:
|
|
22
|
+
"""Forwards a job to its next stage, carrying the envelope context over."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, broker: AsyncBroker, config: RuntimeConfig) -> None:
|
|
25
|
+
self._broker = broker
|
|
26
|
+
self._config = config
|
|
27
|
+
|
|
28
|
+
async def forward(
|
|
29
|
+
self,
|
|
30
|
+
next_task: str,
|
|
31
|
+
payload: BaseModel,
|
|
32
|
+
*,
|
|
33
|
+
source: TaskiqMessage,
|
|
34
|
+
) -> str:
|
|
35
|
+
"""Enqueue ``next_task`` propagating tenant/trace/priority from ``source``."""
|
|
36
|
+
labels = source.labels
|
|
37
|
+
envelope = Envelope(
|
|
38
|
+
task=next_task,
|
|
39
|
+
tenant=str(labels.get(LABEL_TENANT, SINGLE_TENANT)),
|
|
40
|
+
trace=str(labels[LABEL_TRACE]) if labels.get(LABEL_TRACE) else None,
|
|
41
|
+
priority=str(labels.get(LABEL_PRIORITY, "default")),
|
|
42
|
+
)
|
|
43
|
+
kicker: AsyncKicker[Any, Any] = AsyncKicker(
|
|
44
|
+
task_name=next_task,
|
|
45
|
+
broker=self._broker,
|
|
46
|
+
labels=envelope.to_labels(),
|
|
47
|
+
)
|
|
48
|
+
task_obj = await kicker.kiq(**payload.model_dump())
|
|
49
|
+
return str(task_obj.task_id)
|
nexus_queue/ports.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Ports — the contract that makes workers movable across projects.
|
|
2
|
+
|
|
3
|
+
A handler depends only on these Protocols; each project provides the adapters
|
|
4
|
+
(e.g. ZP maps `JobStatePort` to a Payload document's ``parse_*`` fields, nixon
|
|
5
|
+
to its Postgres job state machine). Handlers never import a project's domain.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping, Sequence
|
|
11
|
+
from enum import StrEnum
|
|
12
|
+
from typing import Any, Protocol, runtime_checkable
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, Field
|
|
15
|
+
|
|
16
|
+
from nexus_queue.naming import NQ_VERSION, SINGLE_TENANT
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class JobStatus(StrEnum):
|
|
20
|
+
"""Minimal common job lifecycle. Projects may track richer states in their
|
|
21
|
+
own store; this is the subset the ports speak."""
|
|
22
|
+
|
|
23
|
+
CREATED = "created"
|
|
24
|
+
PROCESSING = "processing"
|
|
25
|
+
COMPLETED = "completed"
|
|
26
|
+
FAILED = "failed"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@runtime_checkable
|
|
30
|
+
class JobStatePort(Protocol):
|
|
31
|
+
"""Transitions the job's state in the project's system-of-record.
|
|
32
|
+
|
|
33
|
+
The producer creates the job before enqueueing; the handler drives it from
|
|
34
|
+
here. Implementations must be idempotent (``complete`` of an already
|
|
35
|
+
completed job is a no-op)."""
|
|
36
|
+
|
|
37
|
+
async def processing(self, job_id: str, *, meta: Mapping[str, Any] | None = None) -> None: ...
|
|
38
|
+
|
|
39
|
+
async def complete(self, job_id: str, *, result: Mapping[str, Any]) -> None: ...
|
|
40
|
+
|
|
41
|
+
async def fail(self, job_id: str, *, error: str, permanent: bool) -> None: ...
|
|
42
|
+
|
|
43
|
+
async def get(self, job_id: str) -> JobStatus: ...
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@runtime_checkable
|
|
47
|
+
class BlobStorePort(Protocol):
|
|
48
|
+
"""Binary storage behind an opaque reference (MinIO, Payload media, S3…)."""
|
|
49
|
+
|
|
50
|
+
async def get(self, ref: str) -> bytes: ...
|
|
51
|
+
|
|
52
|
+
async def put(self, ref: str, data: bytes, *, content_type: str) -> str: ...
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@runtime_checkable
|
|
56
|
+
class IndexPort(Protocol):
|
|
57
|
+
"""Search index upserts/deletes (Typesense in both reference projects)."""
|
|
58
|
+
|
|
59
|
+
async def upsert(self, collection: str, docs: Sequence[Mapping[str, Any]]) -> None: ...
|
|
60
|
+
|
|
61
|
+
async def delete(self, collection: str, ids: Sequence[str]) -> None: ...
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class StatusEvent(BaseModel):
|
|
65
|
+
"""Versioned status event published to ``nq:{project}:status``."""
|
|
66
|
+
|
|
67
|
+
nq_v: str = Field(default=NQ_VERSION)
|
|
68
|
+
job_id: str
|
|
69
|
+
task: str
|
|
70
|
+
tenant: str = Field(default=SINGLE_TENANT)
|
|
71
|
+
state: JobStatus
|
|
72
|
+
ts: str
|
|
73
|
+
trace: str | None = None
|
|
74
|
+
error: str | None = None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@runtime_checkable
|
|
78
|
+
class StatusEventPort(Protocol):
|
|
79
|
+
"""Emits a status event (push-status; generalizes nixon's domain-event stream)."""
|
|
80
|
+
|
|
81
|
+
async def emit(self, event: StatusEvent) -> None: ...
|
nexus_queue/publisher.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Python producer. Stamps the standard envelope labels (and the current OTel
|
|
2
|
+
traceparent for end-to-end tracing) and enqueues onto the namespaced stream.
|
|
3
|
+
|
|
4
|
+
The TypeScript producer (`@zetesis/nexus-queue`) emits the same shape.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from opentelemetry.propagate import inject
|
|
12
|
+
from pydantic import BaseModel
|
|
13
|
+
from taskiq import AsyncBroker
|
|
14
|
+
from taskiq.kicker import AsyncKicker
|
|
15
|
+
|
|
16
|
+
from nexus_queue.config import RuntimeConfig
|
|
17
|
+
from nexus_queue.envelope import Envelope
|
|
18
|
+
from nexus_queue.naming import SINGLE_TENANT
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _current_traceparent() -> str | None:
|
|
22
|
+
carrier: dict[str, str] = {}
|
|
23
|
+
inject(carrier)
|
|
24
|
+
return carrier.get("traceparent")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Publisher:
|
|
28
|
+
"""Enqueues tasks by name with the Nexus-Queue envelope."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, broker: AsyncBroker, config: RuntimeConfig) -> None:
|
|
31
|
+
self._broker = broker
|
|
32
|
+
self._config = config
|
|
33
|
+
|
|
34
|
+
async def enqueue(
|
|
35
|
+
self,
|
|
36
|
+
task: str,
|
|
37
|
+
payload: BaseModel,
|
|
38
|
+
*,
|
|
39
|
+
tenant: str = SINGLE_TENANT,
|
|
40
|
+
idempotency_key: str | None = None,
|
|
41
|
+
priority: str = "default",
|
|
42
|
+
trace: str | None = None,
|
|
43
|
+
) -> str:
|
|
44
|
+
"""Enqueue a typed payload; returns the taskiq task id."""
|
|
45
|
+
return await self.enqueue_raw(
|
|
46
|
+
task,
|
|
47
|
+
payload.model_dump(),
|
|
48
|
+
tenant=tenant,
|
|
49
|
+
idempotency_key=idempotency_key,
|
|
50
|
+
priority=priority,
|
|
51
|
+
trace=trace,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
async def enqueue_raw(
|
|
55
|
+
self,
|
|
56
|
+
task: str,
|
|
57
|
+
payload: dict[str, Any],
|
|
58
|
+
*,
|
|
59
|
+
tenant: str = SINGLE_TENANT,
|
|
60
|
+
idempotency_key: str | None = None,
|
|
61
|
+
priority: str = "default",
|
|
62
|
+
trace: str | None = None,
|
|
63
|
+
) -> str:
|
|
64
|
+
"""Enqueue a raw kwargs payload (used by the HTTP kicker)."""
|
|
65
|
+
envelope = Envelope(
|
|
66
|
+
task=task,
|
|
67
|
+
tenant=tenant,
|
|
68
|
+
idempotency_key=idempotency_key,
|
|
69
|
+
trace=trace or _current_traceparent(),
|
|
70
|
+
priority=priority,
|
|
71
|
+
)
|
|
72
|
+
kicker: AsyncKicker[Any, Any] = AsyncKicker(
|
|
73
|
+
task_name=task,
|
|
74
|
+
broker=self._broker,
|
|
75
|
+
labels=envelope.to_labels(),
|
|
76
|
+
)
|
|
77
|
+
task_obj = await kicker.kiq(**payload)
|
|
78
|
+
return str(task_obj.task_id)
|
nexus_queue/py.typed
ADDED
|
File without changes
|
nexus_queue/tracing.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Optional OpenTelemetry tracing setup for the worker.
|
|
2
|
+
|
|
3
|
+
The runtime instruments spans through the OTel *API* (see :mod:`nexus_queue.handlers`),
|
|
4
|
+
but emitting them needs an SDK + exporter. A deployment opts in by setting the
|
|
5
|
+
standard ``OTEL_EXPORTER_OTLP_ENDPOINT``; with it unset the spans stay no-op, so
|
|
6
|
+
the same image runs untraced where no collector exists.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
|
|
13
|
+
import structlog
|
|
14
|
+
from opentelemetry import trace
|
|
15
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
16
|
+
from opentelemetry.sdk.resources import Resource
|
|
17
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
18
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
19
|
+
|
|
20
|
+
from nexus_queue.config import RuntimeConfig
|
|
21
|
+
|
|
22
|
+
logger = structlog.get_logger("nexus_queue.tracing")
|
|
23
|
+
|
|
24
|
+
OTLP_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_ENDPOINT"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def configure_tracing(config: RuntimeConfig) -> None:
|
|
28
|
+
"""Wire an OTLP span exporter when a collector endpoint is configured.
|
|
29
|
+
|
|
30
|
+
No-op unless ``OTEL_EXPORTER_OTLP_ENDPOINT`` is set. The exporter reads the
|
|
31
|
+
endpoint (and any other ``OTEL_*`` options) from the environment itself."""
|
|
32
|
+
endpoint = os.environ.get(OTLP_ENDPOINT_ENV)
|
|
33
|
+
if not endpoint:
|
|
34
|
+
return
|
|
35
|
+
provider = TracerProvider(resource=Resource.create({"service.name": config.app_name}))
|
|
36
|
+
provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
|
|
37
|
+
trace.set_tracer_provider(provider)
|
|
38
|
+
logger.info("tracing-configured", endpoint=endpoint, service=config.app_name)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nexus-queue
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Nexus-Queue — portable taskiq + Redis Streams worker runtime: namespaced streams, versioned envelope, ports-and-adapters, retry/DLQ/idempotency/tracing/metrics middleware.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Zetesis-Labs/PayloadAgents
|
|
6
|
+
Project-URL: Repository, https://github.com/Zetesis-Labs/PayloadAgents
|
|
7
|
+
Project-URL: Issues, https://github.com/Zetesis-Labs/PayloadAgents/issues
|
|
8
|
+
Author: Zetesis Labs
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: dlq,ports-and-adapters,queue,redis,taskiq,worker
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Framework :: FastAPI
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.12
|
|
20
|
+
Requires-Dist: fastapi>=0.115
|
|
21
|
+
Requires-Dist: opentelemetry-api>=1.35
|
|
22
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.35
|
|
23
|
+
Requires-Dist: opentelemetry-sdk>=1.35
|
|
24
|
+
Requires-Dist: prometheus-client>=0.20
|
|
25
|
+
Requires-Dist: pydantic>=2.9
|
|
26
|
+
Requires-Dist: redis>=5.0
|
|
27
|
+
Requires-Dist: structlog>=24.1
|
|
28
|
+
Requires-Dist: taskiq-redis>=1.1.1
|
|
29
|
+
Requires-Dist: taskiq>=0.11.18
|
|
30
|
+
Requires-Dist: uvicorn[standard]>=0.34
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# nexus-queue
|
|
34
|
+
|
|
35
|
+
Portable worker runtime for the **Nexus-Queue** standard: taskiq + Redis Streams,
|
|
36
|
+
domain-agnostic, ports-and-adapters. One project builds queues the same way as
|
|
37
|
+
the next, and a worker's handlers move between projects unchanged.
|
|
38
|
+
|
|
39
|
+
What this package owns (the parts that are the *same* across projects):
|
|
40
|
+
|
|
41
|
+
- **Namespaced streams** — `nq:{project}:{queue}` (+ `:cg`, `:dlq`) and
|
|
42
|
+
`nq:{project}:status`, so multiple projects/queues coexist on one Redis
|
|
43
|
+
(the default global `"taskiq"` stream is never used).
|
|
44
|
+
- **Versioned envelope** — standard labels (`nq_v`, `nq_task`, `nq_tenant`,
|
|
45
|
+
`nq_idem`, `nq_trace`, `nq_enqueued_at`, `nq_priority`) on top of taskiq's
|
|
46
|
+
message, plus typed pydantic payloads.
|
|
47
|
+
- **Ports** — `JobStatePort`, `BlobStorePort`, `IndexPort`, `StatusEventPort`.
|
|
48
|
+
Handlers depend only on these; each project supplies the adapters
|
|
49
|
+
(e.g. Payload vs Postgres/MinIO).
|
|
50
|
+
- **Middleware stack** (broker-level): idempotency (dedup on `nq_idem`),
|
|
51
|
+
DLQ (dead-letter on retry-exhaustion instead of silent drop), retries with
|
|
52
|
+
exponential backoff + jitter, OTel tracing (`nq_trace` propagation), and
|
|
53
|
+
Prometheus metrics.
|
|
54
|
+
- **Producer + kicker** — a Python `Publisher` and a generic HTTP kicker for
|
|
55
|
+
non-Python producers. The TypeScript producer client ships as
|
|
56
|
+
`@zetesis/nexus-queue`.
|
|
57
|
+
|
|
58
|
+
Spec: `nexus-queue-spec.md`.
|
|
59
|
+
|
|
60
|
+
Released via release-please on every conventional commit to `main`
|
|
61
|
+
(scope `nexus-queue`).
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
nexus_queue/__init__.py,sha256=vZC0Kh4XgC9PJAQUu_bjDHmp7I94L97jkgJVroF_rNg,2121
|
|
2
|
+
nexus_queue/app.py,sha256=xJ58WaTLGrpor0GiSoxfiagqIvzfUN_CjvWmoSM98lY,1484
|
|
3
|
+
nexus_queue/broker.py,sha256=zqUQFOMnZ3XDs0gegWC75wcL1F02514_0WjXu8QZ1hI,1066
|
|
4
|
+
nexus_queue/config.py,sha256=34GIQsRmbR7xb1Xy2iSumZVHtkLrH0xqwqkTCFX2fGU,3979
|
|
5
|
+
nexus_queue/delayed.py,sha256=FCDc88C1GujhH3yB9eEWAaLlv3JDgvgFBJRpM4iHRXU,3227
|
|
6
|
+
nexus_queue/envelope.py,sha256=7k81IrrPBijoDXp8uE60Fh4J57Fpetwc8t-3TRI6XOI,2069
|
|
7
|
+
nexus_queue/exceptions.py,sha256=ceB2nn6jHft_y32Qmmoq40hAgJ7mto3jWILNf2MogJE,475
|
|
8
|
+
nexus_queue/handlers.py,sha256=VWR3FRkalbEPzHbvg6JqaIjA3vjM2uokq5ONr_gGsgs,3254
|
|
9
|
+
nexus_queue/kicker.py,sha256=hwbCfd-vyxFsDPghaF8dBg5lVqYkwHWQftnDngKGhVQ,3648
|
|
10
|
+
nexus_queue/lifecycle.py,sha256=-H9Z-VTYab1CeLGz0iMuQBBxjyNr5I-1xd6G0gaoTEs,4042
|
|
11
|
+
nexus_queue/naming.py,sha256=uAkfFHT-DZMtPL3uKVt5gSSfZZ0yVxBDXXc-rzWdN3U,2108
|
|
12
|
+
nexus_queue/pipeline.py,sha256=L6ytmQhbCSEIRCe_VUqyMa8v_kfJrJNQxlONHRu0JLw,1647
|
|
13
|
+
nexus_queue/ports.py,sha256=wnRavsKSS4JReHhc5MfOTL-dtOxW08U5mQYn8CISjzA,2545
|
|
14
|
+
nexus_queue/publisher.py,sha256=zDgSVvwI4XIJtsEZrcCufEReOtOUOmHmb8zs_hCt6oM,2283
|
|
15
|
+
nexus_queue/tracing.py,sha256=lNSxUTAOO7VJBkOKP7EX0ns0DBWP-8nXdKqrieiF2A4,1520
|
|
16
|
+
nexus_queue/middleware/__init__.py,sha256=7dAc7nzCLrfOsZQOXYLOhxHSW7HUXbQ7c4zpZ2w8Q8M,276
|
|
17
|
+
nexus_queue/middleware/metrics.py,sha256=Dah5GrUSs2VJWNK5P7sSV1dyh4esEfr510tDASO3i64,1875
|
|
18
|
+
nexus_queue/middleware/retry_dlq.py,sha256=zAzdrMhzJjYyUsdntTtL3Jc6LmcrkzfI-U9Gcj8HPWM,4361
|
|
19
|
+
nexus_queue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
+
nexus_queue-0.1.1.dist-info/METADATA,sha256=nuIq6JILH-c3mCN93ND1TBfqb4QK62I6KFf6a_syLqA,2821
|
|
21
|
+
nexus_queue-0.1.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
22
|
+
nexus_queue-0.1.1.dist-info/RECORD,,
|