async-task-api-worker 0.1.0rc2__py3-none-any.whl

async_task_api_worker/observability/health.py

@@ -0,0 +1,299 @@
+"""The probe and metrics HTTP server: a small asyncio HTTP/1.1 server on one port.
+
+- ``/health``: liveness. It consults no dependency, on purpose: a failing liveness probe restarts the container, and
+  tying it to a remote dependency would restart every worker at the first outage of that dependency. What it proves is
+  that the event loop answers and that no task is stuck past its deadline plus ``WORKER_TASK_OVERRUN_GRACE_S``.
+- ``/ready``: readiness, "can I take work now?". The broker connection, the shutdown state and the probes the worker
+  registered, run in parallel under ``WORKER_PROBE_TIMEOUT_S``. Failing it removes the pod from the Service (or keeps a
+  Job pending) without killing it.
+- ``/metrics`` and ``/internal/metrics``: the Prometheus exposition. ``/__version__``: the image's version document.
+
+Every error body is ``{"status": "error", "error": {"code", "message"}}``, the API's format.
+"""
+
+import asyncio
+import contextlib
+import json
+import logging
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass
+from enum import StrEnum
+from http import HTTPStatus
+from pathlib import Path
+from typing import Final
+
+from async_task_api_worker.errors import ProbeError
+from async_task_api_worker.protocol.json_types import JsonObject
+
+_LOGGER: Final[logging.Logger] = logging.getLogger("async_task_api_worker")
+
+# A request line and its headers fit very largely in this: beyond, it is not a probe and the connection is closed.
+MAX_HEADER_BYTES: Final[int] = 8192
+# A client that connects and never finishes its request, or never reads the answer, must not hold a connection
+# forever: reading the request, writing the answer and closing are each bounded by this.
+IO_TIMEOUT_S: Final[float] = 5.0
+# Probes come from the kubelet and a few scrapers. Beyond this many open connections, new ones are closed at once, so
+# that idle clients cannot exhaust the process's file descriptors, which the broker and the dependencies need too.
+MAX_CONNECTIONS: Final[int] = 32
+_MAX_DETAIL_LENGTH: Final[int] = 200
+_METHOD: Final[str] = "GET"
+_REQUEST_LINE_MIN_PARTS: Final[int] = 2  # "METHOD target"
+_HEADER_END: Final[bytes] = b"\r\n\r\n"
+_JSON_CONTENT_TYPE: Final[str] = "application/json; charset=utf-8"
+DEFAULT_VERSION_FILE: Final[Path] = Path("/app/version.json")
+_DEV_VERSION: Final[JsonObject] = {
+    "source": "",
+    "version": "dev",
+    "commit": "",
+    "build": "",
+    "code_date": "",
+    "changes": [],
+    "history": [],
+}
+
+type Probe = Callable[[], Awaitable[None]]
+type Body = JsonObject | tuple[bytes, str]
+
+
+class Status(StrEnum):
+    OK = "ok"
+    ERROR = "error"
+
+
+class Route(StrEnum):
+    HEALTH = "/health"
+    READY = "/ready"
+    METRICS = "/metrics"
+    INTERNAL_METRICS = "/internal/metrics"
+    VERSION = "/__version__"
+
+
+class HealthErrorCode(StrEnum):
+    TASK_OVERRUN = "TASK_OVERRUN"
+    DEPENDENCIES_NOT_READY = "DEPENDENCIES_NOT_READY"
+    NOT_FOUND = "NOT_FOUND"
+    METHOD_NOT_ALLOWED = "METHOD_NOT_ALLOWED"
+    PROBE_FAILED = "PROBE_FAILED"
+
+
+class Component(StrEnum):
+    """Components the SDK reports itself, next to the worker's own probes."""
+
+    BROKER = "broker"
+    DRAINING = "draining"
+
+
+@dataclass(frozen=True, slots=True)
+class RuntimeState:
+    """What the server asks the running worker, through callables so that it knows nothing of the runtime."""
+
+    ping_broker: Callable[[float], Awaitable[bool]]
+    is_draining: Callable[[], bool]
+    overrunning: Callable[[], tuple[str, ...]]
+    metrics: Callable[[], tuple[bytes, str]]
+
+
+@dataclass(frozen=True, slots=True)
+class ComponentState:
+    status: Status
+    details: str | None = None
+
+    def as_json(self) -> JsonObject:
+        return {"status": self.status.value, "details": self.details}
+
+
+def _error(code: HealthErrorCode, message: str, components: JsonObject | None = None) -> JsonObject:
+    error: JsonObject = {"code": code.value, "message": message}
+    if components is not None:
+        error["components"] = components
+    return {"status": Status.ERROR.value, "error": error}
+
+
+def _detail_of(error: Exception) -> str:
+    """Safe text for a probe failure: a :class:`ProbeError` message as is, any other exception by class name only."""
+    if isinstance(error, ProbeError) and (text := " ".join(str(error).split())):
+        return text[:_MAX_DETAIL_LENGTH]
+    return type(error).__name__
+
+
+async def run_probe(name: str, probe: Probe, timeout_s: float) -> ComponentState:
+    """Run one probe. It never raises: a failing dependency must not take the probe server down."""
+    try:
+        await asyncio.wait_for(probe(), timeout_s)
+    except TimeoutError:
+        _LOGGER.warning("Readiness probe %r timed out after %g s.", name, timeout_s)
+        return ComponentState(Status.ERROR, f"Timed out after {timeout_s:g} s")
+    except Exception as error:
+        _LOGGER.warning("Readiness probe %r failed: %s.", name, type(error).__name__)
+        return ComponentState(Status.ERROR, _detail_of(error))
+    return ComponentState(Status.OK)
+
+
+class HealthServer:
+    def __init__(
+        self,
+        *,
+        host: str,
+        port: int,
+        state: RuntimeState,
+        probes: Mapping[str, Probe],
+        probe_timeout_s: float,
+        version_file: Path = DEFAULT_VERSION_FILE,
+        max_connections: int = MAX_CONNECTIONS,
+        io_timeout_s: float = IO_TIMEOUT_S,
+    ) -> None:
+        self._host = host
+        self._port = port
+        self._state = state
+        self._probes = dict(probes)
+        self._probe_timeout_s = probe_timeout_s
+        self._version_file = version_file
+        self._max_connections = max_connections
+        self._io_timeout_s = io_timeout_s
+        self._connections = 0
+        self._readiness_evaluation: asyncio.Future[tuple[HTTPStatus, Body]] | None = None
+        self._server: asyncio.Server | None = None
+
+    @property
+    def port(self) -> int:
+        """The port actually bound (useful when 0 was asked for)."""
+        assert self._server is not None
+        return self._server.sockets[0].getsockname()[1]
+
+    async def start(self) -> None:
+        self._server = await asyncio.start_server(self._handle, self._host, self._port, limit=MAX_HEADER_BYTES)
+        _LOGGER.info("Probes and metrics are served on %s:%d.", self._host, self.port)
+
+    async def stop(self) -> None:
+        if self._server is None:
+            return
+        self._server.close()
+        # The probe server stops last, after the drain: a client that does not read its answer must not hold the
+        # shutdown, so what is still connected is cut at once.
+        self._server.abort_clients()
+        if self._readiness_evaluation is not None:
+            self._readiness_evaluation.cancel()
+        await self._server.wait_closed()
+        self._server = None
+
+    async def _readiness(self) -> tuple[HTTPStatus, Body]:
+        """Single flight: requests that arrive while an evaluation runs share it, instead of each calling the probes."""
+        if self._readiness_evaluation is None:
+            evaluation = asyncio.ensure_future(self._evaluate_readiness())
+            evaluation.add_done_callback(self._forget_evaluation)
+            self._readiness_evaluation = evaluation
+        return await asyncio.shield(self._readiness_evaluation)
+
+    def _forget_evaluation(self, _: object) -> None:
+        self._readiness_evaluation = None
+
+    async def _evaluate_readiness(self) -> tuple[HTTPStatus, Body]:
+        if self._state.is_draining():
+            components: dict[str, ComponentState] = {
+                Component.DRAINING: ComponentState(Status.ERROR, "The worker no longer takes deliveries"),
+            }
+        else:
+            names = sorted(self._probes)
+            results = await asyncio.gather(
+                self._broker_state(),
+                *(run_probe(name, self._probes[name], self._probe_timeout_s) for name in names),
+            )
+            components = dict(zip([Component.BROKER.value, *names], results, strict=True))
+        unavailable = [name for name, component in components.items() if component.status is not Status.OK]
+        rendered: JsonObject = {name: component.as_json() for name, component in components.items()}
+        if unavailable:
+            message = f"Dependencies not ready: {', '.join(unavailable)}."
+            return HTTPStatus.SERVICE_UNAVAILABLE, _error(HealthErrorCode.DEPENDENCIES_NOT_READY, message, rendered)
+        return HTTPStatus.OK, {"status": Status.OK.value, "components": rendered}
+
+    async def _broker_state(self) -> ComponentState:
+        try:
+            connected = await asyncio.wait_for(self._state.ping_broker(self._probe_timeout_s), self._probe_timeout_s)
+        except Exception:
+            connected = False
+        return ComponentState(Status.OK) if connected else ComponentState(Status.ERROR, "Not connected")
+
+    def _liveness(self) -> tuple[HTTPStatus, Body]:
+        if self._state.overrunning():
+            message = "A task is still running past its deadline."
+            return HTTPStatus.SERVICE_UNAVAILABLE, _error(HealthErrorCode.TASK_OVERRUN, message)
+        return HTTPStatus.OK, {"status": Status.OK.value}
+
+    def _version(self) -> JsonObject:
+        try:
+            document = json.loads(self._version_file.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return dict(_DEV_VERSION)
+        return document if isinstance(document, dict) else dict(_DEV_VERSION)
+
+    async def _route(self, method: str, path: str) -> tuple[HTTPStatus, Body]:
+        if method != _METHOD:
+            return HTTPStatus.METHOD_NOT_ALLOWED, _error(
+                HealthErrorCode.METHOD_NOT_ALLOWED,
+                f"Method {method!r} is not allowed.",
+            )
+        match path:
+            case Route.HEALTH:
+                return self._liveness()
+            case Route.READY:
+                return await self._readiness()
+            case Route.METRICS | Route.INTERNAL_METRICS:
+                return HTTPStatus.OK, self._state.metrics()
+            case Route.VERSION:
+                return HTTPStatus.OK, self._version()
+        return HTTPStatus.NOT_FOUND, _error(HealthErrorCode.NOT_FOUND, f"Path {path!r} is unknown.")
+
+    async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+        if self._connections >= self._max_connections:
+            writer.close()  # over the bound: not even read
+            return
+        self._connections += 1
+        try:
+            try:
+                head = await asyncio.wait_for(reader.readuntil(_HEADER_END), self._io_timeout_s)
+            except (asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError, TimeoutError):
+                return  # truncated, oversized or too slow: not a probe, close without a word
+            method, path = _request_line(head)
+            try:
+                status, body = await self._route(method, path)
+            except Exception:
+                _LOGGER.exception("The probe server failed to answer %s.", path)
+                status = HTTPStatus.INTERNAL_SERVER_ERROR
+                body = _error(HealthErrorCode.PROBE_FAILED, "The probe server failed.")
+            writer.write(_response(status, body))
+            with contextlib.suppress(ConnectionError, TimeoutError):
+                await asyncio.wait_for(writer.drain(), self._io_timeout_s)
+        finally:
+            self._connections -= 1
+            await self._close(writer)
+
+    async def _close(self, writer: asyncio.StreamWriter) -> None:
+        """Close, and abort what a client that does not read keeps from flushing: the server must be able to stop."""
+        writer.close()
+        with contextlib.suppress(ConnectionError):
+            try:
+                await asyncio.wait_for(writer.wait_closed(), self._io_timeout_s)
+            except TimeoutError:
+                writer.transport.abort()
+
+
+def _request_line(head: bytes) -> tuple[str, str]:
+    """Method and path of a request. An unreadable line gives an empty method, so a 405; the query string is dropped."""
+    parts = head.split(b"\r\n", 1)[0].decode("latin-1").split()
+    if len(parts) < _REQUEST_LINE_MIN_PARTS:
+        return "", ""
+    return parts[0].upper(), parts[1].split("?", 1)[0]
+
+
+def _response(status: HTTPStatus, body: Body) -> bytes:
+    if isinstance(body, tuple):
+        payload, content_type = body
+    else:
+        payload, content_type = json.dumps(body, ensure_ascii=False).encode(), _JSON_CONTENT_TYPE
+    head = (
+        f"HTTP/1.1 {status.value} {status.phrase}\r\n"
+        f"Content-Type: {content_type}\r\n"
+        f"Content-Length: {len(payload)}\r\n"
+        "Connection: close\r\n\r\n"
+    )
+    return head.encode() + payload

async_task_api_worker/observability/logging.py

@@ -0,0 +1,89 @@
+"""JSON logging: one object per line on stdout, so that a log collector parses it as is."""
+
+import json
+import logging
+import sys
+from datetime import UTC, datetime
+from typing import Final
+
+from async_task_api_worker._runtime.log_context import current_task
+from async_task_api_worker.settings import LogLevel
+
+_ROOT_LOGGER: Final[str] = ""
+_STANDARD_RECORD_FIELDS: Final[frozenset[str]] = frozenset(
+    {*vars(logging.LogRecord("", 0, "", 0, "", (), None)), "message", "asctime", "taskName"},
+)
+_FIXED_FIELDS: Final[tuple[str, ...]] = ("ts", "level", "logger", "msg")
+_EXCEPTION_FIELD: Final[str] = "exc"
+_HANDLER_MARKER: Final[str] = "_async_task_api_worker_handler"
+# aio-pika logs two ERROR lines from its channel callbacks when a connection drops, then reconnects normally: a broker
+# blip is not an error of the worker (a lost connection that does not come back surfaces through the probes).
+_RECONNECT_NOISE_LOGGER: Final[str] = "aio_pika.tools"
+_RECONNECT_CALLBACK: Final[str] = "RobustChannel._on_close"
+
+
+class TaskContextFilter(logging.Filter):
+    """Put the task being processed on the records that do not carry one: library logs inside a handler get it too."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        task = current_task.get()
+        if task is not None and not hasattr(record, "task_id"):
+            record.task_id = task.task_id
+            record.service = task.service
+            record.attempt = task.attempt
+        return True
+
+
+class DemoteReconnectErrorsFilter(logging.Filter):
+    """Log the errors of a channel's close callback, which a dropped connection produces, as warnings.
+
+    Only those records: aio-pika logs the two lines of a reconnect from its callback collections, naming the callback
+    (``RobustChannel._on_close``); any other error of that logger keeps its level.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if record.levelno == logging.ERROR and _RECONNECT_CALLBACK in record.getMessage():
+            record.levelno = logging.WARNING
+            record.levelname = logging.getLevelName(logging.WARNING)
+        return True
+
+
+class JsonFormatter(logging.Formatter):
+    """``ts``, ``level``, ``logger``, ``msg``, then every field bound to the record (``task_id``, ``service``,
+    ``attempt``, the decision fields...), then ``exc`` with the traceback when there is one.
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        entry: dict[str, object] = {
+            "ts": datetime.fromtimestamp(record.created, UTC).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "msg": record.getMessage(),
+        }
+        for name, value in vars(record).items():
+            if name not in _STANDARD_RECORD_FIELDS and name not in _FIXED_FIELDS:
+                entry[name] = value
+        if record.exc_info:
+            entry[_EXCEPTION_FIELD] = self.formatException(record.exc_info)
+        return json.dumps(entry, default=str, ensure_ascii=False)
+
+
+def configure_logging(level: LogLevel) -> None:
+    """Send every log record of the process to stdout as JSON, at ``level``.
+
+    The root logger belongs to the application: if it already has a handler of its own (the application configured
+    logging before running the worker), that handler is kept and none is added, so that no line is printed twice,
+    and only the level is applied (the task fields of libraries' logs come from a filter on the SDK's own handler).
+    When the SDK installs its handler it also logs the errors of the channel close callbacks aio-pika emits while a
+    dropped connection reconnects (logger ``aio_pika.tools``) as warnings. Calling this twice installs the handler once.
+    """
+    root = logging.getLogger(_ROOT_LOGGER)
+    root.setLevel(level.value)
+    if root.handlers:
+        return
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(JsonFormatter())
+    handler.addFilter(TaskContextFilter())
+    setattr(handler, _HANDLER_MARKER, True)
+    root.addHandler(handler)
+    logging.getLogger(_RECONNECT_NOISE_LOGGER).addFilter(DemoteReconnectErrorsFilter())

async_task_api_worker/observability/loop_lag.py

@@ -0,0 +1,56 @@
+"""Event loop lag probe.
+
+A synchronous call of more than a few milliseconds made from a coroutine blocks the loop: nothing else progresses, the
+liveness probe stops answering and the kubelet kills the pod, with no visible cause. The lag is the one signal that
+rises before the break. The probe sleeps a known interval and observes the overshoot when it wakes up: a free loop
+overshoots by about a millisecond, a blocked one by what is left of the blocking call.
+"""
+
+import asyncio
+import contextlib
+import time
+from collections.abc import Callable
+from typing import Final
+
+from async_task_api_worker._runtime.progress import Clock, Sleep
+
+# Shorter would add nothing: the probe looks for blockings of at least a hundred milliseconds, and measures the
+# overshoot, not the moment of the blocking.
+DEFAULT_INTERVAL_S: Final[float] = 1.0
+
+type Observe = Callable[[float], object]
+
+
+class EventLoopLagProbe:
+    def __init__(
+        self,
+        observe: Observe,
+        *,
+        interval_s: float = DEFAULT_INTERVAL_S,
+        clock: Clock = time.perf_counter,
+        sleep: Sleep = asyncio.sleep,
+    ) -> None:
+        self._observe = observe
+        self._interval_s = interval_s
+        self._clock = clock
+        self._sleep = sleep
+        self._task: asyncio.Task[None] | None = None
+
+    def start(self) -> None:
+        if self._task is None:
+            self._task = asyncio.get_running_loop().create_task(self._loop())
+
+    async def stop(self) -> None:
+        if self._task is None:
+            return
+        self._task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await self._task
+        self._task = None
+
+    async def _loop(self) -> None:
+        while True:
+            started = self._clock()
+            await self._sleep(self._interval_s)
+            # Clock granularity can make the overshoot slightly negative: observed as is it would skew the first bucket.
+            self._observe(max(self._clock() - started - self._interval_s, 0.0))

async_task_api_worker/observability/metrics.py

@@ -0,0 +1,188 @@
+"""Prometheus metrics of a worker.
+
+One registry per worker process, of its own (never the default one: a library must not write to a global namespace).
+No task, client or service label: the scrape adds ``service_name`` and ``pod`` itself.
+
+The engine emits events to a :class:`RuntimeObserver`; :class:`PrometheusObserver` turns them into series, so the engine
+never imports this module.
+"""
+
+from collections.abc import Callable
+from typing import Final
+
+from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Counter, Gauge, Histogram, generate_latest
+from prometheus_client.gc_collector import GCCollector
+from prometheus_client.platform_collector import PlatformCollector
+from prometheus_client.process_collector import ProcessCollector
+
+from async_task_api_worker._runtime.observer import MessageType, ProgressDropReason, TaskStatus
+from async_task_api_worker._version import __version__
+from async_task_api_worker.protocol.envelope import PoisonReason
+from async_task_api_worker.protocol.version import PROTOCOL_VERSION
+
+_MS: Final[float] = 0.001
+_S: Final[float] = 1.0
+_MINUTE: Final[float] = 60.0
+_INF: Final[float] = float("inf")
+
+# The same buckets as the listener measures the interval between the two messages of a task with: the two views are
+# comparable bucket by bucket. A test at the repository root forbids drift.
+TASK_DURATION_BUCKETS: Final[tuple[float, ...]] = (
+    0.005 * _S,
+    0.01 * _S,
+    0.025 * _S,
+    0.05 * _S,
+    0.1 * _S,
+    0.15 * _S,
+    0.2 * _S,
+    0.25 * _S,
+    0.3 * _S,
+    0.4 * _S,
+    0.5 * _S,
+    1.0 * _S,
+    2.5 * _S,
+    5.0 * _S,
+    10.0 * _S,
+    30.0 * _S,
+    1.0 * _MINUTE,
+    2.0 * _MINUTE,
+    5.0 * _MINUTE,
+    _INF,
+)
+
+# The same buckets as the API, so that the event loop lag of every component overlays on one dashboard.
+EVENT_LOOP_LAG_BUCKETS: Final[tuple[float, ...]] = (
+    1.0 * _MS,
+    5.0 * _MS,
+    10.0 * _MS,
+    50.0 * _MS,
+    100.0 * _MS,
+    250.0 * _MS,
+    500.0 * _MS,
+    1.0 * _S,
+    2.5 * _S,
+    5.0 * _S,
+    _INF,
+)
+
+LOOP_LAG_COMPONENT: Final[str] = "worker"
+
+
+class WorkerMetrics:
+    """The series of one worker, and the observer that feeds them."""
+
+    def __init__(self) -> None:
+        self.registry: Final[CollectorRegistry] = CollectorRegistry()
+        ProcessCollector(registry=self.registry)
+        PlatformCollector(registry=self.registry)
+        GCCollector(registry=self.registry)
+        registry = self.registry
+        self.tasks = Counter("worker_tasks_total", "Tasks concluded, by outcome.", ["status"], registry=registry)
+        self.task_duration = Histogram(
+            "worker_task_duration_seconds",
+            "Time from taking a delivery to its conclusion.",
+            buckets=TASK_DURATION_BUCKETS,
+            registry=registry,
+        )
+        self.messages_published = Counter(
+            "worker_messages_published_total",
+            "Messages published to the output queue, by type.",
+            ["message_type"],
+            registry=registry,
+        )
+        self.loop_lag = Histogram(
+            "event_loop_lag_seconds",
+            "Overshoot observed when a wait of a known interval wakes up.",
+            ["component"],
+            buckets=EVENT_LOOP_LAG_BUCKETS,
+            registry=registry,
+        ).labels(component=LOOP_LAG_COMPONENT)
+        self.task_failures = Counter(
+            "worker_task_failures_total",
+            "Tasks concluded as a failure, by error code.",
+            ["code"],
+            registry=registry,
+        )
+        self.task_retries = Counter(
+            "worker_task_retries_total",
+            "Deliveries requeued after a retryable error.",
+            registry=registry,
+        )
+        self.poison_messages = Counter(
+            "worker_poison_messages_total",
+            "Undecodable deliveries, acked and dropped, by reason.",
+            ["reason"],
+            registry=registry,
+        )
+        self.publish_failures = Counter(
+            "worker_publish_failures_total",
+            "Publishes that failed, by message type.",
+            ["message_type"],
+            registry=registry,
+        )
+        self.ack_failures = Counter(
+            "worker_ack_failures_total",
+            "Acks that failed after the result was published.",
+            registry=registry,
+        )
+        self.in_flight = Gauge("worker_tasks_in_flight", "Tasks being processed right now.", registry=registry)
+        self.shutdown_requeued = Counter(
+            "worker_shutdown_requeued_total",
+            "Deliveries requeued because the worker was shutting down.",
+            registry=registry,
+        )
+        self.progress_dropped = Counter(
+            "worker_progress_dropped_total",
+            "Progress updates not published as such, by reason.",
+            ["reason"],
+            registry=registry,
+        )
+        Gauge(
+            "worker_sdk_info",
+            "Constant 1, labelled with the SDK and protocol versions.",
+            ["sdk_version", "protocol_version"],
+            registry=registry,
+        ).labels(sdk_version=__version__, protocol_version=PROTOCOL_VERSION).set(1)
+        self.observer: Final[PrometheusObserver] = PrometheusObserver(self)
+
+    def track_in_flight(self, count: Callable[[], int]) -> None:
+        """Read the in-flight gauge from ``count`` at scrape time."""
+        self.in_flight.set_function(lambda: float(count()))
+
+    def exposition(self) -> tuple[bytes, str]:
+        """The body of ``/metrics`` and its content type."""
+        return generate_latest(self.registry), CONTENT_TYPE_LATEST
+
+
+class PrometheusObserver:
+    """A :class:`RuntimeObserver` that increments the series of a :class:`WorkerMetrics`."""
+
+    def __init__(self, metrics: WorkerMetrics) -> None:
+        self._metrics = metrics
+
+    def poison(self, reason: PoisonReason) -> None:
+        self._metrics.poison_messages.labels(reason=reason.value).inc()
+
+    def task_retried(self) -> None:
+        self._metrics.task_retries.inc()
+
+    def task_concluded(self, status: TaskStatus, code: str | None, duration_s: float) -> None:
+        self._metrics.tasks.labels(status=status.value).inc()
+        self._metrics.task_duration.observe(duration_s)
+        if code is not None:
+            self._metrics.task_failures.labels(code=code).inc()
+
+    def message_published(self, message_type: MessageType) -> None:
+        self._metrics.messages_published.labels(message_type=message_type.value).inc()
+
+    def publish_failed(self, message_type: MessageType) -> None:
+        self._metrics.publish_failures.labels(message_type=message_type.value).inc()
+
+    def ack_failed(self) -> None:
+        self._metrics.ack_failures.inc()
+
+    def progress_dropped(self, reason: ProgressDropReason) -> None:
+        self._metrics.progress_dropped.labels(reason=reason.value).inc()
+
+    def shutdown_requeued(self) -> None:
+        self._metrics.shutdown_requeued.inc()

async_task_api_worker/protocol/__init__.py

@@ -0,0 +1,41 @@
+"""Wire protocol: inbound envelope decoding and outbound message encoding."""
+
+from async_task_api_worker.protocol.envelope import (
+    EnvelopeContext,
+    InputRef,
+    PoisonDelivery,
+    PoisonReason,
+    SubmissionData,
+    TaskEnvelope,
+    decode_envelope,
+)
+from async_task_api_worker.protocol.messages import (
+    ErrorDetail,
+    FailureData,
+    InvalidProgressError,
+    MessageEncoder,
+    OutMessage,
+    ProgressData,
+    StartedData,
+    SuccessData,
+)
+from async_task_api_worker.protocol.version import PROTOCOL_VERSION
+
+__all__ = [
+    "PROTOCOL_VERSION",
+    "EnvelopeContext",
+    "ErrorDetail",
+    "FailureData",
+    "InputRef",
+    "InvalidProgressError",
+    "MessageEncoder",
+    "OutMessage",
+    "PoisonDelivery",
+    "PoisonReason",
+    "ProgressData",
+    "StartedData",
+    "SubmissionData",
+    "SuccessData",
+    "TaskEnvelope",
+    "decode_envelope",
+]