plone.observability 1.0.0__py3-none-any.whl

plone/observability/auth.py

@@ -0,0 +1,28 @@
+"""Authentication detection for request metrics and tracing (no OTel import)."""
+
+from AccessControl.SecurityManagement import getSecurityManager
+
+
+ENVIRON_KEY = "plone.observability.authenticated"
+
+
+def get_auth_info():
+    """Return (authenticated: bool, user_id: str | None) for the current context.
+
+    Uses CMFCore's anonymous semantics: a request is anonymous when there is no
+    user or the user's name is the Zope/PAS "Anonymous User".
+    """
+    user = getSecurityManager().getUser()
+    if user is None or user.getUserName() == "Anonymous User":
+        return False, None
+    return True, user.getId()
+
+
+def capture_auth(event):
+    """Pubevent subscriber: stash the auth boolean on the request environ.
+
+    Runs while the security context is live (IPubSuccess/IPubFailure) so the
+    WSGI middleware can read it after publication teardown.
+    """
+    authenticated, _ = get_auth_info()
+    event.request.environ[ENVIRON_KEY] = authenticated

plone/observability/configure.zcml

@@ -0,0 +1,30 @@
+<configure
+    xmlns="http://namespaces.zope.org/zope"
+    xmlns:zcml="http://namespaces.zope.org/zcml"
+    i18n_domain="plone.observability"
+    >
+
+  <include package=".health" />
+  <include package=".metrics" />
+
+  <include
+      package=".otel"
+      zcml:condition="installed opentelemetry.sdk"
+      />
+
+  <subscriber
+      for="ZPublisher.interfaces.IPubSuccess"
+      handler=".auth.capture_auth"
+      />
+
+  <subscriber
+      for="ZPublisher.interfaces.IPubFailure"
+      handler=".auth.capture_auth"
+      />
+
+  <subscriber
+      for="ZPublisher.interfaces.IPubBeforeAbort"
+      handler=".metrics.providers.conflict.on_pub_before_abort"
+      />
+
+</configure>

plone/observability/health/checks.py

@@ -0,0 +1,31 @@
+from plone.observability.interfaces import IReadinessCheck
+from zope.interface import implementer
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+@implementer(IReadinessCheck)
+class ZODBReadinessCheck:
+    """Verifies that the ZODB is accessible by opening a connection and reading the root."""
+
+    name = "zodb"
+
+    def __init__(self):
+        self.db = None  # Set during startup
+
+    def __call__(self):
+        if self.db is None:
+            return False, "No database reference available"
+        try:
+            conn = self.db.open()
+            try:
+                conn.root()
+                return True, "ZODB connection ok"
+            finally:
+                conn.close()
+        except Exception as e:
+            logger.warning("ZODB readiness check failed: %s", e)
+            return False, f"ZODB connection failed: {e}"

plone/observability/health/configure.zcml

@@ -0,0 +1,9 @@
+<configure xmlns="http://namespaces.zope.org/zope">
+
+  <utility
+      factory=".checks.ZODBReadinessCheck"
+      provides="plone.observability.interfaces.IReadinessCheck"
+      name="zodb"
+      />
+
+</configure>

plone/observability/health/server.py

@@ -0,0 +1,146 @@
+from http.server import BaseHTTPRequestHandler
+from http.server import HTTPServer
+from plone.observability.interfaces import ILivenessCheck
+from plone.observability.interfaces import IReadinessCheck
+from socketserver import ThreadingMixIn
+from zope.component import getUtilitiesFor
+
+import json
+import logging
+import os
+import sys
+import threading
+
+
+logger = logging.getLogger(__name__)
+
+
+class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
+    """HTTPServer that handles each request in a new thread."""
+
+    daemon_threads = True
+
+    def handle_error(self, request, client_address):
+        exc = sys.exc_info()[1]
+        if isinstance(exc, (BrokenPipeError, ConnectionResetError)):
+            # Probe client disconnected before we finished writing the
+            # response (common during warmup when readiness returns 503).
+            # Harmless, so do not dump a traceback.
+            logger.debug("Health probe connection dropped from %s", client_address)
+            return
+        super().handle_error(request, client_address)
+
+
+class HealthRequestHandler(BaseHTTPRequestHandler):
+    """HTTP handler for health probe endpoints."""
+
+    def do_GET(self):
+        if self.path == "/live":
+            self._handle_live()
+        elif self.path == "/ready":
+            self._handle_ready()
+        elif self.path == "/startup":
+            self._handle_startup()
+        else:
+            self.send_error(404)
+
+    def _run_checks(self, interface):
+        checks = {}
+        ok = True
+        for name, check in getUtilitiesFor(interface):
+            check_ok, message = check()
+            checks[name] = {"ok": check_ok, "message": message}
+            if not check_ok:
+                ok = False
+        return ok, checks
+
+    def _handle_live(self):
+        ok, checks = self._run_checks(ILivenessCheck)
+        self._send_json(
+            200 if ok else 503, {"status": "ok" if ok else "failed", "checks": checks}
+        )
+
+    def _handle_ready(self):
+        ok, checks = self._run_checks(IReadinessCheck)
+        if ok:
+            self.server.health_server._started = True
+        self._send_json(
+            200 if ok else 503, {"status": "ok" if ok else "failed", "checks": checks}
+        )
+
+    def _handle_startup(self):
+        # Determine readiness ourselves and latch on first success. Kubernetes
+        # does not run the readiness probe until the startup probe has already
+        # succeeded, so /startup cannot wait for /ready to have been polled --
+        # it must evaluate the readiness checks itself. Once started it stays
+        # green (startup semantics: "did the process finish booting").
+        hs = self.server.health_server
+        if not hs._started:
+            ok, _ = self._run_checks(IReadinessCheck)
+            if ok:
+                hs._started = True
+        started = hs._started
+        self._send_json(
+            200 if started else 503,
+            {"status": "ok" if started else "starting"},
+        )
+
+    def _send_json(self, status, data):
+        body = json.dumps(data).encode("utf-8")
+        try:
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        except (BrokenPipeError, ConnectionResetError):
+            # Probe client disconnected before we finished writing (common
+            # during warmup when readiness returns 503). Harmless.
+            logger.debug("Health probe client disconnected early")
+
+    def log_message(self, format, *args):  # noqa: A002 (stdlib override signature)
+        # Suppress default stderr logging
+        logger.debug(format, *args)
+
+
+class HealthServer:
+    """Manages the health check HTTP server on a separate daemon thread."""
+
+    def __init__(self, host=None, port=None):
+        self.host = host or os.environ.get("PLONE_OBSERVABILITY_HEALTH_HOST", "0.0.0.0")
+        self.port = port or int(
+            os.environ.get("PLONE_OBSERVABILITY_HEALTH_PORT", "8081")
+        )
+        self._started = False
+        self._thread = None
+        self._httpd = None
+        self.db = None  # Set during startup to hold ZODB Database reference
+
+    def start(self):
+        if self.port == 0:
+            logger.info("Health server disabled (port=0)")
+            return
+        try:
+            self._httpd = ThreadingHTTPServer(
+                (self.host, self.port), HealthRequestHandler
+            )
+        except OSError as exc:
+            logger.error(
+                "Health server could not bind %s:%s: %s", self.host, self.port, exc
+            )
+            self._httpd = None
+            return
+        self._httpd.health_server = self
+        self._thread = threading.Thread(
+            target=self._httpd.serve_forever,
+            name="plone.observability.health",
+            daemon=True,
+        )
+        self._thread.start()
+        logger.info("Health server started on %s:%s", self.host, self.port)
+
+    def stop(self):
+        if self._httpd is not None:
+            self._httpd.shutdown()
+            self._httpd = None
+        self._thread = None

plone/observability/health/wsgi.py

@@ -0,0 +1,30 @@
+"""Serving-only startup of the health server via a PasteDeploy filter.
+
+The WSGI pipeline (zope.ini) is only built when the server actually serves;
+zconsole/scripts load zope.conf via make_wsgi_app and never build the pipeline.
+So this filter is a serving-only hook and never runs under zconsole.
+"""
+
+from plone.observability.health.server import HealthServer
+from plone.observability.interfaces import IReadinessCheck
+from zope.component import queryUtility
+
+import logging
+import Zope2
+
+
+logger = logging.getLogger(__name__)
+
+_health_server = HealthServer()
+
+
+def make_filter(app, global_conf, **local_conf):
+    """PasteDeploy filter factory: start the health server (serving only)."""
+    db = Zope2.DB
+    _health_server.db = db
+    zodb_check = queryUtility(IReadinessCheck, name="zodb")
+    if zodb_check is not None:
+        zodb_check.db = db
+    logger.info("Starting plone.observability health server")
+    _health_server.start()
+    return app

plone/observability/interfaces.py

@@ -0,0 +1,63 @@
+from zope.interface import Attribute
+from zope.interface import Interface
+
+
+class ILivenessCheck(Interface):
+    """Named utility for liveness checks.
+
+    Implementations MUST NOT access ZODB or any blocking resource.
+    These run on the health server thread, outside of Zope context.
+    """
+
+    name = Attribute("Human-readable name of the check")
+
+    def __call__():
+        """Run the liveness check.
+
+        Returns a tuple of (ok: bool, message: str).
+        """
+
+
+class IReadinessCheck(Interface):
+    """Named utility for readiness checks.
+
+    May access ZODB and other resources. These checks verify the
+    application can serve real requests.
+    """
+
+    name = Attribute("Human-readable name of the check")
+
+    def __call__():
+        """Run the readiness check.
+
+        Returns a tuple of (ok: bool, message: str).
+        """
+
+
+class IMetricProvider(Interface):
+    """Adapter on the application root providing metrics.
+
+    The scope attribute indicates whether metrics are global (same across
+    all instances sharing a ZODB) or instance-specific.
+    """
+
+    name = Attribute("Provider name")
+    scope = Attribute("'global' (shared across instances) or 'instance'")
+
+    def collect():
+        """Collect current metrics.
+
+        Yields Metric instances.
+        """
+
+
+class IMetricFormatter(Interface):
+    """Named utility that serializes metrics to a specific wire format."""
+
+    content_type = Attribute("MIME type for the response")
+
+    def format(metrics):
+        """Format a sequence of Metric objects.
+
+        Returns a string in the target format.
+        """

plone/observability/metric.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from dataclasses import field
+
+
+@dataclass
+class Metric:
+    """A single metric data point."""
+
+    name: str
+    value: float
+    type: str  # "gauge", "counter", "histogram", "info"
+    scope: str  # "global" or "instance"
+    help: str
+    labels: dict = field(default_factory=dict)

plone/observability/metrics/access.py

@@ -0,0 +1,54 @@
+from ipaddress import ip_address
+from ipaddress import ip_network
+
+import os
+
+
+def _get_cidrs(env_var, default=""):
+    raw = os.environ.get(env_var, default).strip()
+    if not raw:
+        return []
+    return [ip_network(c.strip(), strict=False) for c in raw.split(",") if c.strip()]
+
+
+def _get_client_ip(peer_ip, headers):
+    """Determine the real client IP, respecting trusted proxies."""
+    trusted = _get_cidrs("PLONE_OBSERVABILITY_TRUSTED_PROXIES", default="127.0.0.1,::1")
+    try:
+        peer = ip_address(peer_ip)
+    except ValueError:
+        return peer_ip
+
+    if not any(peer in net for net in trusted):
+        return peer_ip
+
+    forwarded = headers.get("X-Forwarded-For", "")
+    if not forwarded:
+        return peer_ip
+
+    # Walk the chain from right to left, skipping trusted proxies.
+    # The first non-trusted IP is the real client.
+    parts = [p.strip() for p in forwarded.split(",")]
+    for part in reversed(parts):
+        try:
+            addr = ip_address(part)
+        except ValueError:
+            continue
+        if not any(addr in net for net in trusted):
+            return part
+    return peer_ip
+
+
+def is_allowed(peer_ip, headers):
+    """Check if a request is allowed based on IP allow-list."""
+    allowlist = _get_cidrs("PLONE_OBSERVABILITY_METRICS_ALLOWLIST")
+    if not allowlist:
+        return True
+
+    client_ip_str = _get_client_ip(peer_ip, headers)
+    try:
+        client_ip = ip_address(client_ip_str)
+    except ValueError:
+        return False
+
+    return any(client_ip in net for net in allowlist)

plone/observability/metrics/configure.zcml

@@ -0,0 +1,70 @@
+<configure
+    xmlns="http://namespaces.zope.org/zope"
+    xmlns:browser="http://namespaces.zope.org/browser"
+    >
+
+  <!-- Formatters -->
+  <utility
+      factory=".formatters.PrometheusFormatter"
+      provides="plone.observability.interfaces.IMetricFormatter"
+      name="prometheus"
+      />
+
+  <utility
+      factory=".formatters.JSONFormatter"
+      provides="plone.observability.interfaces.IMetricFormatter"
+      name="json"
+      />
+
+  <!-- Metrics view -->
+  <browser:page
+      name="metrics"
+      for="OFS.interfaces.IApplication"
+      class=".view.MetricsView"
+      permission="zope.Public"
+      />
+
+  <!-- Built-in metric providers -->
+  <adapter
+      factory=".providers.zope_runtime.ZopeRuntimeMetricProvider"
+      provides="plone.observability.interfaces.IMetricProvider"
+      for="OFS.interfaces.IApplication"
+      name="zope_runtime"
+      />
+
+  <adapter
+      factory=".providers.zodb.ZODBMetricProvider"
+      provides="plone.observability.interfaces.IMetricProvider"
+      for="OFS.interfaces.IApplication"
+      name="zodb"
+      />
+
+  <adapter
+      factory=".providers.content.ContentMetricProvider"
+      provides="plone.observability.interfaces.IMetricProvider"
+      for="OFS.interfaces.IApplication"
+      name="content"
+      />
+
+  <adapter
+      factory=".providers.request.RequestMetricProvider"
+      provides="plone.observability.interfaces.IMetricProvider"
+      for="OFS.interfaces.IApplication"
+      name="request"
+      />
+
+  <adapter
+      factory=".providers.system.SystemMetricProvider"
+      provides="plone.observability.interfaces.IMetricProvider"
+      for="OFS.interfaces.IApplication"
+      name="system"
+      />
+
+  <adapter
+      factory=".providers.conflict.ConflictMetricProvider"
+      provides="plone.observability.interfaces.IMetricProvider"
+      for="OFS.interfaces.IApplication"
+      name="conflict"
+      />
+
+</configure>

plone/observability/metrics/formatters.py

@@ -0,0 +1,48 @@
+from plone.observability.interfaces import IMetricFormatter
+from zope.interface import implementer
+
+import json
+
+
+@implementer(IMetricFormatter)
+class PrometheusFormatter:
+    """Formats metrics in Prometheus text exposition format."""
+
+    content_type = "text/plain; version=0.0.4; charset=utf-8"
+
+    def format(self, metrics):
+        lines = []
+        seen = set()
+        for m in metrics:
+            if m.name not in seen:
+                lines.append(f"# HELP {m.name} {m.help}")
+                lines.append(f"# TYPE {m.name} {m.type}")
+                seen.add(m.name)
+            labels = {"scope": m.scope}
+            labels.update(m.labels)
+            label_str = ",".join(f'{k}="{v}"' for k, v in labels.items())
+            value = int(m.value) if m.value == int(m.value) else m.value
+            lines.append(f"{m.name}{{{label_str}}} {value}")
+        return "\n".join(lines) + "\n"
+
+
+@implementer(IMetricFormatter)
+class JSONFormatter:
+    """Formats metrics as JSON."""
+
+    content_type = "application/json"
+
+    def format(self, metrics):
+        data = []
+        for m in metrics:
+            entry = {
+                "name": m.name,
+                "value": m.value,
+                "type": m.type,
+                "scope": m.scope,
+                "help": m.help,
+            }
+            if m.labels:
+                entry["labels"] = m.labels
+            data.append(entry)
+        return json.dumps(data)

plone/observability/metrics/providers/conflict.py

@@ -0,0 +1,55 @@
+from collections import defaultdict
+from plone.observability.interfaces import IMetricProvider
+from plone.observability.metric import Metric
+from ZODB.POSException import ConflictError
+from zope.interface import implementer
+
+import threading
+
+
+class ConflictTracker:
+    """Thread-safe ZODB conflict counter, partitioned by retry outcome."""
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self.counts = defaultdict(int)  # {"true"|"false": int}
+
+    def record(self, retry):
+        key = "true" if retry else "false"
+        with self._lock:
+            self.counts[key] += 1
+
+
+tracker = ConflictTracker()
+
+
+def on_pub_before_abort(event):
+    """IPubBeforeAbort subscriber: count ConflictErrors, labelled by retry."""
+    exc = event.exc_info[1] if event.exc_info else None
+    if isinstance(exc, ConflictError):
+        tracker.record(bool(event.retry))
+
+
+@implementer(IMetricProvider)
+class ConflictMetricProvider:
+    """Provides ZODB conflict metrics from the pubevent tracker."""
+
+    name = "conflict"
+    scope = "instance"
+
+    def __init__(self, context):
+        self.context = context
+
+    def collect(self):
+        for retry in ("true", "false"):
+            yield Metric(
+                name="plone_zodb_conflicts_total",
+                value=tracker.counts[retry],
+                type="counter",
+                scope="instance",
+                help=(
+                    "ZODB conflict errors during publish "
+                    "(retry=true was retried, retry=false gave up)"
+                ),
+                labels={"retry": retry},
+            )