devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/gateway/metrics_webhook.py

@@ -0,0 +1,252 @@
+"""
+devtorch_core.gateway.metrics_webhook
+=======================================
+Push aggregate gateway metrics to an external webhook endpoint.
+
+PRIVACY GUARANTEE:
+    The payload NEVER includes prompt text, code, reasoning blocks, or any
+    developer-authored content.  Only counts and aggregate scores are sent.
+
+Payload schema:
+    {
+        "ts":              ISO-8601 timestamp (UTC),
+        "gateway_version": str,
+        "token_counts": {
+            "total":      int,              # sum across all sessions
+            "by_session": {user_id: int}    # per-user totals (user IDs only)
+        },
+        "mcs_score":       float | null,
+        "dhs_score":       float | null,
+        "collision_count": int,
+        "session_count":   int
+    }
+
+Design decisions:
+- Uses stdlib urllib.request so no new dependencies are introduced.
+- All errors are caught silently and push() returns False — the gateway
+  must never crash because a webhook is unreachable.
+- enabled=False is the safe default so metrics are opt-in.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import urllib.request
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Optional
+
+logger = logging.getLogger("devtorch.gateway.metrics_webhook")
+
+GATEWAY_VERSION = "0.7.0"
+
+DEFAULT_METRICS_INTERVAL_SECONDS = 300
+
+
+@dataclass
+class MetricsWebhookConfig:
+    """
+    Environment-driven configuration for the metrics webhook.
+
+    All fields are optional and default to safe/opt-in-off values.
+    """
+
+    url: str = ""
+    enabled: bool = False
+    api_key: str = ""
+    interval_seconds: int = DEFAULT_METRICS_INTERVAL_SECONDS
+
+
+# Keys that must NEVER appear in the webhook payload (privacy guard)
+_FORBIDDEN_KEYS = frozenset(
+    {"prompt", "prompts", "content", "messages", "reasoning", "text", "code"}
+)
+
+
+def load_metrics_webhook_config() -> MetricsWebhookConfig:
+    """
+    Load metrics webhook configuration from environment variables.
+
+    Variables:
+        - DEVTORCH_METRICS_WEBHOOK_URL
+        - DEVTORCH_METRICS_WEBHOOK_ENABLED
+        - DEVTORCH_METRICS_WEBHOOK_API_KEY
+        - DEVTORCH_METRICS_WEBHOOK_INTERVAL_SECONDS
+
+    Returns
+    -------
+    MetricsWebhookConfig with safe defaults (disabled by default).
+    """
+    url = os.environ.get("DEVTORCH_METRICS_WEBHOOK_URL", "").strip()
+    enabled = _parse_bool_env(os.environ.get("DEVTORCH_METRICS_WEBHOOK_ENABLED", ""))
+    api_key = os.environ.get("DEVTORCH_METRICS_WEBHOOK_API_KEY", "").strip()
+
+    interval = DEFAULT_METRICS_INTERVAL_SECONDS
+    interval_raw = os.environ.get("DEVTORCH_METRICS_WEBHOOK_INTERVAL_SECONDS", "")
+    if interval_raw:
+        try:
+            interval = max(1, int(interval_raw))
+        except ValueError:
+            interval = DEFAULT_METRICS_INTERVAL_SECONDS
+
+    return MetricsWebhookConfig(
+        url=url,
+        enabled=enabled,
+        api_key=api_key,
+        interval_seconds=interval,
+    )
+
+
+def _parse_bool_env(value: str) -> bool:
+    """Return True if *value* looks like an affirmative env string."""
+    if not value:
+        return False
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+class MetricsWebhook:
+    """
+    Push aggregate metrics to an external HTTP webhook.
+
+    Parameters
+    ----------
+    webhook_url:
+        Full URL to POST metrics to (e.g. "https://analytics.example.com/devtorch").
+    enabled:
+        Set True to activate. Defaults to False so metrics are opt-in.
+    api_key:
+        Optional bearer token sent as `Authorization: Bearer <api_key>`.
+    interval_seconds:
+        How often (seconds) metrics are pushed when scheduled by the gateway.
+    """
+
+    def __init__(
+        self,
+        webhook_url: str = "",
+        enabled: bool = False,
+        api_key: str = "",
+        interval_seconds: int = DEFAULT_METRICS_INTERVAL_SECONDS,
+    ) -> None:
+        self._url = webhook_url.strip()
+        self._enabled = enabled
+        self._api_key = api_key.strip()
+        self._interval_seconds = max(1, interval_seconds)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def push(self, payload: dict) -> bool:
+        """
+        POST *payload* as JSON to the configured webhook URL.
+
+        Returns True on HTTP 200, False on any error or if disabled.
+        NEVER raises — all exceptions are caught silently.
+        """
+        if not self._enabled or not self._url:
+            return False
+
+        # Privacy guard: strip any forbidden keys recursively.
+        safe_payload = _strip_forbidden(payload)
+
+        try:
+            body = json.dumps(safe_payload, default=str).encode("utf-8")
+            headers = {
+                "Content-Type": "application/json",
+                "User-Agent": f"DevTorch-Gateway/{GATEWAY_VERSION}",
+            }
+            if self._api_key:
+                headers["Authorization"] = f"Bearer {self._api_key}"
+
+            req = urllib.request.Request(
+                self._url,
+                data=body,
+                method="POST",
+                headers=headers,
+            )
+            with urllib.request.urlopen(req, timeout=5) as resp:
+                return resp.status == 200
+        except Exception as exc:
+            logger.debug("MetricsWebhook.push failed: %s", exc)
+            return False
+
+    def build_payload(self, sessions: list[dict]) -> dict:
+        """
+        Build a privacy-safe webhook payload from session data.
+
+        Parameters
+        ----------
+        sessions:
+            List of session dicts.  Each dict may contain:
+                - user_id: str
+                - tokens_used: int
+                - mcs_score: float | None
+                - dhs_score: float | None
+                - collision_count: int
+
+        Returns a dict matching the payload schema.  No prompt/content fields
+        will ever appear.
+        """
+        total_tokens = 0
+        by_session: dict[str, int] = {}
+        mcs_scores: list[float] = []
+        dhs_scores: list[float] = []
+        total_collisions = 0
+
+        for sess in sessions:
+            uid = str(sess.get("user_id", "unknown"))
+            tokens = int(sess.get("tokens_used", 0))
+            total_tokens += tokens
+            by_session[uid] = by_session.get(uid, 0) + tokens
+
+            mcs = sess.get("mcs_score")
+            if mcs is not None:
+                try:
+                    mcs_scores.append(float(mcs))
+                except (TypeError, ValueError):
+                    pass
+
+            dhs = sess.get("dhs_score")
+            if dhs is not None:
+                try:
+                    dhs_scores.append(float(dhs))
+                except (TypeError, ValueError):
+                    pass
+
+            total_collisions += int(sess.get("collision_count", 0))
+
+        return {
+            "ts": datetime.now(tz=timezone.utc).isoformat(),
+            "gateway_version": GATEWAY_VERSION,
+            "token_counts": {
+                "total": total_tokens,
+                "by_session": by_session,
+            },
+            "mcs_score": (sum(mcs_scores) / len(mcs_scores)) if mcs_scores else None,
+            "dhs_score": (sum(dhs_scores) / len(dhs_scores)) if dhs_scores else None,
+            "collision_count": total_collisions,
+            "session_count": len(sessions),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+def _strip_forbidden(obj: object) -> object:
+    """
+    Recursively remove any dict keys in _FORBIDDEN_KEYS.
+
+    This is the privacy firewall between internal session data and the
+    outgoing webhook payload.
+    """
+    if isinstance(obj, dict):
+        return {
+            k: _strip_forbidden(v)
+            for k, v in obj.items()
+            if k.lower() not in _FORBIDDEN_KEYS
+        }
+    if isinstance(obj, list):
+        return [_strip_forbidden(item) for item in obj]
+    return obj

devtorch_core/gateway/policy.py

@@ -0,0 +1,262 @@
+"""
+devtorch_core.gateway.policy
+=============================
+Governance policy dataclass and enforcement engine for the enterprise gateway.
+
+GovernancePolicy defines what is allowed for an organisation.
+PolicyEngine enforces it on every request and response.
+
+Design decisions:
+- check_request returns (False, reason) to let the caller decide HTTP status.
+- check_response only *warns* on missing RACP blocks — it does not block
+  responses so that developer flow is never interrupted by compliance issues.
+- load_from_file accepts a JSON file whose top-level keys map 1:1 to the
+  dataclass fields, making policy files human-editable without a schema editor.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import threading
+from dataclasses import dataclass, field
+from typing import ClassVar
+
+logger = logging.getLogger("devtorch.gateway.policy")
+
+
+# ---------------------------------------------------------------------------
+# Dataclass
+# ---------------------------------------------------------------------------
+
+@dataclass
+class GovernancePolicy:
+    """
+    Org-level governance policy.
+
+    All fields have permissive defaults so that a gateway started with
+    ``GovernancePolicy.default()`` never blocks any legitimate request.
+    """
+
+    # Which LLM models are permitted. Empty list = all permitted.
+    allowed_models: list[str] = field(default_factory=list)
+
+    # Maximum cumulative tokens consumed by one (user, date) session.
+    max_tokens_per_session: int = 1_000_000
+
+    # Whether RACP compliance is mandatory. If True, a missing RACP block
+    # triggers a WARNING log (but does NOT block the response).
+    racp_compliance_required: bool = False
+
+    # Minimum ratio (0.0–1.0) of responses that must contain RACP blocks.
+    # Informational for now; used by the metrics webhook payload.
+    racp_compliance_threshold: float = 0.0
+
+    # If True, a warning is emitted when the agent skips devtorch_commit.
+    require_devtorch_commit: bool = False
+
+    # Concepts that trigger an alert when found in a response (informational).
+    blocked_concepts: list[str] = field(default_factory=list)
+
+    # ---------------------------------------------------------------------------
+    # Class methods
+    # ---------------------------------------------------------------------------
+
+    @classmethod
+    def default(cls) -> "GovernancePolicy":
+        """Return a maximally permissive policy suitable for development."""
+        return cls(
+            allowed_models=[],           # empty = all models allowed
+            max_tokens_per_session=1_000_000,
+            racp_compliance_required=False,
+            racp_compliance_threshold=0.0,
+            require_devtorch_commit=False,
+            blocked_concepts=[],
+        )
+
+    @classmethod
+    def load_from_file(cls, path: str) -> "GovernancePolicy":
+        """
+        Load a GovernancePolicy from a JSON file.
+
+        The JSON must have top-level keys matching the dataclass field names.
+        Unknown keys are ignored. Missing keys fall back to the default.
+        """
+        with open(path, "r", encoding="utf-8") as fh:
+            raw = json.load(fh)
+
+        defaults = cls.default()
+        return cls(
+            allowed_models=raw.get("allowed_models", defaults.allowed_models),
+            max_tokens_per_session=int(
+                raw.get("max_tokens_per_session", defaults.max_tokens_per_session)
+            ),
+            racp_compliance_required=bool(
+                raw.get("racp_compliance_required", defaults.racp_compliance_required)
+            ),
+            racp_compliance_threshold=float(
+                raw.get("racp_compliance_threshold", defaults.racp_compliance_threshold)
+            ),
+            require_devtorch_commit=bool(
+                raw.get("require_devtorch_commit", defaults.require_devtorch_commit)
+            ),
+            blocked_concepts=raw.get("blocked_concepts", defaults.blocked_concepts),
+        )
+
+    def to_summary(self) -> dict:
+        """Return a JSON-safe summary for the /health endpoint (no secrets)."""
+        return {
+            "allowed_models": self.allowed_models or ["*"],
+            "max_tokens_per_session": self.max_tokens_per_session,
+            "racp_compliance_required": self.racp_compliance_required,
+            "racp_compliance_threshold": self.racp_compliance_threshold,
+            "require_devtorch_commit": self.require_devtorch_commit,
+            "blocked_concepts_count": len(self.blocked_concepts),
+        }
+
+
+# ---------------------------------------------------------------------------
+# Engine
+# ---------------------------------------------------------------------------
+
+class PolicyEngine:
+    """
+    Enforce a GovernancePolicy on individual requests and responses.
+
+    Thread-safe: the mutable RACP compliance window is protected by a lock.
+    """
+
+    # Default size of the rolling RACP compliance window.
+    _DEFAULT_RACP_WINDOW_SIZE: ClassVar[int] = 20
+
+    def __init__(
+        self,
+        policy: GovernancePolicy,
+        racp_window_size: int = _DEFAULT_RACP_WINDOW_SIZE,
+    ) -> None:
+        self._policy = policy
+        self._racp_window_size = max(1, racp_window_size)
+        self._racp_window: list[bool] = []
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def check_request(
+        self,
+        model: str,
+        messages: list,
+        session_tokens_used: int,
+    ) -> tuple[bool, str]:
+        """
+        Validate an outgoing request against the current policy.
+
+        Returns:
+            (True, "")            — request is permitted
+            (False, reason_str)   — request should be blocked with 403
+        """
+        p = self._policy
+
+        # 1. Model allowlist (empty list = all models permitted)
+        if p.allowed_models and model not in p.allowed_models:
+            reason = (
+                f"Model '{model}' is not in the org allowed_models list. "
+                f"Permitted: {p.allowed_models}"
+            )
+            logger.warning("Gateway BLOCKED request: %s", reason)
+            return False, reason
+
+        # 2. Token quota
+        if session_tokens_used >= p.max_tokens_per_session:
+            reason = (
+                f"Session token limit reached: {session_tokens_used} >= "
+                f"{p.max_tokens_per_session}. Start a new session."
+            )
+            logger.warning("Gateway BLOCKED request: %s", reason)
+            return False, reason
+
+        return True, ""
+
+    def check_response(
+        self,
+        response_body: dict,
+        has_racp_blocks: bool,
+    ) -> tuple[bool, str]:
+        """
+        Validate a response from the upstream LLM.
+
+        This method only *warns* — it never blocks a response — so the return
+        value is always (True, warning_or_empty).
+
+        Returns:
+            (True, "")            — response is clean
+            (True, warning_str)   — response has compliance issue but is allowed
+        """
+        p = self._policy
+
+        # Track RACP presence in a rolling window for threshold enforcement.
+        self._record_racp_presence(has_racp_blocks)
+
+        if p.racp_compliance_required and not has_racp_blocks:
+            warning = (
+                "RACP compliance required but this response contains no RACP blocks. "
+                "Ensure the agent is using devtorch_commit / RACP annotations."
+            )
+            logger.warning("Gateway compliance warning: %s", warning)
+            return True, warning
+
+        # Enforce the rolling RACP compliance threshold (informational only).
+        threshold_warning = self._check_racp_threshold()
+        if threshold_warning:
+            logger.warning("Gateway compliance warning: %s", threshold_warning)
+            return True, threshold_warning
+
+        return True, ""
+
+    @property
+    def policy(self) -> GovernancePolicy:
+        return self._policy
+
+    @property
+    def racp_ratio(self) -> float:
+        """Return the current ratio of responses in the window that contain RACP blocks."""
+        with self._lock:
+            if not self._racp_window:
+                return 0.0
+            return sum(self._racp_window) / len(self._racp_window)
+
+    @property
+    def racp_window_count(self) -> int:
+        """Return the number of responses currently in the RACP compliance window."""
+        with self._lock:
+            return len(self._racp_window)
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _record_racp_presence(self, has_racp_blocks: bool) -> None:
+        """Append *has_racp_blocks* to the rolling window, trimming to size."""
+        with self._lock:
+            self._racp_window.append(bool(has_racp_blocks))
+            if len(self._racp_window) > self._racp_window_size:
+                self._racp_window.pop(0)
+
+    def _check_racp_threshold(self) -> str:
+        """
+        Return a warning string if the rolling RACP ratio is below the policy
+        threshold.  Returns an empty string when the threshold is disabled or
+        when the ratio is satisfactory.
+        """
+        p = self._policy
+        if p.racp_compliance_threshold <= 0:
+            return ""
+
+        ratio = self.racp_ratio
+        if ratio < p.racp_compliance_threshold:
+            return (
+                f"Rolling RACP compliance ratio {ratio:.2f} is below the org "
+                f"threshold {p.racp_compliance_threshold:.2f}. "
+                f"Ensure agents are using devtorch_commit / RACP annotations."
+            )
+        return ""