spanforge 1.0.0__py3-none-any.whl

spanforge/regression.py

@@ -0,0 +1,192 @@
+"""spanforge.regression — Generic pass/fail regression detection.
+
+Provides :class:`RegressionDetector` for comparing two evaluation runs and
+surfacing cases that have *regressed*: passing in the baseline but failing in
+the current run, or whose score dropped by more than a configurable threshold.
+
+Unlike :class:`spanforge.eval.RegressionDetector` (which compares mean metric
+scores between runs), this detector operates on individual result records with
+explicit ``passed`` and ``score`` fields — making it well-suited for CI gates
+where each test case must individually pass.
+
+Usage::
+
+    from spanforge.regression import RegressionDetector
+
+    detector = RegressionDetector(score_drop_threshold=0.1)
+    report = detector.compare(
+        baseline=baseline_results,
+        current=current_results,
+        key_fn=lambda r: (r["case_id"], r["scorer_name"]),
+        passed_fn=lambda r: r["passed"],
+        score_fn=lambda r: r["score"],
+    )
+
+    if report.has_regression:
+        for item in report.new_failures:
+            print("NEW FAILURE:", item)
+        for base, curr in report.score_drops:
+            print(f"SCORE DROP: {base} → {curr}")
+        sys.exit(1)
+
+Works with any record type (dicts, dataclasses, etc.) via the *key_fn*,
+*passed_fn*, and *score_fn* callbacks.  There is also a convenience
+:func:`compare` top-level function for one-shot use.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Generic, TypeVar
+
+__all__ = [
+    "RegressionDetector",
+    "RegressionReport",
+    "compare",
+]
+
+T = TypeVar("T")
+
+
+@dataclass
+class RegressionReport(Generic[T]):
+    """Summary of regressions found between two evaluation runs.
+
+    Attributes:
+        new_failures:  Items that *passed* in the baseline but *fail* in the
+                       current run.
+        score_drops:   ``(baseline_item, current_item)`` pairs where the score
+                       dropped by at least the configured threshold.
+    """
+
+    new_failures: list[T] = field(default_factory=list)
+    score_drops: list[tuple[T, T]] = field(default_factory=list)
+
+    @property
+    def has_regression(self) -> bool:
+        """``True`` when at least one regression was detected."""
+        return bool(self.new_failures or self.score_drops)
+
+    def summary(self) -> str:
+        """Return a short human-readable summary string."""
+        parts: list[str] = []
+        if self.new_failures:
+            parts.append(f"{len(self.new_failures)} new failure(s)")
+        if self.score_drops:
+            parts.append(f"{len(self.score_drops)} score drop(s)")
+        if not parts:
+            return "no regression detected"
+        return "; ".join(parts)
+
+
+class RegressionDetector(Generic[T]):
+    """Compare two evaluation runs and report regressions.
+
+    A *regression* is one of:
+
+    * A key that **passed** in the baseline but **fails** in the current run.
+    * A key whose score **dropped** by at least *score_drop_threshold*
+      (even when the current result still passes).
+
+    New keys that appear only in the current run are **not** flagged as
+    regressions (they may be new test cases).  Keys that disappear from the
+    current run are also silently ignored.
+
+    Args:
+        score_drop_threshold:  Minimum absolute score decrease that
+                               constitutes a regression.  Default is ``0.1``.
+
+    Example::
+
+        detector = RegressionDetector[dict](score_drop_threshold=0.05)
+        report = detector.compare(
+            baseline, current,
+            key_fn=lambda r: (r["case_id"], r["scorer"]),
+            passed_fn=lambda r: r["passed"],
+            score_fn=lambda r: r["score"],
+        )
+        print(report.summary())
+    """
+
+    def __init__(self, score_drop_threshold: float = 0.1) -> None:
+        self.score_drop_threshold = score_drop_threshold
+
+    def compare(
+        self,
+        baseline: list[T],
+        current: list[T],
+        *,
+        key_fn: Callable[[T], Any],
+        passed_fn: Callable[[T], bool],
+        score_fn: Callable[[T], float],
+    ) -> RegressionReport[T]:
+        """Compare *current* against *baseline* and return a :class:`RegressionReport`.
+
+        Args:
+            baseline:   Results from a known-good previous run.
+            current:    Results from the run being checked.
+            key_fn:     Callable that returns a hashable key identifying a
+                        result (e.g. ``lambda r: (r.case_id, r.scorer_name)``).
+            passed_fn:  Callable that returns ``True`` when a result passed.
+            score_fn:   Callable that returns the numeric score of a result.
+
+        Returns:
+            A :class:`RegressionReport` describing found regressions.
+        """
+        baseline_map: dict[Any, T] = {key_fn(r): r for r in baseline}
+        current_map: dict[Any, T] = {key_fn(r): r for r in current}
+
+        new_failures: list[T] = []
+        score_drops: list[tuple[T, T]] = []
+
+        for key, curr in current_map.items():
+            base = baseline_map.get(key)
+            if base is None:
+                continue  # new key — not a regression
+
+            if passed_fn(base) and not passed_fn(curr):
+                new_failures.append(curr)
+            elif (score_fn(base) - score_fn(curr)) >= self.score_drop_threshold:
+                score_drops.append((base, curr))
+
+        return RegressionReport(new_failures=new_failures, score_drops=score_drops)
+
+
+def compare(
+    baseline: list[Any],
+    current: list[Any],
+    *,
+    key_fn: Callable[[Any], Any],
+    passed_fn: Callable[[Any], bool],
+    score_fn: Callable[[Any], float],
+    score_drop_threshold: float = 0.1,
+) -> RegressionReport[Any]:
+    """One-shot convenience wrapper around :class:`RegressionDetector`.
+
+    Args:
+        baseline:              Results from the baseline run.
+        current:               Results from the run being checked.
+        key_fn:                Returns a unique key for each result.
+        passed_fn:             Returns ``True`` when a result passed.
+        score_fn:              Returns the numeric score of a result.
+        score_drop_threshold:  Minimum score drop to flag as regression.
+
+    Returns:
+        A :class:`RegressionReport`.
+
+    Example::
+
+        report = compare(
+            baseline, current,
+            key_fn=lambda r: r["id"],
+            passed_fn=lambda r: r["ok"],
+            score_fn=lambda r: r["score"],
+        )
+    """
+    return RegressionDetector(score_drop_threshold=score_drop_threshold).compare(
+        baseline,
+        current,
+        key_fn=key_fn,
+        passed_fn=passed_fn,
+        score_fn=score_fn,
+    )

spanforge/runtime_policy.py

@@ -0,0 +1,159 @@
+"""spanforge.runtime_policy - Phase 0 runtime policy schema contracts.
+
+This module freezes the policy object model used by the GA runtime governance
+control plane. Enforcement engines can evolve behind these contracts without
+changing the configuration shape exposed to users.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+__all__ = [
+    "RuntimePolicyBundle",
+    "RuntimePolicyRule",
+]
+
+_VALID_ENVIRONMENTS = frozenset({"dev", "staging", "prod"})
+_VALID_POLICY_ACTIONS = frozenset({"allow", "allow+log", "redact", "block", "human_review"})
+_VALID_SERVICES = frozenset({"sf_explain", "sf_scope", "sf_rbac", "sf_rag", "sf_lineage"})
+
+
+def _require_mapping(data: Any, type_name: str) -> dict[str, Any]:
+    if not isinstance(data, dict):
+        raise ValueError(f"{type_name} input must be a dict")
+    return data
+
+
+def _require_fields(data: dict[str, Any], type_name: str, fields: tuple[str, ...]) -> None:
+    missing = [field for field in fields if field not in data]
+    if missing:
+        raise ValueError(f"{type_name} is missing required fields: {', '.join(missing)}")
+
+
+@dataclass
+class RuntimePolicyRule:
+    """One runtime governance rule bound to a service and control."""
+
+    rule_id: str
+    service: str
+    control: str
+    action: str
+    enabled: bool = True
+    threshold: float | None = None
+    rationale: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.rule_id:
+            raise ValueError("RuntimePolicyRule.rule_id must be non-empty")
+        if self.service not in _VALID_SERVICES:
+            raise ValueError(
+                f"RuntimePolicyRule.service must be one of {sorted(_VALID_SERVICES)}"
+            )
+        if not self.control:
+            raise ValueError("RuntimePolicyRule.control must be non-empty")
+        if self.action not in _VALID_POLICY_ACTIONS:
+            raise ValueError(
+                f"RuntimePolicyRule.action must be one of {sorted(_VALID_POLICY_ACTIONS)}"
+            )
+        if self.threshold is not None and not (0.0 <= self.threshold <= 1.0):
+            raise ValueError("RuntimePolicyRule.threshold must be in [0.0, 1.0]")
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "rule_id": self.rule_id,
+            "service": self.service,
+            "control": self.control,
+            "action": self.action,
+            "enabled": self.enabled,
+        }
+        if self.threshold is not None:
+            data["threshold"] = self.threshold
+        if self.rationale:
+            data["rationale"] = self.rationale
+        if self.metadata:
+            data["metadata"] = self.metadata
+        return data
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RuntimePolicyRule:
+        parsed = _require_mapping(data, "RuntimePolicyRule")
+        _require_fields(
+            parsed,
+            "RuntimePolicyRule",
+            ("rule_id", "service", "control", "action"),
+        )
+        return cls(
+            rule_id=parsed["rule_id"],
+            service=parsed["service"],
+            control=parsed["control"],
+            action=parsed["action"],
+            enabled=bool(parsed.get("enabled", True)),
+            threshold=float(parsed["threshold"]) if "threshold" in parsed else None,
+            rationale=parsed.get("rationale", ""),
+            metadata=dict(parsed.get("metadata", {})),
+        )
+
+
+@dataclass
+class RuntimePolicyBundle:
+    """Versioned runtime policy bundle for one deployment environment."""
+
+    policy_id: str
+    version: str
+    environment: str
+    owner: str
+    effective_at: str
+    rules: list[RuntimePolicyRule] = field(default_factory=list)
+    rationale: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.policy_id:
+            raise ValueError("RuntimePolicyBundle.policy_id must be non-empty")
+        if not self.version:
+            raise ValueError("RuntimePolicyBundle.version must be non-empty")
+        if self.environment not in _VALID_ENVIRONMENTS:
+            raise ValueError(
+                f"RuntimePolicyBundle.environment must be one of {sorted(_VALID_ENVIRONMENTS)}"
+            )
+        if not self.owner:
+            raise ValueError("RuntimePolicyBundle.owner must be non-empty")
+        if not self.effective_at:
+            raise ValueError("RuntimePolicyBundle.effective_at must be non-empty")
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "policy_id": self.policy_id,
+            "version": self.version,
+            "environment": self.environment,
+            "owner": self.owner,
+            "effective_at": self.effective_at,
+            "rules": [rule.to_dict() for rule in self.rules],
+        }
+        if self.rationale:
+            data["rationale"] = self.rationale
+        if self.metadata:
+            data["metadata"] = self.metadata
+        return data
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RuntimePolicyBundle:
+        parsed = _require_mapping(data, "RuntimePolicyBundle")
+        _require_fields(
+            parsed,
+            "RuntimePolicyBundle",
+            ("policy_id", "version", "environment", "owner", "effective_at"),
+        )
+        return cls(
+            policy_id=parsed["policy_id"],
+            version=parsed["version"],
+            environment=parsed["environment"],
+            owner=parsed["owner"],
+            effective_at=parsed["effective_at"],
+            rules=[RuntimePolicyRule.from_dict(item) for item in parsed.get("rules", [])],
+            rationale=parsed.get("rationale", ""),
+            metadata=dict(parsed.get("metadata", {})),
+        )