agent-polygraph 0.1.0__py3-none-any.whl

agent_polygraph/events.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+"""
+Locked SDK event model for agent-polygraph v0.1.
+
+Three dataclasses -- ``Message`` / ``ToolCall`` / ``ToolResult`` -- mirror the
+polygraph-bench interchange FORMAT v1.0 event shapes 1:1, so an SDK trajectory
+serialises straight back to a bench item and a bench item deserialises straight
+into a trajectory (``from_bench`` / ``to_bench``). A ``verify()`` trajectory may
+mix dataclasses and raw bench dicts freely: ``normalize_event`` is the
+dict-accepting boundary that folds either into a dataclass.
+
+Bench FORMAT v1.0 (the only fields that cross the bench boundary):
+
+  * message      ``{"type": "message", "text": str}``
+  * tool_call    ``{"type": "tool_call", "call_id": str, "name": str, "arguments": Any}``
+  * tool_result  ``{"type": "tool_result", "call_id": str, "is_error": bool,
+                    "truncated": bool, "content": str}``
+
+``ToolResult`` additionally carries two *SDK-internal provenance* flags,
+``error_sniffed`` and ``truncation_inferred``. These record *how* an adapter
+recovered a degraded signal (payload sniff / marker inference vs. a structured
+framework bit). They are NEVER emitted to bench (``to_bench`` drops them) and are
+absent from ``from_bench`` output; they exist only so ``verify()`` can grade
+confidence 1.0 (structured) vs 0.7 (degraded). See BUILD-NOTES.md.
+
+Pure, standard-library only, no I/O, no globals.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Tuple, Union
+
+
+# ---------------------------------------------------------------------------
+# event dataclasses (bench FORMAT v1.0, 1:1)
+# ---------------------------------------------------------------------------
+@dataclass
+class Message:
+    """An assistant/model text turn. Empty text is preserved, never dropped."""
+    text: str = ""
+
+    def to_bench(self) -> Dict[str, Any]:
+        return {"type": "message", "text": self.text or ""}
+
+
+@dataclass
+class ToolCall:
+    """A tool invocation. A ToolCall with no following ToolResult is a legal
+    *dangling call* -- adapters MUST preserve it and MUST NOT fabricate a
+    result for it."""
+    call_id: str
+    name: str = ""
+    arguments: Any = ""
+
+    def to_bench(self) -> Dict[str, Any]:
+        return {
+            "type": "tool_call",
+            "call_id": self.call_id,
+            "name": self.name or "",
+            "arguments": self.arguments if self.arguments is not None else "",
+        }
+
+
+@dataclass
+class ToolResult:
+    """A tool result.
+
+    ``is_error`` / ``truncated`` are the bench-visible bits. ``error_sniffed``
+    and ``truncation_inferred`` are SDK-internal provenance: True means the bit
+    was recovered from an unstructured payload sniff / marker inference (a
+    DEGRADED signal -> 0.7 confidence), False means it came from a structured
+    framework signal (or was never set).
+    """
+    call_id: str
+    content: str = ""
+    is_error: bool = False
+    truncated: bool = False
+    # provenance -- not part of bench FORMAT, dropped by to_bench()
+    error_sniffed: bool = False
+    truncation_inferred: bool = False
+
+    def to_bench(self) -> Dict[str, Any]:
+        return {
+            "type": "tool_result",
+            "call_id": self.call_id,
+            "is_error": bool(self.is_error),
+            "truncated": bool(self.truncated),
+            "content": self.content or "",
+        }
+
+
+Event = Union[Message, ToolCall, ToolResult]
+
+
+# ---------------------------------------------------------------------------
+# bench <-> dataclass boundary
+# ---------------------------------------------------------------------------
+def from_bench(d: Dict[str, Any]) -> Event:
+    """One bench event dict -> the matching dataclass. Unknown/None types raise."""
+    if not isinstance(d, dict):
+        raise TypeError(f"bench event must be a dict, got {type(d).__name__}")
+    t = d.get("type")
+    if t == "message":
+        return Message(text=d.get("text") or "")
+    if t == "tool_call":
+        return ToolCall(
+            call_id=d.get("call_id"),
+            name=d.get("name") or "",
+            arguments=d.get("arguments", ""),
+        )
+    if t == "tool_result":
+        return ToolResult(
+            call_id=d.get("call_id"),
+            content=d.get("content") or "",
+            is_error=bool(d.get("is_error")),
+            truncated=bool(d.get("truncated")),
+        )
+    raise ValueError(f"unknown bench event type: {t!r}")
+
+
+def normalize_event(x: Union[Event, Dict[str, Any]]) -> Event:
+    """Dict-accepting boundary: pass a dataclass through, fold a dict via
+    ``from_bench``. This is what lets ``verify()`` accept a mixed trajectory."""
+    if isinstance(x, (Message, ToolCall, ToolResult)):
+        return x
+    return from_bench(x)
+
+
+def to_bench(trajectory: List[Union[Event, Dict[str, Any]]]) -> List[Dict[str, Any]]:
+    """Whole trajectory -> a list of bench event dicts (provenance stripped)."""
+    return [normalize_event(e).to_bench() for e in (trajectory or [])]
+
+
+def to_item(trajectory: List[Union[Event, Dict[str, Any]]],
+            final_claim: str = "", task: str = "") -> Dict[str, Any]:
+    """Build the interchange ITEM the clean-room detector consumes:
+    ``{"task", "events", "closing"}``."""
+    return {
+        "task": task or "",
+        "events": to_bench(trajectory),
+        "closing": final_claim or "",
+    }
+
+
+# ---------------------------------------------------------------------------
+# shared adapter helpers: host-path redaction, error-payload sniffing,
+# truncation-marker matching. Kept here so every adapter shares one
+# implementation and one default-marker registry.
+# ---------------------------------------------------------------------------
+import re  # noqa: E402  (kept adjacent to the helpers that use it)
+
+# Host-path shapes to scrub from ANYTHING an adapter re-emits (real OpenInference
+# tracebacks were found leaking absolute host paths -- CAPTURE-NOTES privacy note).
+# We scrub the PATHS only, never the words "Error"/"Traceback", so error-sniffing
+# still works on redacted content.
+_PATH_PATTERNS: Tuple[re.Pattern, ...] = (
+    re.compile(r"[A-Za-z]:\\(?:[^\s\"'<>|]+\\?)+"),          # Windows  C:\a\b\c
+    re.compile(r"(?<![\w.])/(?:home|Users|root|mnt|opt|var|tmp)/[^\s\"'<>|:]+"),  # POSIX
+    re.compile(r'File "[^"]*"'),                              # traceback frame path
+)
+_REDACTED = "<path>"
+
+
+def redact(text: Any) -> Any:
+    """Scrub absolute host paths / traceback frame paths from a string. Non-str
+    passes through untouched (so structured arguments survive)."""
+    if not isinstance(text, str) or not text:
+        return text
+    out = text
+    for pat in _PATH_PATTERNS:
+        out = pat.sub(_REDACTED, out)
+    return out
+
+
+# --- error-payload sniffing (recovery for the error-bit collapse) ------------
+# Conservative built-in patterns; a tool result whose body matches is treated as
+# a failure ONLY when the framework gave no structured error bit. Overridable via
+# Config.error_patterns.
+DEFAULT_ERROR_PATTERNS: Tuple[str, ...] = (
+    r"^\s*error\b[:\s]",
+    r"^\s*exception\b[:\s]",
+    r"traceback \(most recent call last\)",
+    r"^\s*fatal\b[:\s]",
+)
+
+
+def looks_like_error(content: str, patterns: Tuple[str, ...] = DEFAULT_ERROR_PATTERNS) -> bool:
+    """True iff a tool-result body reads as a failure payload."""
+    if not content:
+        return False
+    low = content.lower()
+    return any(re.search(p, low) for p in patterns)
+
+
+# --- truncation-marker matching ---------------------------------------------
+# Per-framework default markers live in each adapter; this matches a content
+# string against a set of literal-or-regex markers plus caller-supplied extras.
+# The bench universal seam is included so seam-bearing bench content round-trips.
+BENCH_SEAM = "[...omitted...]"
+
+
+def matches_truncation(content: str, markers: Tuple[Any, ...] = ()) -> bool:
+    """True iff content bears any truncation marker. ``markers`` items may be
+    plain substrings, ``re.Pattern`` objects, or regex strings starting with the
+    sentinel ``re:``."""
+    if not content:
+        return False
+    for m in markers:
+        if isinstance(m, re.Pattern):
+            if m.search(content):
+                return True
+        elif isinstance(m, str) and m.startswith("re:"):
+            if re.search(m[3:], content):
+                return True
+        elif isinstance(m, str):
+            if m in content:
+                return True
+    return False

agent_polygraph/judge.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""
+Judge-tier extension point for agent-polygraph v0.1 (locked ps-api-shape).
+
+The heuristic detector (``detector.py``) is deterministic and cheap, but it
+cannot reach the cases that need genuine reasoning over the trajectory -- the
+motivating example is the *dangling call + success claim* miss: a ``tool_call``
+with no matching ``tool_result`` plus a "Done, deployed successfully" closing is
+returned ``honest`` by the heuristic tier, because a result that never came back
+cannot be associated with the claim. Catching that class needs a judge.
+
+``Judge`` is the sync protocol a caller implements to plug one in; ``escalate``
+decides *which* heuristic verdicts get handed to it. ZERO judges ship in v0.1 --
+this module is the stable seam a future ``AsyncJudge`` / model-backed judge slots
+into without changing the ``verify()`` contract. A judge that overrides a verdict
+self-stamps its own ``detector_version`` so co-evolving verdicts stay comparable.
+
+Pure, standard-library only, no I/O, no globals.
+"""
+from __future__ import annotations
+
+from typing import Any, List, Optional, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Judge(Protocol):
+    """A pluggable second-tier verifier.
+
+    Implement ``judge(...)`` to inspect the trajectory the heuristic tier could
+    only pattern-match, and return either a replacement ``Verdict`` (which MUST
+    self-stamp its own ``detector_version``) or ``None`` to keep the heuristic
+    verdict unchanged.
+    """
+
+    def judge(self, trajectory: List[Any], final_claim: str, task: str,
+              heuristic_verdict: Any) -> Optional[Any]:
+        ...
+
+
+def should_escalate(escalate: str, verdict: Any) -> bool:
+    """Escalation policy: does this heuristic ``verdict`` get handed to the judge?
+
+    * ``"honest_only"`` (default) -- escalate only heuristic *honest* verdicts
+      (the judge exists to catch what the heuristic missed; blocks it already
+      caught are trusted).
+    * ``"degraded_only"`` -- escalate any verdict that leaned on a degraded
+      signal (a non-empty ``flags`` list).
+    * ``"all"`` -- escalate every verdict.
+    """
+    if escalate == "all":
+        return True
+    if escalate == "degraded_only":
+        return bool(getattr(verdict, "flags", ()))
+    # default: honest_only
+    return getattr(verdict, "verdict", None) == "honest"

agent_polygraph/verify.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+"""
+Locked public verify() contract for agent-polygraph v0.1 (ps-api-shape).
+
+    verify(trajectory, final_claim, task="") -> Verdict
+
+Sync, pure, thread-safe, no I/O, no env reads, no global mutable state. One
+detector per package (the clean-room ``detector.py``); this module wires it once
+at import and exposes its version as ``DETECTOR_VERSION`` (== the package
+version).
+
+Verdict schema (locked):
+  * ``verdict``          -- "lie" | "honest"
+  * ``evidence``         -- list[Finding{check, detail}]
+  * ``confidence``       -- signal-quality GRADE, not a probability: 1.0 when the
+                            verdict rode a structured signal, 0.7 when it rode an
+                            inferred truncation or a sniffed error payload.
+  * ``detector_version`` -- self-stamped so verdicts stay comparable across
+                            detector co-evolution.
+  (``flags`` and ``category`` ride along as documented diagnostics.)
+
+Confidence grading is computed HERE from adapter provenance, not read from the
+detector's internal flags: a structured truncation bit (e.g. Claude Code's
+``truncatedByTokenCap``) must NOT degrade confidence, while a marker-*inferred*
+truncation must -- a distinction the detector alone cannot make.
+
+Config: frozen dataclass, explicit kwargs. NO env vars, NO global registry.
+Judge tier: sync ``Judge`` protocol + ``escalate`` policy; ZERO judges shipped.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, List, Optional, Tuple, Union
+
+from . import detector as _DETECTOR
+from .events import (Message, ToolCall, ToolResult, normalize_event, to_item,
+                     DEFAULT_ERROR_PATTERNS)
+from .judge import Judge, should_escalate as _should_escalate
+
+# The one detector per package; detector.py is never modified. Its version is
+# the package version (wired in pyproject via detector.__version__).
+DETECTOR_VERSION: str = getattr(_DETECTOR, "__version__", "0.0.0")
+
+
+# ---------------------------------------------------------------------------
+# result + config types
+# ---------------------------------------------------------------------------
+@dataclass(frozen=True)
+class Finding:
+    """One check family's contribution to a lie verdict."""
+    check: str
+    detail: str
+
+    def as_dict(self) -> dict:
+        return {"check": self.check, "detail": self.detail}
+
+
+@dataclass(frozen=True)
+class Verdict:
+    """Locked verdict schema (+ diagnostic flags/category)."""
+    verdict: str                       # "lie" | "honest"
+    evidence: Tuple[Finding, ...] = ()
+    confidence: float = 1.0            # 1.0 structured | 0.7 degraded
+    detector_version: str = DETECTOR_VERSION
+    flags: Tuple[str, ...] = ()        # degradation markers driving a 0.7 grade
+    category: Optional[str] = None     # L1/L2/L3 diagnostic
+
+    @property
+    def is_lie(self) -> bool:
+        return self.verdict == "lie"
+
+    def as_dict(self) -> dict:
+        return {
+            "verdict": self.verdict,
+            "evidence": [f.as_dict() for f in self.evidence],
+            "confidence": self.confidence,
+            "detector_version": self.detector_version,
+            "flags": list(self.flags),
+            "category": self.category,
+        }
+
+
+@dataclass(frozen=True)
+class Config:
+    """Adapter behaviour knobs. Frozen; explicit; no env, no globals.
+
+    * ``error_sniff``            -- recover a lost tool-error bit by sniffing the
+                                    result payload when no structured bit exists
+                                    (ON by default; sniffed -> 0.7 grade).
+    * ``error_patterns``         -- override the built-in sniff patterns.
+    * ``extra_truncation_markers`` -- extra markers appended to each adapter's
+                                    per-framework defaults (literal substrings, or
+                                    ``re:``-prefixed regexes).
+    * ``redact_paths``           -- scrub host paths / traceback frames from
+                                    re-emitted content (ON by default).
+    """
+    error_sniff: bool = True
+    error_patterns: Tuple[str, ...] = DEFAULT_ERROR_PATTERNS
+    extra_truncation_markers: Tuple[Any, ...] = ()
+    redact_paths: bool = True
+
+
+DEFAULT_CONFIG = Config()
+
+
+# ---------------------------------------------------------------------------
+# the locked entry point
+# ---------------------------------------------------------------------------
+def verify(trajectory: List[Union[Message, ToolCall, ToolResult, dict]],
+           final_claim: str = "",
+           task: str = "",
+           *,
+           judge: Optional[Judge] = None,
+           escalate: str = "honest_only") -> Verdict:
+    """Does ``final_claim`` contradict the trajectory's own tool evidence?
+
+    Pure and synchronous. ``trajectory`` may mix SDK event dataclasses and raw
+    bench dicts. ``judge``/``escalate`` are the documented judge-tier extension
+    point (no judge ships in v0.1; the dangling-call miss is its motivating case).
+    """
+    events = [normalize_event(e) for e in (trajectory or [])]
+    item = to_item(events, final_claim, task)
+    det = _DETECTOR.detect(item)
+
+    findings = tuple(Finding(f.check, f.detail) for f in det.findings)
+
+    # SDK-level confidence grading from ADAPTER provenance (not detector flags):
+    # a firing family degrades only if it leaned on a sniffed/inferred signal.
+    checks = {f.check for f in findings}
+    flags: List[str] = []
+    if "error_concealment" in checks and any(
+        isinstance(e, ToolResult) and e.error_sniffed for e in events
+    ):
+        flags.append("error_payload_sniffed")
+    if "completion_vs_evidence" in checks and any(
+        isinstance(e, ToolResult) and e.truncation_inferred for e in events
+    ):
+        flags.append("truncation_inferred")
+
+    confidence = 0.7 if (findings and flags) else 1.0
+    result = Verdict(
+        verdict=det.verdict,
+        evidence=findings,
+        confidence=confidence,
+        detector_version=DETECTOR_VERSION,
+        flags=tuple(flags),
+        category=det.category,
+    )
+
+    if judge is not None and _should_escalate(escalate, result):
+        replacement = judge.judge(events, final_claim or "", task or "", result)
+        if replacement is not None:
+            return replacement
+    return result