agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/judgment_types.py

@@ -0,0 +1,241 @@
+"""AgentLoop-owned contracts for optional offline semantic judgments."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from dataclasses import asdict, dataclass
+from typing import Any, Protocol
+
+JUDGMENT_VERSION = "1.0"
+JUDGMENT_KEY = "agentloop.judgments"
+
+
+def canonical(value: Any) -> str:
+    try:
+        return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
+    except (TypeError, ValueError, RecursionError):
+        raise ValueError("judgment data must contain finite JSON values") from None
+
+
+def fingerprint(value: Any) -> str:
+    return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()
+
+
+def text(value: Any, name: str, *, optional: bool = False) -> None:
+    if value is None and optional:
+        return
+    if not isinstance(value, str) or not value.strip() or len(value) > 4096:
+        raise ValueError(f"{name} must be nonempty text of at most 4096 characters")
+
+
+def finite(value: Any) -> bool:
+    try:
+        return type(value) in {int, float} and math.isfinite(value)
+    except OverflowError:
+        return False
+
+
+def is_hash(value: Any) -> bool:
+    return (
+        isinstance(value, str)
+        and len(value) == 64
+        and all(char in "0123456789abcdef" for char in value)
+    )
+
+
+@dataclass(frozen=True)
+class JudgeIdentity:
+    implementation: str
+    version: str
+    config_hash: str
+    provider: str = "local"
+    model_or_rule: str = "callback"
+    revision: str = "1"
+
+    def __post_init__(self):
+        for key, value in asdict(self).items():
+            text(value, key)
+        if not is_hash(self.config_hash):
+            raise ValueError("config_hash must be a SHA-256 digest")
+
+    @classmethod
+    def configured(cls, implementation: str, version: str, configuration: Any, **kwargs):
+        """Hash caller-owned configuration without retaining or forwarding its contents."""
+        return cls(implementation, version, fingerprint(configuration), **kwargs)
+
+
+@dataclass(frozen=True)
+class JudgmentSpec:
+    question: str
+    kind: str
+    choices: tuple[str, ...] = ()
+    minimum: float | None = None
+    maximum: float | None = None
+
+    def __post_init__(self):
+        text(self.question, "question")
+        if self.kind not in {"boolean", "probability", "choice", "score"}:
+            raise ValueError("unsupported judgment kind")
+        if not isinstance(self.choices, (list, tuple)):
+            raise ValueError("choices must be a list or tuple")
+        object.__setattr__(self, "choices", tuple(self.choices))
+        if self.kind == "choice":
+            for item in self.choices:
+                text(item, "choice")
+            if not self.choices or len(set(self.choices)) != len(self.choices):
+                raise ValueError("choices must be nonempty and unique")
+        elif self.choices:
+            raise ValueError("choices apply only to choice judgments")
+        if self.kind == "score":
+            if not finite(self.minimum) or not finite(self.maximum) or self.minimum >= self.maximum:
+                raise ValueError("score requires finite minimum < maximum")
+        elif self.minimum is not None or self.maximum is not None:
+            raise ValueError("explicit bounds apply only to score judgments")
+
+    def accepts(self, value: Any) -> bool:
+        if self.kind == "boolean":
+            return type(value) is bool
+        if self.kind == "choice":
+            return isinstance(value, str) and value in self.choices
+        if not finite(value):
+            return False
+        low, high = (0, 1) if self.kind == "probability" else (self.minimum, self.maximum)
+        return low <= value <= high
+
+
+@dataclass(frozen=True)
+class JudgmentEvidence:
+    span_id: str
+    source_hash: str
+    operation_kind: str
+    status: str
+    duration_ms: float
+    summary: str | None = None
+
+    def __post_init__(self):
+        text(self.span_id, "span_id")
+        text(self.operation_kind, "operation_kind")
+        if not is_hash(self.source_hash):
+            raise ValueError("source_hash must be a SHA-256 digest")
+        if self.status not in {"ok", "error", "unknown"}:
+            raise ValueError("unsupported observed status")
+        if not finite(self.duration_ms) or self.duration_ms < 0:
+            raise ValueError("duration_ms must be finite and nonnegative")
+        text(self.summary, "summary", optional=True)
+
+
+@dataclass(frozen=True)
+class JudgmentRequest:
+    run_id: str
+    spec: JudgmentSpec
+    evidence: tuple[JudgmentEvidence, ...]
+
+    def __post_init__(self):
+        text(self.run_id, "run_id")
+        if type(self.spec) is not JudgmentSpec:
+            raise ValueError("spec must be JudgmentSpec")
+        if not isinstance(self.evidence, (tuple, list)):
+            raise ValueError("evidence must be a list or tuple")
+        object.__setattr__(self, "evidence", tuple(self.evidence))
+        if not self.evidence or any(type(item) is not JudgmentEvidence for item in self.evidence):
+            raise ValueError("at least one JudgmentEvidence is required")
+        if len({item.span_id for item in self.evidence}) != len(self.evidence):
+            raise ValueError("evidence span IDs must be unique")
+
+    def to_dict(self):
+        return json.loads(canonical(asdict(self)))
+
+    @classmethod
+    def from_dict(cls, value):
+        return cls(
+            value["run_id"],
+            JudgmentSpec(**value["spec"]),
+            tuple(JudgmentEvidence(**item) for item in value["evidence"]),
+        )
+
+
+@dataclass(frozen=True)
+class JudgeUsage:
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    cost_usd: float | None = None
+    token_basis: str = "unknown"
+    cost_basis: str = "unknown"
+
+    def __post_init__(self):
+        for count in (self.input_tokens, self.output_tokens):
+            if count is not None and (type(count) is not int or count < 0):
+                raise ValueError("known token counts must be nonnegative integers")
+        if self.cost_usd is not None and (not finite(self.cost_usd) or self.cost_usd < 0):
+            raise ValueError("known cost must be finite and nonnegative")
+        if self.token_basis not in {"unknown", "reported", "estimated", "not_dispatched"}:
+            raise ValueError("unsupported token basis")
+        if self.cost_basis not in {"unknown", "reported", "calculated", "not_dispatched"}:
+            raise ValueError("unsupported cost basis")
+        if (self.input_tokens is not None or self.output_tokens is not None) == (
+            # A model-usage provenance label, not an authentication credential.
+            self.token_basis == "unknown"  # nosec B105
+        ):
+            raise ValueError("token counts and basis must agree")
+        if (self.cost_usd is not None) == (self.cost_basis == "unknown"):
+            raise ValueError("cost and basis must agree")
+        # This public label records that no model invocation occurred.
+        if self.token_basis == "not_dispatched" and (self.input_tokens, self.output_tokens) != (  # nosec B105
+            0,
+            0,
+        ):
+            raise ValueError("undispatched usage must be zero")
+        if self.cost_basis == "not_dispatched" and self.cost_usd != 0:
+            raise ValueError("undispatched cost must be zero")
+
+
+@dataclass(frozen=True)
+class JudgmentUncertainty:
+    confidence: float | None = None
+    calibration_status: str = "uncalibrated"
+    calibration_ref: str | None = None
+
+    def __post_init__(self):
+        if self.confidence is not None and (
+            not finite(self.confidence) or not 0 <= self.confidence <= 1
+        ):
+            raise ValueError("confidence must be in [0, 1]")
+        if self.calibration_status not in {"uncalibrated", "measured"}:
+            raise ValueError("unsupported calibration status")
+        text(self.calibration_ref, "calibration_ref", optional=True)
+        if self.calibration_status == "measured" and self.calibration_ref is None:
+            raise ValueError("measured calibration requires a caller-supplied evidence reference")
+
+
+@dataclass(frozen=True)
+class JudgmentAnswer:
+    value: Any = None
+    status: str = "known"
+    reason: str | None = None
+    usage: JudgeUsage = JudgeUsage()
+    uncertainty: JudgmentUncertainty = JudgmentUncertainty()
+
+    def __post_init__(self):
+        if self.status not in {"known", "unknown"}:
+            raise ValueError("answer status must be known or unknown")
+        if self.status == "unknown":
+            if self.value is not None or self.reason not in {"abstained", "insufficient_evidence"}:
+                raise ValueError("unknown answers require a reason and no value")
+        elif self.reason is not None:
+            raise ValueError("known answers cannot carry a failure reason")
+        if type(self.usage) is not JudgeUsage or type(self.uncertainty) is not JudgmentUncertainty:
+            raise ValueError("answer requires typed usage and uncertainty")
+        if self.value is not None and type(self.value) not in {str, bool, int, float}:
+            raise ValueError("answer must be a scalar")
+        canonical(self.value)
+
+
+class JudgmentBackend(Protocol):
+    """Adapters own cooperative time limits and must return a typed answer."""
+
+    @property
+    def identity(self) -> JudgeIdentity: ...
+
+    def judge(self, request: JudgmentRequest, *, timeout_s: float | None) -> JudgmentAnswer: ...

agentloop/judgment_views.py

@@ -0,0 +1,38 @@
+"""Text projection of saved judgment evidence; never executes a backend."""
+
+from agentloop.markdown import markdown_table_cell
+
+
+def judgment_markdown(evidence):
+    lines = [
+        "",
+        "## Offline semantic judgments",
+        "",
+        "Recorded execution facts, deterministic rule inferences, and semantic judge answers "
+        "are separate evidence categories. Judge answers do not establish task quality or "
+        "calibrated confidence. Judge latency and cost are analysis overhead, separate from workflow totals.",
+        "",
+        f"Evidence status: {markdown_table_cell(evidence['status'])}",
+        "",
+        "| Judge / version | Kind | Status | Value | Confidence basis | Cache hit | Incurred cost USD |",
+        "| --- | --- | --- | --- | --- | --- | --- |",
+    ]
+    for record in evidence["records"]:
+        judge, result = record["judge"], record["evaluation"]
+        cells = [
+            f"{judge['implementation']} / {judge['version']}",
+            record["request"]["spec"]["kind"],
+            record["effective_status"],
+            record["effective_value"],
+            result["uncertainty"]["calibration_status"],
+            record["invocation"]["cache_hit"],
+            record["invocation"]["usage"]["cost_usd"],
+        ]
+        lines.append(
+            "| "
+            + " | ".join(
+                markdown_table_cell("unavailable" if item is None else str(item)) for item in cells
+            )
+            + " |"
+        )
+    return lines

agentloop/judgments.py

@@ -0,0 +1,342 @@
+"""Explicit offline judgment execution, bounded caching and portable receipts."""
+
+from __future__ import annotations
+
+import inspect
+import json
+import time
+from collections import OrderedDict
+from dataclasses import asdict, dataclass
+from typing import Callable
+from uuid import uuid4
+
+from agentloop.events import AgentEvent, utc_now_iso
+from agentloop.judgment_types import (
+    JUDGMENT_KEY,
+    JUDGMENT_VERSION,
+    JudgeIdentity,
+    JudgeUsage,
+    JudgmentAnswer,
+    JudgmentBackend,
+    JudgmentEvidence,
+    JudgmentRequest,
+    JudgmentSpec,
+    JudgmentUncertainty,
+    canonical,
+    fingerprint,
+    finite,
+)
+from agentloop.operations import OPERATION_KINDS, operation_kind
+
+_NO_USAGE = JudgeUsage(0, 0, 0.0, "not_dispatched", "not_dispatched")
+_FAILURES = {"disabled", "timeout", "error", "invalid_result"}
+
+
+def _copy(value):
+    return json.loads(canonical(value))
+
+
+def _event_hash(event):
+    # Use the importer normalization so an integer duration and its native JSON
+    # round trip have the same fingerprint. Bodies are hashed, never forwarded.
+    source = AgentEvent.from_dict(event.to_dict()).to_dict()
+    source["metadata"] = {
+        key: value
+        for key, value in source["metadata"].items()
+        if key not in {"otel_span_id", "otel_trace_id"}
+    }
+    return fingerprint(source)
+
+
+def _evidence(event, summary, hashes):
+    if event.event_id not in hashes:
+        hashes[event.event_id] = _event_hash(event)
+    kind = operation_kind(event)
+    return JudgmentEvidence(
+        event.event_id,
+        hashes[event.event_id],
+        kind if kind in OPERATION_KINDS else "unknown",
+        event.status if event.status in {"ok", "error"} else "unknown",
+        float(event.duration_ms),
+        summary,
+    )
+
+
+def judgment_request(trace, spec: JudgmentSpec, span_ids, *, summaries=None) -> JudgmentRequest:
+    """Select references/observations; only explicitly supplied summaries carry text."""
+    if not isinstance(span_ids, (list, tuple)) or not span_ids:
+        raise ValueError("span_ids must be a nonempty list or tuple")
+    if any(not isinstance(item, str) for item in span_ids) or len(set(span_ids)) != len(span_ids):
+        raise ValueError("span_ids must contain unique strings")
+    summaries = {} if summaries is None else summaries
+    if not isinstance(summaries, dict) or set(summaries) - set(span_ids):
+        raise ValueError("summaries must map selected span IDs to explicitly supplied text")
+    events = {event.event_id: event for event in trace.events}
+    if len(events) != len(trace.events) or set(span_ids) - events.keys():
+        raise ValueError("selected span IDs must resolve uniquely in the trace")
+    hashes = {}
+    evidence = [_evidence(events[span_id], summaries.get(span_id), hashes) for span_id in span_ids]
+    return JudgmentRequest(trace.run_id, spec, tuple(evidence))
+
+
+@dataclass(frozen=True)
+class LocalCallbackJudge:
+    """Trusted synchronous callback; no imports, provider client or worker threads."""
+
+    identity: JudgeIdentity
+    callback: Callable
+
+    def __post_init__(self):
+        if type(self.identity) is not JudgeIdentity or not callable(self.callback):
+            raise ValueError("local judge requires an identity and callable")
+        if inspect.iscoroutinefunction(self.callback) or inspect.isasyncgenfunction(self.callback):
+            raise ValueError("local callbacks must be synchronous")
+
+    def judge(self, request, *, timeout_s=None):
+        return self.callback(request, timeout_s=timeout_s)
+
+
+class JudgmentSession:
+    """One backend and a bounded session-local LRU; execution requires enabled=True.
+
+    Not thread-safe. Use one session per analysis worker. No cache is shared
+    across backend instances or restored from untrusted trace metadata.
+    """
+
+    def __init__(self, backend: JudgmentBackend, *, cache_size: int = 128):
+        if type(cache_size) is not int or cache_size < 0:
+            raise ValueError("cache_size must be a nonnegative integer")
+        if type(backend.identity) is not JudgeIdentity or not callable(backend.judge):
+            raise ValueError("backend must implement JudgmentBackend")
+        self._backend = backend
+        self._cache_size = cache_size
+        self._cache = OrderedDict()
+
+    def clear_cache(self):
+        self._cache.clear()
+
+    def evaluate(self, request: JudgmentRequest, *, enabled=False, timeout_s=None):
+        if type(request) is not JudgmentRequest or type(enabled) is not bool:
+            raise ValueError("evaluate requires a typed request and boolean enabled flag")
+        if timeout_s is not None and (not finite(timeout_s) or timeout_s <= 0):
+            raise ValueError("timeout_s must be finite and positive")
+        identity = self._backend.identity
+        if type(identity) is not JudgeIdentity:
+            raise ValueError("backend identity must be JudgeIdentity")
+        request_data, judge = request.to_dict(), asdict(identity)
+        key = fingerprint(
+            {"schema_version": JUDGMENT_VERSION, "request": request_data, "judge": judge}
+        )
+        started = time.perf_counter()
+        hit = enabled and key in self._cache
+        if hit:
+            evaluation = _copy(self._cache[key])
+            self._cache.move_to_end(key)
+        else:
+            evaluation = self._evaluate(request, enabled, timeout_s)
+            if enabled and self._backend.identity != identity:
+                evaluation.update(status="invalid_result", value=None, reason="identity_changed")
+            if evaluation["status"] == "known" and self._cache_size:
+                self._cache[key] = _copy(evaluation)
+                if len(self._cache) > self._cache_size:
+                    self._cache.popitem(last=False)
+        record = {
+            "schema_version": JUDGMENT_VERSION,
+            "evidence_kind": "semantic_judgment",
+            "request": request_data,
+            "judge": judge,
+            "cache_key": key,
+            "evaluation": evaluation,
+            "invocation": {
+                "id": uuid4().hex,
+                "cache_hit": hit,
+                "dispatched": enabled and not hit,
+                "latency_ms": (time.perf_counter() - started) * 1000,
+                "usage": asdict(_NO_USAGE) if hit or not enabled else evaluation["usage"],
+            },
+        }
+        record["record_hash"] = fingerprint(record)
+        return _copy(record)
+
+    def _evaluate(self, request, enabled, timeout_s):
+        started = time.perf_counter()
+        result = {
+            "id": uuid4().hex,
+            "evaluated_at": utc_now_iso(),
+            "status": "disabled",
+            "value": None,
+            "reason": "disabled",
+            "usage": asdict(_NO_USAGE if not enabled else JudgeUsage()),
+            "uncertainty": asdict(JudgmentUncertainty()),
+        }
+        if enabled:
+            try:
+                answer = self._backend.judge(request, timeout_s=timeout_s)
+                if type(answer) is not JudgmentAnswer:
+                    if inspect.iscoroutine(answer) or inspect.isgenerator(answer):
+                        answer.close()
+                    result.update(status="invalid_result", reason="invalid_result")
+                else:
+                    result.update(asdict(answer))
+                    if answer.status == "known" and not request.spec.accepts(answer.value):
+                        result.update(status="invalid_result", value=None, reason="invalid_result")
+            except TimeoutError:
+                result.update(status="timeout", reason="timeout")
+            except Exception:
+                # Neither exception text nor arbitrary class names leave the callback.
+                result.update(status="error", reason="error")
+        elapsed = time.perf_counter() - started
+        if enabled and timeout_s is not None and elapsed > timeout_s:
+            result.update(status="timeout", value=None, reason="timeout")
+        result["latency_ms"] = elapsed * 1000
+        return result
+
+
+def validate_judgment_record(record):
+    """Validate portable integrity and typed semantics, not a judge's truthfulness."""
+    value = _copy(record)
+    try:
+        digest = value.pop("record_hash")
+        if digest != fingerprint(value) or value["schema_version"] != JUDGMENT_VERSION:
+            raise ValueError("unsupported or altered judgment record")
+        if value["evidence_kind"] != "semantic_judgment":
+            raise ValueError("invalid evidence kind")
+        request = JudgmentRequest.from_dict(value["request"])
+        judge = JudgeIdentity(**value["judge"])
+        key = fingerprint(
+            {
+                "schema_version": JUDGMENT_VERSION,
+                "request": request.to_dict(),
+                "judge": asdict(judge),
+            }
+        )
+        if value["cache_key"] != key:
+            raise ValueError("judgment cache identity mismatch")
+        result, invocation = value["evaluation"], value["invocation"]
+        usage = JudgeUsage(**result["usage"])
+        uncertainty = JudgmentUncertainty(**result["uncertainty"])
+        if result["status"] in {"known", "unknown"}:
+            answer = JudgmentAnswer(
+                result["value"], result["status"], result["reason"], usage, uncertainty
+            )
+            if answer.status == "known" and not request.spec.accepts(answer.value):
+                raise ValueError("answer violates typed bounds")
+        elif (
+            result["status"] not in _FAILURES
+            or result["value"] is not None
+            or result["reason"] not in _FAILURES | {"identity_changed"}
+        ):
+            raise ValueError("invalid failure result")
+        for part in (result, invocation):
+            if (
+                not isinstance(part["id"], str)
+                or not part["id"]
+                or not finite(part["latency_ms"])
+                or part["latency_ms"] < 0
+            ):
+                raise ValueError("invalid execution receipt")
+        if not isinstance(result["evaluated_at"], str):
+            raise ValueError("invalid evaluation timestamp")
+        if type(invocation["cache_hit"]) is not bool or type(invocation["dispatched"]) is not bool:
+            raise ValueError("invalid cache receipt")
+        incurred = JudgeUsage(**invocation["usage"])
+        if invocation["cache_hit"]:
+            if invocation["dispatched"] or result["status"] != "known":
+                raise ValueError("only successful undispatched evaluations can be cached")
+        if not invocation["dispatched"]:
+            if incurred != _NO_USAGE or (
+                not invocation["cache_hit"] and result["status"] != "disabled"
+            ):
+                raise ValueError("invalid undispatched receipt")
+        elif incurred != usage or result["status"] == "disabled":
+            raise ValueError("invalid dispatched receipt")
+        value["record_hash"] = digest
+        return value
+    except (KeyError, TypeError, AttributeError):
+        raise ValueError("invalid judgment record") from None
+
+
+def _bound(trace, record, *, events=None, hashes=None):
+    request = JudgmentRequest.from_dict(record["request"])
+    if request.run_id != trace.run_id:
+        return False
+    if events is None:
+        events = {event.event_id: event for event in trace.events}
+    if len(events) != len(trace.events):
+        return False
+    hashes = {} if hashes is None else hashes
+    return all(
+        item.span_id in events and _evidence(events[item.span_id], item.summary, hashes) == item
+        for item in request.evidence
+    )
+
+
+def attach_judgment(trace, record):
+    """Attach a validated receipt without replacing host metadata or prior results."""
+    owned = validate_judgment_record(record)
+    if not _bound(trace, owned):
+        raise ValueError("judgment evidence does not match the trace")
+    existing = trace.metadata.get(JUDGMENT_KEY)
+    if JUDGMENT_KEY not in trace.metadata:
+        existing = {"schema_version": JUDGMENT_VERSION, "records": []}
+    elif read_judgments(trace)["status"] in {"invalid", "unsupported"}:
+        raise ValueError("existing judgment evidence is invalid or unsupported")
+    existing = _copy(existing)
+    for previous in existing["records"]:
+        if previous["invocation"]["id"] == owned["invocation"]["id"]:
+            if previous != owned:
+                raise ValueError("conflicting invocation receipt")
+            return
+    existing["records"].append(owned)
+    trace.metadata[JUDGMENT_KEY] = existing
+
+
+def read_judgments(trace):
+    """Read saved evidence only. Stale or failed judgments never become booleans."""
+    metadata = getattr(trace, "metadata", {})
+    envelope = metadata.get(JUDGMENT_KEY)
+    if JUDGMENT_KEY not in metadata:
+        return None
+    empty = {"status": "invalid", "records": [], "known_cost_usd": None, "cost_complete": False}
+    if not isinstance(envelope, dict):
+        return empty
+    if envelope.get("schema_version") != JUDGMENT_VERSION:
+        return {**empty, "status": "unsupported"}
+    try:
+        if not isinstance(envelope.get("records"), list):
+            return empty
+        records, identities, known_cost, complete_cost = [], set(), 0.0, True
+        events = {event.event_id: event for event in trace.events}
+        hashes = {}
+        for raw in envelope["records"]:
+            record = validate_judgment_record(raw)
+            identity = record["invocation"]["id"]
+            if identity in identities:
+                return empty
+            identities.add(identity)
+            try:
+                bound = _bound(trace, record, events=events, hashes=hashes)
+            except ValueError:
+                bound = False
+            result = record["evaluation"]
+            record["effective_status"] = result["status"] if bound else "stale"
+            record["effective_value"] = (
+                result["value"] if bound and result["status"] == "known" else None
+            )
+            cost = record["invocation"]["usage"]["cost_usd"]
+            if cost is None:
+                complete_cost = False
+            else:
+                known_cost += cost
+            records.append(record)
+        return {
+            "status": "complete"
+            if records and all(item["effective_status"] == "known" for item in records)
+            else "incomplete",
+            "records": records,
+            "known_cost_usd": known_cost if finite(known_cost) else None,
+            "cost_complete": complete_cost and finite(known_cost),
+            "cost_scope": "offline_judge_invocations_only",
+        }
+    except (ValueError, TypeError, KeyError, AttributeError):
+        return empty