costtrace 0.1.0__py3-none-any.whl

CostTrace/__init__.py

@@ -0,0 +1,60 @@
+"""CostTrace — a deterministic cost-attribution profiler for LLM spend.
+
+Public API:
+
+    from CostTrace import capture
+    capture(result, conversation_id)          # the dev one-liner
+
+    from CostTrace import load_captures, fetch_captures
+    captures = load_captures("costtrace_captures.json")   # validated ids
+    report = fetch_captures(captures)                      # validated traces
+    # report.conversations -> analyze;  report.rejections -> what was dropped
+"""
+
+from __future__ import annotations
+
+from .capture import capture
+from .capture_schema import (
+    CAPTURE_VERSION,
+    CaptureFile,
+    CaptureRecord,
+    CaptureSchemaError,
+    Provider,
+)
+from .fetch import fetch_captures
+from .ingest import IngestError, load_captures, parse_captures
+from .pricing import Pricer
+from .providers import FetchReport, ProviderAuthError, Rejection, supported_providers
+from .trace import CallTrace, ConversationTrace, Message, MessageKind
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # capture
+    "capture",
+    # ingest
+    "load_captures",
+    "parse_captures",
+    "IngestError",
+    # fetch
+    "fetch_captures",
+    "FetchReport",
+    "Rejection",
+    "ProviderAuthError",
+    "supported_providers",
+    # schema
+    "CaptureFile",
+    "CaptureRecord",
+    "CaptureSchemaError",
+    "Provider",
+    "CAPTURE_VERSION",
+    # trace model
+    "ConversationTrace",
+    "CallTrace",
+    "Message",
+    "MessageKind",
+    # pricing
+    "Pricer",
+    "__version__",
+]
+

CostTrace/analysis/__init__.py

@@ -0,0 +1,42 @@
+"""CostTrace analysis — the brains.
+
+Pipeline over validated ConversationTraces:
+    metrics   -> per-session rows (cost, bucket split, prompt hash)
+    store     -> durable JSONL history (append/upsert/prune)
+    aggregate -> overall spend + per-prompt-version views
+    detect    -> what changed across every dimension (grounded findings)
+    judge     -> LLM explains the likely cause in plain English
+"""
+
+from .aggregate import (
+    OverallSummary,
+    ModelSummary,
+    PromptVersion,
+    group_by_prompt_version,
+    summarize_by_model,
+    summarize_overall,
+)
+from .detect import Finding, detect_all, detect_changes
+from .judge import Judgment, build_digest, judge
+from .metrics import SessionMetric, compute_session_metric, system_prompt_hash
+from .store import load_metrics, upsert_metrics
+
+__all__ = [
+    "SessionMetric",
+    "compute_session_metric",
+    "system_prompt_hash",
+    "load_metrics",
+    "upsert_metrics",
+    "OverallSummary",
+    "ModelSummary",
+    "PromptVersion",
+    "summarize_overall",
+    "summarize_by_model",
+    "group_by_prompt_version",
+    "Finding",
+    "detect_all",
+    "detect_changes",
+    "Judgment",
+    "build_digest",
+    "judge",
+]

CostTrace/analysis/aggregate.py

@@ -0,0 +1,149 @@
+"""Aggregation — roll SessionMetrics into higher-level views for the judge.
+
+Two views:
+  * overall spend + where it goes (bucket totals across all sessions)
+  * per system-prompt version over time (the basis for regression detection)
+
+Pure functions over already-computed metric rows — no fetching, no LLM.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+from .metrics import SessionMetric
+
+
+@dataclass
+class OverallSummary:
+    """Where the whole bill goes, across every session in the store."""
+
+    num_sessions: int
+    total_cost: float
+    bucket_cost: Dict[str, float]          # $ per bucket
+    bucket_pct: Dict[str, float]           # % of total per bucket
+    avg_cost_per_turn: float
+    avg_cache_hit_rate: float
+
+
+@dataclass
+class PromptVersion:
+    """Stats for one system-prompt fingerprint over the sessions that used it."""
+
+    system_prompt_hash: str
+    system_prompt_text: str
+    num_sessions: int
+    total_cost: float
+    avg_cost_per_turn: float
+    avg_cache_hit_rate: float
+    first_seen: str = ""
+    last_seen: str = ""
+    _cpt_samples: List[float] = field(default_factory=list, repr=False)
+
+
+@dataclass
+class ModelSummary:
+    """Cost profile for one model, across the sessions that used it."""
+
+    model: str
+    num_sessions: int
+    total_turns: int
+    total_cost: float
+    avg_cost_per_turn: float
+    avg_cache_hit_rate: float
+    avg_output_tokens_per_turn: float
+
+
+def summarize_by_model(metrics: List[SessionMetric]) -> List[ModelSummary]:
+    """Group sessions by model; return summaries sorted by cost/turn (desc)."""
+    groups: Dict[str, List[SessionMetric]] = {}
+    for m in metrics:
+        groups.setdefault(m.model, []).append(m)
+
+    out: List[ModelSummary] = []
+    for model, rows in groups.items():
+        n = len(rows)
+        total_turns = sum(r.num_turns for r in rows)
+        out.append(ModelSummary(
+            model=model,
+            num_sessions=n,
+            total_turns=total_turns,
+            total_cost=round(sum(r.total_cost for r in rows), 6),
+            avg_cost_per_turn=round(sum(r.cost_per_turn for r in rows) / n, 8),
+            avg_cache_hit_rate=round(sum(r.cache_hit_rate for r in rows) / n, 4),
+            avg_output_tokens_per_turn=round(
+                sum((r.output_tokens / r.num_turns) if r.num_turns else 0
+                    for r in rows) / n, 1),
+        ))
+    out.sort(key=lambda s: s.avg_cost_per_turn, reverse=True)
+    return out
+
+
+def summarize_overall(metrics: List[SessionMetric]) -> OverallSummary:
+    """Total spend and bucket split across all sessions."""
+    if not metrics:
+        return OverallSummary(0, 0.0, {}, {}, 0.0, 0.0)
+
+    bucket_cost: Dict[str, float] = {}
+    total_cost = 0.0
+    cpt_sum = 0.0
+    cache_sum = 0.0
+    for m in metrics:
+        total_cost += m.total_cost
+        cpt_sum += m.cost_per_turn
+        cache_sum += m.cache_hit_rate
+        for bucket, cost in m.bucket_cost.items():
+            bucket_cost[bucket] = bucket_cost.get(bucket, 0.0) + cost
+
+    bucket_pct = {
+        b: round(100.0 * c / total_cost, 2) if total_cost else 0.0
+        for b, c in bucket_cost.items()
+    }
+    n = len(metrics)
+    return OverallSummary(
+        num_sessions=n,
+        total_cost=round(total_cost, 6),
+        bucket_cost={b: round(c, 6) for b, c in bucket_cost.items()},
+        bucket_pct=bucket_pct,
+        avg_cost_per_turn=round(cpt_sum / n, 8),
+        avg_cache_hit_rate=round(cache_sum / n, 4),
+    )
+
+
+def group_by_prompt_version(metrics: List[SessionMetric]) -> List[PromptVersion]:
+    """Group sessions by system-prompt hash; return versions sorted by time."""
+    versions: Dict[str, PromptVersion] = {}
+    for m in metrics:
+        v = versions.get(m.system_prompt_hash)
+        if v is None:
+            v = PromptVersion(
+                system_prompt_hash=m.system_prompt_hash,
+                system_prompt_text=m.system_prompt_text,
+                num_sessions=0,
+                total_cost=0.0,
+                avg_cost_per_turn=0.0,
+                avg_cache_hit_rate=0.0,
+                first_seen=m.last_seen or m.computed_at or "",
+                last_seen=m.last_seen or m.computed_at or "",
+            )
+            versions[m.system_prompt_hash] = v
+        v.num_sessions += 1
+        v.total_cost += m.total_cost
+        v._cpt_samples.append(m.cost_per_turn)
+        v.avg_cache_hit_rate += m.cache_hit_rate
+        stamp = m.last_seen or m.computed_at or ""
+        if stamp:
+            v.first_seen = min(v.first_seen or stamp, stamp)
+            v.last_seen = max(v.last_seen or stamp, stamp)
+
+    out: List[PromptVersion] = []
+    for v in versions.values():
+        n = v.num_sessions or 1
+        v.total_cost = round(v.total_cost, 6)
+        v.avg_cost_per_turn = round(sum(v._cpt_samples) / n, 8)
+        v.avg_cache_hit_rate = round(v.avg_cache_hit_rate / n, 4)
+        out.append(v)
+
+    out.sort(key=lambda v: v.first_seen)
+    return out

CostTrace/analysis/detect.py

@@ -0,0 +1,245 @@
+"""Detection — deterministic signals about *what changed*, across any dimension.
+
+We do NOT assume a single cause. Cost-per-turn can rise from a model switch,
+history bloat, longer outputs, a broken cache, bigger retrieved context, more
+turns, or a system-prompt change. So we compare a recent window against a prior
+window and surface every dimension that moved, each as grounded evidence.
+
+The LLM judge then reasons over these signals to name the likely cause(s). The
+system-prompt change is just one of the signals here, not the whole story.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from ..capture_schema import _parse_ts
+from .aggregate import OverallSummary
+from .metrics import SessionMetric
+
+# Thresholds (deliberately simple; tune later).
+_COST_JUMP_PCT = 20.0          # cost-per-turn rise worth flagging
+_CACHE_DROP_ABS = 0.20         # absolute cache-hit-rate drop
+_BUCKET_SHIFT_PCT = 10.0       # percentage-point shift in a bucket's share
+_OUTPUT_GROWTH_PCT = 25.0      # rise in output tokens per turn
+_DOMINANT_BUCKET_PCT = 50.0    # a bucket over this share is worth calling out
+
+
+@dataclass
+class Finding:
+    """One flagged signal, with evidence for the judge to reason over."""
+
+    kind: str          # e.g. "cost_increase", "model_shift", "bucket_shift",
+                       #      "cache_drop", "output_growth", "prompt_change"
+    severity: str      # "high" | "medium" | "low"
+    summary: str       # human-readable, number-grounded
+    evidence: dict = field(default_factory=dict)
+
+
+@dataclass
+class PeriodStats:
+    """Aggregate stats for one time window."""
+
+    num_sessions: int
+    avg_cost_per_turn: float
+    avg_cache_hit_rate: float
+    avg_output_tokens_per_turn: float
+    bucket_pct: Dict[str, float]
+    model_mix: Dict[str, float]           # model -> fraction of sessions
+    dominant_prompt_hash: str
+    dominant_prompt_text: str
+
+
+def _period_stats(metrics: List[SessionMetric]) -> PeriodStats:
+    n = len(metrics)
+    if n == 0:
+        return PeriodStats(0, 0.0, 0.0, 0.0, {}, {}, "", "")
+
+    cpt = sum(m.cost_per_turn for m in metrics) / n
+    cache = sum(m.cache_hit_rate for m in metrics) / n
+    out_per_turn = sum(
+        (m.output_tokens / m.num_turns) if m.num_turns else 0 for m in metrics
+    ) / n
+
+    bucket_cost: Dict[str, float] = {}
+    total = 0.0
+    for m in metrics:
+        total += m.total_cost
+        for b, c in m.bucket_cost.items():
+            bucket_cost[b] = bucket_cost.get(b, 0.0) + c
+    bucket_pct = {
+        b: round(100.0 * c / total, 2) if total else 0.0 for b, c in bucket_cost.items()
+    }
+
+    model_counts: Dict[str, int] = {}
+    hash_counts: Dict[str, int] = {}
+    hash_text: Dict[str, str] = {}
+    for m in metrics:
+        model_counts[m.model] = model_counts.get(m.model, 0) + 1
+        hash_counts[m.system_prompt_hash] = hash_counts.get(m.system_prompt_hash, 0) + 1
+        hash_text[m.system_prompt_hash] = m.system_prompt_text
+    model_mix = {mdl: round(c / n, 3) for mdl, c in model_counts.items()}
+    dom_hash = max(hash_counts, key=hash_counts.get) if hash_counts else ""
+
+    return PeriodStats(
+        num_sessions=n,
+        avg_cost_per_turn=round(cpt, 8),
+        avg_cache_hit_rate=round(cache, 4),
+        avg_output_tokens_per_turn=round(out_per_turn, 1),
+        bucket_pct=bucket_pct,
+        model_mix=model_mix,
+        dominant_prompt_hash=dom_hash,
+        dominant_prompt_text=hash_text.get(dom_hash, ""),
+    )
+
+
+def split_periods(
+    metrics: List[SessionMetric],
+) -> Optional[tuple]:
+    """Split metrics into (prior, recent) halves by time. None if too few."""
+    dated = [m for m in metrics if _parse_ts(m.last_seen or m.computed_at)]
+    if len(dated) < 4:
+        return None
+    dated.sort(key=lambda m: _parse_ts(m.last_seen or m.computed_at))
+    mid = len(dated) // 2
+    return dated[:mid], dated[mid:]
+
+
+def detect_changes(metrics: List[SessionMetric]) -> List[Finding]:
+    """Compare recent vs prior window; flag every dimension that moved."""
+    split = split_periods(metrics)
+    if split is None:
+        return []
+    prior_m, recent_m = split
+    prior = _period_stats(prior_m)
+    recent = _period_stats(recent_m)
+    findings: List[Finding] = []
+
+    # 1) headline: did cost-per-turn rise?
+    cost_jump_pct = None
+    if prior.avg_cost_per_turn > 0:
+        cost_jump_pct = 100.0 * (
+            recent.avg_cost_per_turn - prior.avg_cost_per_turn
+        ) / prior.avg_cost_per_turn
+        if cost_jump_pct >= _COST_JUMP_PCT:
+            findings.append(Finding(
+                kind="cost_increase",
+                severity="high" if cost_jump_pct >= 40 else "medium",
+                summary=(
+                    f"Cost per turn rose {cost_jump_pct:.0f}% "
+                    f"(${prior.avg_cost_per_turn:.6f} -> ${recent.avg_cost_per_turn:.6f})"
+                ),
+                evidence={
+                    "prior_cost_per_turn": prior.avg_cost_per_turn,
+                    "recent_cost_per_turn": recent.avg_cost_per_turn,
+                    "pct": round(cost_jump_pct, 1),
+                },
+            ))
+
+    # 2) candidate drivers (surface regardless; judge decides which explains #1)
+    # model mix shift / new model
+    new_models = {m: f for m, f in recent.model_mix.items()
+                  if f - prior.model_mix.get(m, 0.0) >= 0.2}
+    if new_models:
+        findings.append(Finding(
+            kind="model_shift",
+            severity="medium",
+            summary=f"Model mix shifted toward {', '.join(new_models)}",
+            evidence={"prior_mix": prior.model_mix, "recent_mix": recent.model_mix},
+        ))
+
+    # bucket share shifts (history bloat, output growth, retrieval growth, ...)
+    for bucket in set(prior.bucket_pct) | set(recent.bucket_pct):
+        delta = recent.bucket_pct.get(bucket, 0.0) - prior.bucket_pct.get(bucket, 0.0)
+        if delta >= _BUCKET_SHIFT_PCT:
+            findings.append(Finding(
+                kind="bucket_shift",
+                severity="medium",
+                summary=(
+                    f"'{bucket}' share grew "
+                    f"{prior.bucket_pct.get(bucket, 0):.0f}% -> "
+                    f"{recent.bucket_pct.get(bucket, 0):.0f}% of spend"
+                ),
+                evidence={"bucket": bucket,
+                          "prior_pct": prior.bucket_pct.get(bucket, 0.0),
+                          "recent_pct": recent.bucket_pct.get(bucket, 0.0)},
+            ))
+
+    # cache degradation
+    cache_drop = prior.avg_cache_hit_rate - recent.avg_cache_hit_rate
+    if cache_drop >= _CACHE_DROP_ABS:
+        findings.append(Finding(
+            kind="cache_drop",
+            severity="high" if cache_drop >= 0.4 else "medium",
+            summary=(
+                f"Cache-hit rate fell {prior.avg_cache_hit_rate:.0%} -> "
+                f"{recent.avg_cache_hit_rate:.0%}"
+            ),
+            evidence={"prior": prior.avg_cache_hit_rate, "recent": recent.avg_cache_hit_rate},
+        ))
+
+    # output length growth
+    if prior.avg_output_tokens_per_turn > 0:
+        out_pct = 100.0 * (
+            recent.avg_output_tokens_per_turn - prior.avg_output_tokens_per_turn
+        ) / prior.avg_output_tokens_per_turn
+        if out_pct >= _OUTPUT_GROWTH_PCT:
+            findings.append(Finding(
+                kind="output_growth",
+                severity="medium",
+                summary=(
+                    f"Output length per turn grew {out_pct:.0f}% "
+                    f"({prior.avg_output_tokens_per_turn:.0f} -> "
+                    f"{recent.avg_output_tokens_per_turn:.0f} tokens)"
+                ),
+                evidence={"prior": prior.avg_output_tokens_per_turn,
+                          "recent": recent.avg_output_tokens_per_turn,
+                          "pct": round(out_pct, 1)},
+            ))
+
+    # system-prompt change (one signal among the above)
+    if (prior.dominant_prompt_hash and recent.dominant_prompt_hash
+            and prior.dominant_prompt_hash != recent.dominant_prompt_hash):
+        findings.append(Finding(
+            kind="prompt_change",
+            severity="medium",
+            summary=(
+                f"Dominant system prompt changed "
+                f"({prior.dominant_prompt_hash} -> {recent.dominant_prompt_hash})"
+            ),
+            evidence={
+                "prior_hash": prior.dominant_prompt_hash,
+                "recent_hash": recent.dominant_prompt_hash,
+                "prior_prompt": prior.dominant_prompt_text,
+                "recent_prompt": recent.dominant_prompt_text,
+            },
+        ))
+
+    return findings
+
+
+def detect_dominant_bucket(overall: OverallSummary) -> List[Finding]:
+    """Static view: flag when one bucket dominates the whole bill."""
+    findings: List[Finding] = []
+    for bucket, pct in overall.bucket_pct.items():
+        if pct >= _DOMINANT_BUCKET_PCT:
+            findings.append(Finding(
+                kind="dominant_bucket",
+                severity="medium",
+                summary=f"'{bucket}' is {pct:.0f}% of total spend "
+                        f"(${overall.bucket_cost.get(bucket, 0):.6f})",
+                evidence={"bucket": bucket, "pct": pct,
+                          "cost": overall.bucket_cost.get(bucket, 0.0)},
+            ))
+    return findings
+
+
+def detect_all(
+    overall: OverallSummary, metrics: List[SessionMetric]
+) -> List[Finding]:
+    """Run every detector; return findings ordered high -> low severity."""
+    order = {"high": 0, "medium": 1, "low": 2}
+    findings = detect_changes(metrics) + detect_dominant_bucket(overall)
+    findings.sort(key=lambda f: order.get(f.severity, 3))
+    return findings

CostTrace/analysis/judge.py

@@ -0,0 +1,141 @@
+"""LLM-as-judge — explain, in plain English, what drove the cost.
+
+We never send raw conversations. We send a compact, number-grounded digest:
+the overall spend split plus the deterministic findings (what changed across
+every dimension). The LLM's job is to reason over those signals and name the
+likely cause(s) and a fix — NOT to compute numbers.
+
+The HTTP call is injectable so this is unit-testable without network/keys.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Callable, List, Optional
+
+from .aggregate import ModelSummary, OverallSummary
+from .detect import Finding
+
+_API_URL = "https://api.openai.com/v1/chat/completions"
+_DEFAULT_MODEL = "gpt-4o-mini"
+_PROMPT_TRUNC = 1200   # chars of each system prompt to include
+
+_RUBRIC = (
+    "You are a FinOps cost auditor for LLM applications. You are given a "
+    "GROUNDED digest of real usage: an overall spend breakdown and a list of "
+    "detected changes across dimensions (cost/turn, model mix, cost buckets, "
+    "cache-hit rate, output length, system-prompt fingerprint).\n\n"
+    "Your job:\n"
+    "1. State where the money is going (one line).\n"
+    "2. If a per-model comparison is present, explain the cost difference "
+    "between the models for the same workload (cite cost/turn and output "
+    "length).\n"
+    "3. If cost rose over time, name the MOST LIKELY cause by correlating the "
+    "signals — do NOT assume it was the system prompt; it may be a model "
+    "switch, history growth, longer outputs, a broken cache, or something "
+    "else. Cite the specific numbers.\n"
+    "4. Give one concrete, actionable recommendation.\n\n"
+    "Rules: use ONLY the numbers provided; never invent figures. Be concise "
+    "(a few sentences). If nothing significant changed, say so."
+)
+
+
+@dataclass
+class Judgment:
+    verdict: str          # the model's plain-English explanation
+    model: str
+    prompt_chars: int     # size of the digest we sent (for transparency)
+
+
+def _default_chat(payload: dict, api_key: str) -> dict:
+    req = urllib.request.Request(
+        _API_URL,
+        data=json.dumps(payload).encode("utf-8"),
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        },
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as r:
+            return json.load(r)
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8", "ignore")[:200]
+        raise RuntimeError(f"judge LLM call failed: HTTP {e.code} {detail}") from e
+    except urllib.error.URLError as e:
+        raise RuntimeError(f"judge LLM call failed: {e.reason}") from e
+
+
+def build_digest(
+    overall: OverallSummary,
+    findings: List[Finding],
+    by_model: Optional[List[ModelSummary]] = None,
+) -> str:
+    """Assemble the compact, grounded text sent to the judge."""
+    lines: List[str] = []
+    lines.append("=== OVERALL SPEND ===")
+    lines.append(f"sessions: {overall.num_sessions}")
+    lines.append(f"total_cost: ${overall.total_cost:.6f}")
+    lines.append(f"avg_cost_per_turn: ${overall.avg_cost_per_turn:.6f}")
+    lines.append(f"avg_cache_hit_rate: {overall.avg_cache_hit_rate:.0%}")
+    lines.append("bucket_share_%: " + ", ".join(
+        f"{b}={p:.0f}%" for b, p in sorted(overall.bucket_pct.items())
+    ))
+
+    if by_model and len(by_model) > 1:
+        lines.append("\n=== PER-MODEL COMPARISON (same workload) ===")
+        for s in by_model:
+            lines.append(
+                f"{s.model}: cost/turn=${s.avg_cost_per_turn:.6f}, "
+                f"cache_hit={s.avg_cache_hit_rate:.0%}, "
+                f"out_tokens/turn={s.avg_output_tokens_per_turn:.0f}, "
+                f"sessions={s.num_sessions}, turns={s.total_turns}"
+            )
+
+    lines.append("\n=== DETECTED CHANGES (recent window vs prior) ===")
+    if not findings:
+        lines.append("none significant")
+    for i, f in enumerate(findings, 1):
+        lines.append(f"[{i}] ({f.severity}) {f.kind}: {f.summary}")
+        if f.kind == "prompt_change":
+            lines.append("    prior_prompt: "
+                         + (f.evidence.get("prior_prompt", "")[:_PROMPT_TRUNC]))
+            lines.append("    recent_prompt: "
+                         + (f.evidence.get("recent_prompt", "")[:_PROMPT_TRUNC]))
+    return "\n".join(lines)
+
+
+def judge(
+    overall: OverallSummary,
+    findings: List[Finding],
+    by_model: Optional[List[ModelSummary]] = None,
+    api_key: Optional[str] = None,
+    model: str = _DEFAULT_MODEL,
+    chat: Optional[Callable[[dict, str], dict]] = None,
+) -> Judgment:
+    """Ask the LLM to explain the cost picture from the grounded digest."""
+    key = api_key or os.environ.get("OPENAI_API_KEY", "")
+    if not key:
+        raise RuntimeError("no API key: set OPENAI_API_KEY or pass api_key=")
+    caller = chat or _default_chat
+
+    digest = build_digest(overall, findings, by_model)
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": _RUBRIC},
+            {"role": "user", "content": digest},
+        ],
+        "temperature": 0,
+    }
+    resp = caller(payload, key)
+    verdict = (
+        resp.get("choices", [{}])[0].get("message", {}).get("content", "").strip()
+        or "(no verdict returned)"
+    )
+    return Judgment(verdict=verdict, model=model, prompt_chars=len(digest))