memtrust-cli 0.3.0__py3-none-any.whl

memtrust/receipt.py

@@ -0,0 +1,318 @@
+"""Cryptographic signing and verification for memtrust's own JSON run output.
+
+This exists to close a specific, narrow credibility gap: a `memtrust run`
+JSON report is just a file on disk. Anyone with filesystem or transport
+access between the run and a reader can edit a number in it, and nothing
+about the file itself would show that. A *signed receipt* fixes that one
+problem -- it does not fix, and does not claim to fix, whether the
+underlying benchmark numbers are accurate in the first place. See
+docs/methodology.md's "Signed receipts" section for the full claim/caveat
+split; the short version is repeated in `verify_receipt()`'s docstring
+below because that is the function whose return value people will actually
+build trust decisions on.
+
+Design:
+
+- `canonicalize()` produces a deterministic byte encoding of a JSON-able
+  payload (sorted keys, no extraneous whitespace, UTF-8) so the same
+  logical report always signs to the same bytes regardless of dict
+  insertion order or formatting.
+- A receipt is a small JSON document: the canonicalized payload's SHA-256
+  hex digest, an Ed25519 signature over the *canonical payload bytes*
+  (not the hash -- Ed25519 handles arbitrary-length messages natively, so
+  hashing first buys nothing and only adds a place for the two to
+  silently disagree), the signer's public key (raw 32 bytes, base64), an
+  ISO-8601 UTC timestamp, and the payload itself, embedded so the receipt
+  is self-contained and reproducible without the original report file.
+- Verification always requires a public key supplied by the *caller*
+  (a file path or an env var) -- never the public key embedded in the
+  receipt being verified. Trusting the embedded key would let an attacker
+  who can rewrite the payload and signature also rewrite the key field
+  and produce an internally-"consistent" but meaningless forgery. The
+  embedded key is carried only for display/reference (e.g. "who does this
+  receipt claim signed it") and is cross-checked against the caller's
+  trusted key as an extra diagnostic, never as the basis for trust.
+
+Generating a keypair:
+
+    memtrust keygen --private-key-out mykey.pem --public-key-out mykey.pub
+
+or, using plain `cryptography`/`openssl` directly (this is exactly what
+`memtrust keygen` does under the hood, so either path produces
+interoperable PEM files):
+
+    openssl genpkey -algorithm ed25519 -out mykey.pem
+    openssl pkey -in mykey.pem -pubout -out mykey.pub
+"""
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import hashlib
+import json
+import os
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+    Ed25519PrivateKey,
+    Ed25519PublicKey,
+)
+
+#: Bumped only if the receipt's JSON shape changes in a way that would
+#: break an older verifier -- new optional fields do not require a bump.
+RECEIPT_FORMAT_VERSION = 1
+
+ALGORITHM = "Ed25519"
+
+#: Fallback env var `memtrust verify` reads a trusted public key from when
+#: no --public-key path is given on the command line.
+PUBLIC_KEY_ENV_VAR = "MEMTRUST_RECEIPT_PUBLIC_KEY"
+
+
+class ReceiptError(Exception):
+    """Raised for malformed receipts, keys, or signing inputs -- never for
+    "signature didn't match" (that's a normal, expected verify() outcome
+    reported via VerifyResult.valid=False, not an exception)."""
+
+
+def canonicalize(payload: dict[str, Any]) -> bytes:
+    """Deterministic JSON encoding: sorted keys, compact separators, UTF-8.
+
+    Two payloads that are equal as Python dicts always canonicalize to the
+    same bytes regardless of original key order. This is what actually
+    gets signed and hashed -- never the payload's original on-disk
+    formatting, which is not guaranteed stable.
+    """
+    return json.dumps(
+        payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True, default=str
+    ).encode("utf-8")
+
+
+def payload_sha256_hex(payload: dict[str, Any]) -> str:
+    """SHA-256 hex digest of the payload's canonical bytes."""
+    return hashlib.sha256(canonicalize(payload)).hexdigest()
+
+
+def generate_keypair() -> tuple[Ed25519PrivateKey, Ed25519PublicKey]:
+    """Generate a fresh Ed25519 keypair. Callers own persisting it (or not)."""
+    private_key = Ed25519PrivateKey.generate()
+    return private_key, private_key.public_key()
+
+
+def write_keypair(
+    private_key_path: Path, public_key_path: Path, *, overwrite: bool = False
+) -> None:
+    """Generate a keypair and write both halves as PEM files.
+
+    Private key: PKCS8, unencrypted (matches `openssl genpkey -algorithm
+    ed25519`). Public key: SubjectPublicKeyInfo (matches `openssl pkey
+    -pubout`). Either file produced by the `openssl` commands in this
+    module's docstring loads correctly here, and vice versa.
+    """
+    if not overwrite:
+        existing = [p for p in (private_key_path, public_key_path) if p.exists()]
+        if existing:
+            names = ", ".join(str(p) for p in existing)
+            raise ReceiptError(f"refusing to overwrite existing file(s): {names} (use --force)")
+
+    private_key, public_key = generate_keypair()
+
+    private_pem = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.PKCS8,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    public_pem = public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    private_key_path.write_bytes(private_pem)
+    public_key_path.write_bytes(public_pem)
+    # Best-effort -- some filesystems (e.g. certain CI/tmpfs mounts) don't
+    # support chmod; the file still gets written correctly either way.
+    with contextlib.suppress(OSError):
+        private_key_path.chmod(0o600)
+
+
+def load_private_key(path: Path) -> Ed25519PrivateKey:
+    """Load an Ed25519 private key from a PEM file (unencrypted PKCS8 or
+    traditional format -- whatever `serialization.load_pem_private_key`
+    accepts)."""
+    key = serialization.load_pem_private_key(path.read_bytes(), password=None)
+    if not isinstance(key, Ed25519PrivateKey):
+        raise ReceiptError(f"{path} does not contain an Ed25519 private key")
+    return key
+
+
+def _public_key_from_text(text: str, *, source: str) -> Ed25519PublicKey:
+    stripped = text.strip()
+    if not stripped:
+        raise ReceiptError(f"{source} is empty")
+    if "BEGIN PUBLIC KEY" in stripped:
+        key = serialization.load_pem_public_key(stripped.encode("utf-8"))
+        if not isinstance(key, Ed25519PublicKey):
+            raise ReceiptError(f"{source} does not contain an Ed25519 public key")
+        return key
+    # Otherwise treat it as the base64 encoding of the raw 32-byte key --
+    # the same form receipts embed and `memtrust keygen` never writes to
+    # disk on its own, but is convenient for passing via an env var.
+    try:
+        raw = base64.b64decode(stripped, validate=True)
+    except ValueError as exc:
+        raise ReceiptError(f"{source} is neither a PEM public key nor valid base64: {exc}") from exc
+    try:
+        return Ed25519PublicKey.from_public_bytes(raw)
+    except ValueError as exc:
+        raise ReceiptError(f"{source} is not a valid Ed25519 public key: {exc}") from exc
+
+
+def load_public_key(path: Path) -> Ed25519PublicKey:
+    """Load an Ed25519 public key from a file -- PEM or raw-base64, either works."""
+    return _public_key_from_text(path.read_text(), source=str(path))
+
+
+def load_public_key_from_env(env_var: str = PUBLIC_KEY_ENV_VAR) -> Ed25519PublicKey:
+    value = os.environ.get(env_var)
+    if value is None:
+        raise ReceiptError(f"environment variable {env_var} is not set")
+    return _public_key_from_text(value, source=f"${env_var}")
+
+
+def resolve_public_key(
+    key_path: Path | None, *, env_var: str = PUBLIC_KEY_ENV_VAR
+) -> Ed25519PublicKey:
+    """Resolve the caller's trusted public key: an explicit --public-key
+    path always wins; otherwise fall back to the env var. Raises
+    ReceiptError if neither is available -- verification never silently
+    trusts the receipt's own embedded key as a substitute."""
+    if key_path is not None:
+        return load_public_key(key_path)
+    return load_public_key_from_env(env_var)
+
+
+def _public_key_b64(public_key: Ed25519PublicKey) -> str:
+    raw = public_key.public_bytes(
+        encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw
+    )
+    return base64.b64encode(raw).decode("ascii")
+
+
+def sign_report(report: dict[str, Any], private_key: Ed25519PrivateKey) -> dict[str, Any]:
+    """Build a signed receipt for `report`. Returns the receipt as a plain
+    dict, ready to be `json.dump`ed."""
+    canonical = canonicalize(report)
+    signature = private_key.sign(canonical)
+    return {
+        "memtrust_receipt_version": RECEIPT_FORMAT_VERSION,
+        "algorithm": ALGORITHM,
+        "signed_at": datetime.now(UTC).isoformat(),
+        "payload_sha256": hashlib.sha256(canonical).hexdigest(),
+        "public_key": _public_key_b64(private_key.public_key()),
+        "signature": base64.b64encode(signature).decode("ascii"),
+        "payload": report,
+    }
+
+
+def sign_report_with_keyfile(report: dict[str, Any], private_key_path: Path) -> dict[str, Any]:
+    return sign_report(report, load_private_key(private_key_path))
+
+
+@dataclass(frozen=True)
+class VerifyResult:
+    """Outcome of verifying a receipt.
+
+    `valid=True` proves exactly two things: (1) `payload` is byte-for-byte
+    identical, under canonicalization, to what was signed -- nothing in it
+    was altered after signing -- and (2) the signature was produced by the
+    holder of the private key matching the trusted public key the caller
+    supplied. It proves nothing about whether the benchmark numbers inside
+    `payload` are themselves correct, complete, or run against a real
+    backend -- that is a separate concern docs/methodology.md covers on
+    its own terms, not something a signature can attest to.
+    """
+
+    valid: bool
+    reason: str
+    embedded_key_matches_trusted_key: bool | None = None
+
+
+def verify_receipt(receipt: dict[str, Any], trusted_public_key: Ed25519PublicKey) -> VerifyResult:
+    """Verify `receipt` against `trusted_public_key` (supplied by the
+    caller -- never derived from the receipt itself; see module docstring).
+    """
+    try:
+        payload = receipt["payload"]
+        payload_sha256 = receipt["payload_sha256"]
+        signature_b64 = receipt["signature"]
+        embedded_public_key_b64 = receipt.get("public_key")
+    except KeyError as exc:
+        return VerifyResult(valid=False, reason=f"receipt is missing required field {exc}")
+
+    if receipt.get("algorithm", ALGORITHM) != ALGORITHM:
+        return VerifyResult(
+            valid=False,
+            reason=f"unsupported algorithm {receipt.get('algorithm')!r} (expected {ALGORITHM})",
+        )
+
+    canonical = canonicalize(payload)
+    recomputed_hash = hashlib.sha256(canonical).hexdigest()
+    if recomputed_hash != payload_sha256:
+        return VerifyResult(
+            valid=False,
+            reason=(
+                "payload does not match its own recorded payload_sha256 -- the "
+                "payload was edited after the receipt was written "
+                f"(recomputed {recomputed_hash}, receipt says {payload_sha256})"
+            ),
+        )
+
+    try:
+        signature = base64.b64decode(signature_b64, validate=True)
+    except ValueError as exc:
+        return VerifyResult(valid=False, reason=f"signature is not valid base64: {exc}")
+
+    embedded_key_matches: bool | None = None
+    if embedded_public_key_b64 is not None:
+        embedded_key_matches = embedded_public_key_b64 == _public_key_b64(trusted_public_key)
+
+    try:
+        trusted_public_key.verify(signature, canonical)
+    except InvalidSignature:
+        return VerifyResult(
+            valid=False,
+            reason=(
+                "signature does not verify against the supplied trusted public key "
+                "-- either the payload was tampered with, or it was signed by a "
+                "different key than the one supplied to `memtrust verify`"
+            ),
+            embedded_key_matches_trusted_key=embedded_key_matches,
+        )
+
+    return VerifyResult(
+        valid=True,
+        reason="signature verified: payload is unaltered and was signed by the supplied key",
+        embedded_key_matches_trusted_key=embedded_key_matches,
+    )
+
+
+def verify_receipt_file(
+    receipt_path: Path, *, public_key_path: Path | None, env_var: str = PUBLIC_KEY_ENV_VAR
+) -> VerifyResult:
+    trusted_key = resolve_public_key(public_key_path, env_var=env_var)
+    receipt = json.loads(receipt_path.read_text())
+    if not isinstance(receipt, dict):
+        return VerifyResult(valid=False, reason="receipt file does not contain a JSON object")
+    return verify_receipt(receipt, trusted_key)
+
+
+def receipt_path_for(report_path: Path) -> Path:
+    """The conventional receipt filename for a given report path:
+    `memtrust-report-2026-07-16.json` -> `memtrust-report-2026-07-16.receipt.json`.
+    """
+    return report_path.with_name(report_path.stem + ".receipt.json")

memtrust/scoring/__init__.py

@@ -0,0 +1 @@
+"""LLM-graded scoring pipeline and cost tracking."""

memtrust/scoring/cost_tracker.py

@@ -0,0 +1,199 @@
+"""Tracks estimated token/cost per run so `memtrust run` can print a cost
+summary. Pricing is an approximate, dated estimate -- not a billing
+guarantee. See PRICING_LAST_VERIFIED below and docs/methodology.md.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+
+#: Last time the per-1M-token prices below were checked against the
+#: provider's published pricing page. Prices change; treat this table as
+#: an estimate for cost-awareness, not an invoice.
+PRICING_LAST_VERIFIED = "2026-07-11"
+
+#: USD per 1,000,000 tokens, (input, output). Approximate, from each
+#: provider's public pricing page as of PRICING_LAST_VERIFIED.
+MODEL_PRICING_PER_MILLION_TOKENS: dict[str, tuple[float, float]] = {
+    "deepseek-chat": (0.28, 0.42),
+    "deepseek-reasoner": (0.56, 1.68),
+    "gemini-2.5-flash-lite": (0.10, 0.40),
+    "gemini-2.0-flash-lite": (0.075, 0.30),
+}
+
+DEFAULT_UNKNOWN_MODEL_PRICE = (0.50, 1.50)
+
+#: Approximate USD per 1,000,000 embedded input characters, keyed by
+#: embedder provider name (matching `Mem0DirectAdapter.SUPPORTED_EMBEDDER_
+#: PROVIDERS`'s naming) -- NOT per-token. Vendor embedder-call
+#: instrumentation (see mem0_direct_adapter.py's `_CountingEmbedder`) only
+#: has each `embed()`/`embed_batch()` call's raw input text available, not
+#: the vendor's own tokenizer's real token count, so a char/4 heuristic is
+#: applied at the call site before this table's price is used -- the same
+#: "estimate, not an invoice" caveat `MODEL_PRICING_PER_MILLION_TOKENS`
+#: above already carries, extended here to vendor-side embedding spend
+#: rather than memtrust's own judge-LLM spend. Approximate, from each
+#: provider's public embedding-pricing page as of PRICING_LAST_VERIFIED.
+#: `fastembed` runs a local ONNX model with no per-call vendor API cost at
+#: all, hence $0 -- not an "unknown, guess" $0, a genuine $0.
+EMBEDDER_PRICING_PER_MILLION_TOKENS: dict[str, float] = {
+    "openai": 0.02,  # text-embedding-3-small, OpenAI's cheapest/default embedding model
+    "aws_bedrock": 0.02,  # Amazon Titan Text Embeddings V2
+    "gemini": 0.00,  # no public per-token embedding price line-item as of PRICING_LAST_VERIFIED
+    "fastembed": 0.00,  # local ONNX model, genuinely no vendor API call
+}
+
+DEFAULT_UNKNOWN_EMBEDDER_PRICE = 0.10
+
+
+@dataclass
+class CostEntry:
+    label: str
+    model: str
+    input_tokens: int
+    output_tokens: int
+    cost_usd: float
+
+
+@dataclass
+class EmbedCallEntry:
+    """Vendor-side embedder-call/cost attribution -- distinct from
+    `CostEntry` above, which tracks only memtrust's own judge-LLM spend.
+    See `CostTracker.record_embed_calls()` and
+    mem0_direct_adapter.py's `_CountingEmbedder` for where this gets
+    populated. Motivating case: spike-spiegel-21 (rank 104,
+    mem0ai/mem0#1900) landed a merged fix eliminating a duplicate
+    embedding-API call in an earlier mem0 architecture when content was
+    unchanged after search -- memtrust had zero vendor-side embedder-call
+    instrumentation anywhere to observe whether that class of waste is
+    still happening in the currently installed mem0ai version's own
+    pipeline. See evals/embedder_cost.py.
+    """
+
+    label: str
+    provider: str
+    call_count: int
+    estimated_tokens: int
+    cost_usd: float
+
+
+@dataclass
+class CostTracker:
+    """Accumulates cost entries across a `memtrust run` invocation.
+
+    memtrust's eval runners are sequential today, so ``_lock`` guards a
+    mutation that is not actually contended yet. It's here preemptively:
+    a real, merged vendor bug (volcengine/OpenViking PR #3091) shipped a
+    benchmark tool that silently ignored its own --concurrency flag,
+    quietly invalidating its self-reported numbers. Keeping this shared
+    list append synchronized now means that if/when memtrust adds
+    concurrent eval execution, cost accounting can't silently drop or
+    corrupt entries the way an unsynchronized append would. The same
+    `_lock` guards `embed_entries` below.
+    """
+
+    entries: list[CostEntry] = field(default_factory=list)
+    embed_entries: list[EmbedCallEntry] = field(default_factory=list)
+    """Vendor-side embedder-call entries -- kept as a separate list from
+    `entries` above (rather than folded into the same list/shape) because
+    the two track fundamentally different spend categories: `entries` is
+    memtrust's own judge-LLM cost (real input/output token counts from a
+    real LLM API response), `embed_entries` is vendor embedder-API call
+    count/cost attribution derived from a char-length heuristic, never a
+    real token count the vendor itself reported. Conflating the two into
+    one list/one set of properties would misrepresent the second
+    category's evidentiary strength as equal to the first's."""
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False, compare=False)
+
+    def record(self, label: str, model: str, input_tokens: int, output_tokens: int) -> CostEntry:
+        price_in, price_out = MODEL_PRICING_PER_MILLION_TOKENS.get(
+            model, DEFAULT_UNKNOWN_MODEL_PRICE
+        )
+        cost = (input_tokens / 1_000_000) * price_in + (output_tokens / 1_000_000) * price_out
+        entry = CostEntry(
+            label=label,
+            model=model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            cost_usd=round(cost, 6),
+        )
+        with self._lock:
+            self.entries.append(entry)
+        return entry
+
+    def record_embed_calls(
+        self, label: str, provider: str, call_count: int, estimated_tokens: int
+    ) -> EmbedCallEntry:
+        """Record vendor-side embedder-call/cost attribution for a single
+        adapter operation (typically one `Mem0DirectAdapter.store()` call)
+        -- distinct from `record()` above, which is memtrust's own
+        judge-LLM spend only. See `EmbedCallEntry`'s docstring for why
+        this is a separate list/method rather than reusing `record()`'s
+        shape.
+        """
+        price = EMBEDDER_PRICING_PER_MILLION_TOKENS.get(provider, DEFAULT_UNKNOWN_EMBEDDER_PRICE)
+        cost = (estimated_tokens / 1_000_000) * price
+        entry = EmbedCallEntry(
+            label=label,
+            provider=provider,
+            call_count=call_count,
+            estimated_tokens=estimated_tokens,
+            cost_usd=round(cost, 6),
+        )
+        with self._lock:
+            self.embed_entries.append(entry)
+        return entry
+
+    @property
+    def total_cost_usd(self) -> float:
+        return round(sum(e.cost_usd for e in self.entries), 6)
+
+    @property
+    def total_input_tokens(self) -> int:
+        return sum(e.input_tokens for e in self.entries)
+
+    @property
+    def total_output_tokens(self) -> int:
+        return sum(e.output_tokens for e in self.entries)
+
+    @property
+    def total_embed_calls(self) -> int:
+        return sum(e.call_count for e in self.embed_entries)
+
+    @property
+    def total_embed_cost_usd(self) -> float:
+        return round(sum(e.cost_usd for e in self.embed_entries), 6)
+
+    def summary_lines(self) -> list[str]:
+        if not self.entries and not self.embed_entries:
+            return [
+                "Cost: $0.00 (no LLM-judged evals ran -- structural evals only, "
+                "or judge not configured)"
+            ]
+        lines: list[str] = []
+        if self.entries:
+            lines.append(
+                f"Cost: ${self.total_cost_usd:.4f} estimated "
+                f"({self.total_input_tokens:,} in / {self.total_output_tokens:,} out tokens, "
+                f"pricing last verified {PRICING_LAST_VERIFIED})"
+            )
+            by_model: dict[str, float] = {}
+            for e in self.entries:
+                by_model[e.model] = by_model.get(e.model, 0.0) + e.cost_usd
+            for model, cost in sorted(by_model.items()):
+                lines.append(f"  {model}: ${cost:.4f}")
+        if self.embed_entries:
+            lines.append(
+                f"Vendor embedder calls: {self.total_embed_calls:,} "
+                f"(~${self.total_embed_cost_usd:.4f} estimated, pricing last verified "
+                f"{PRICING_LAST_VERIFIED})"
+            )
+            by_provider: dict[str, int] = {}
+            for embed_entry in self.embed_entries:
+                by_provider[embed_entry.provider] = (
+                    by_provider.get(embed_entry.provider, 0) + embed_entry.call_count
+                )
+            for provider, calls in sorted(by_provider.items()):
+                lines.append(f"  {provider}: {calls:,} calls")
+        return lines

memtrust/scoring/llm_judge.py

@@ -0,0 +1,161 @@
+"""LLM-graded scoring for eval outputs that need semantic judgment (does
+this recalled answer actually match the expected fact, allowing for
+paraphrase) rather than exact string match.
+
+Model-configurable via environment variables:
+  MEMTRUST_JUDGE_API_KEY   -- required to actually run a judged eval
+  MEMTRUST_JUDGE_MODEL     -- default: deepseek-chat
+  MEMTRUST_JUDGE_BASE_URL  -- default: https://api.deepseek.com
+                              (OpenAI-compatible chat-completions shape;
+                              point this at any OpenAI-compatible endpoint,
+                              including a Gemini OpenAI-compat proxy, to
+                              use a different judge model)
+
+If no API key is configured, judge_answer() returns a JudgeResult with
+verdict=NOT_RUN and an explicit reason -- it never fabricates a score.
+Every eval runner that calls this module must propagate NOT_RUN as "could
+not be graded," not as a failing score, so an unconfigured judge cannot be
+mistaken for a backend that failed the eval.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from enum import StrEnum
+
+import httpx
+
+from memtrust.scoring.cost_tracker import CostTracker
+
+DEFAULT_MODEL = "deepseek-chat"
+DEFAULT_BASE_URL = "https://api.deepseek.com"
+
+#: Exact judge prompt template, published here and mirrored in
+#: docs/methodology.md so every scored run is reproducible from the repo
+#: alone -- no prompt lives only in a maintainer's head.
+JUDGE_PROMPT_TEMPLATE = """You are grading whether a memory system's recalled answer is factually \
+equivalent to the expected answer. Ignore differences in phrasing, tense, or extra \
+detail -- grade only whether the core fact matches.
+
+Question asked: {question}
+Expected answer: {expected}
+System's actual answer: {actual}
+
+Respond with exactly one word on the first line: CORRECT, INCORRECT, or PARTIAL.
+On the second line, give a one-sentence reason.
+"""
+
+
+class JudgeVerdict(StrEnum):
+    CORRECT = "correct"
+    INCORRECT = "incorrect"
+    PARTIAL = "partial"
+    NOT_RUN = "not_run"
+    """The judge could not run -- no API key configured, or the call
+    failed. Never averaged into a score; callers must report this
+    verdict's cases as "could not be graded," separate from the scored
+    cases."""
+
+
+@dataclass
+class JudgeResult:
+    verdict: JudgeVerdict
+    reasoning: str
+    model: str | None = None
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+class LLMJudge:
+    """Thin, provider-agnostic client for the configured judge model.
+
+    Constructed once per run and passed a shared CostTracker so every
+    judged eval call's estimated cost rolls up into one printed total.
+    """
+
+    def __init__(self, cost_tracker: CostTracker | None = None, timeout: float = 30.0) -> None:
+        self.api_key = os.environ.get("MEMTRUST_JUDGE_API_KEY")
+        self.model = os.environ.get("MEMTRUST_JUDGE_MODEL", DEFAULT_MODEL)
+        self.base_url = os.environ.get("MEMTRUST_JUDGE_BASE_URL", DEFAULT_BASE_URL)
+        self.cost_tracker = cost_tracker if cost_tracker is not None else CostTracker()
+        self._timeout = timeout
+        self._http: httpx.Client | None = None
+
+    @property
+    def is_configured(self) -> bool:
+        return bool(self.api_key)
+
+    def _client(self) -> httpx.Client:
+        if self._http is None:
+            self._http = httpx.Client(
+                base_url=self.base_url,
+                headers={
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                },
+                timeout=self._timeout,
+            )
+        return self._http
+
+    def judge_answer(self, question: str, expected: str, actual: str) -> JudgeResult:
+        if not self.is_configured:
+            return JudgeResult(
+                verdict=JudgeVerdict.NOT_RUN,
+                reasoning=(
+                    "No judge API key configured (set MEMTRUST_JUDGE_API_KEY). "
+                    "This eval could not be graded -- reporting as not-run, "
+                    "not as a failing score."
+                ),
+            )
+
+        prompt = JUDGE_PROMPT_TEMPLATE.format(question=question, expected=expected, actual=actual)
+        payload = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": 0.0,
+            "max_tokens": 100,
+        }
+        try:
+            resp = self._client().post("/chat/completions", json=payload)
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPError as exc:
+            return JudgeResult(
+                verdict=JudgeVerdict.NOT_RUN,
+                reasoning=(
+                    f"Judge API call failed: {exc}. Reporting as not-run, not a failing score."
+                ),
+                model=self.model,
+            )
+
+        text = data["choices"][0]["message"]["content"].strip()
+        usage = data.get("usage", {})
+        input_tokens = int(usage.get("prompt_tokens", 0))
+        output_tokens = int(usage.get("completion_tokens", 0))
+        self.cost_tracker.record(
+            label=f"judge:{question[:40]}",
+            model=self.model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+        )
+
+        first_line = text.splitlines()[0].strip().upper() if text else ""
+        reasoning = text.splitlines()[1].strip() if len(text.splitlines()) > 1 else text
+        if "CORRECT" in first_line and "INCORRECT" not in first_line:
+            verdict = JudgeVerdict.CORRECT
+        elif "PARTIAL" in first_line:
+            verdict = JudgeVerdict.PARTIAL
+        else:
+            verdict = JudgeVerdict.INCORRECT
+        return JudgeResult(
+            verdict=verdict,
+            reasoning=reasoning,
+            model=self.model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+        )
+
+    def close(self) -> None:
+        if self._http is not None:
+            self._http.close()