cacheverifier 0.1.0__tar.gz → 0.3.0__tar.gz

cacheverifier-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e ".[dev]"
+      - run: ruff check .
+      - run: pytest -q
+      - run: mypy cacheverifier
+        if: matrix.python-version != '3.9'  # mypy config targets 3.10+
+
+  # `cacheverifier healthcheck` -- the base `test` job above installs only
+  # `.[dev]`, so it exercises the CLI dispatch and the missing-extra path;
+  # this job installs `.[healthcheck,dev]` and runs the real fine-tune
+  # end-to-end. One Python version is enough (the training path isn't
+  # version-sensitive) and CPU-only torch keeps the install lean -- the
+  # default wheel drags in ~3 GB of unused CUDA libraries whose import-time
+  # mmap alone can OOM a small runner.
+  healthcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: python -m pip install --upgrade pip
+      - run: pip install torch --index-url https://download.pytorch.org/whl/cpu
+      - run: pip install -e ".[healthcheck,dev]"
+      - run: ruff check .
+      - run: mypy cacheverifier
+      - run: pytest -q tests/test_healthcheck.py
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build twine
+      - run: python -m build
+      - run: twine check dist/*

cacheverifier-0.3.0/CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog
+
+## 0.3.0
+
+Make the request-path defaults safe. `verify()` and `verify_batch()` now use a
+separate **1s timeout** (`verify_timeout=`, vs the unchanged 10s `timeout=` for
+control-plane calls), and on a timeout / connection error / 5xx they **fail
+closed** instead of raising: you get a synthetic `VerifyResult` with
+`degraded=True` and `approved=False` (fall through to your LLM). Pass
+`fail_open=True` to have those cases return `approved=True` instead. 4xx
+responses (bad key, bad request, rate limit) still raise `CacheVerifierError`.
+`VerifyResult` gains a `degraded` field; `model_version` is
+`"verify_unavailable"` on a synthesized result. The GPTCache adapter takes the
+same `verify_timeout` / `fail_open` arguments and returns `0.0` (don't reuse) on
+a verifier outage by default.
+
+## 0.2.0
+
+Add a `cacheverifier` console script with a `healthcheck` subcommand:
+`cacheverifier healthcheck traffic.jsonl` runs the hosted Health Check
+Report's stock-vs-fine-tuned held-out AUC evaluation (`POST /v1/finetune/dry-run`)
+entirely offline — no queries or answers leave the machine. Needs the new
+`healthcheck` extra (`pip install "cacheverifier[healthcheck]"`: torch,
+sentence-transformers, numpy); the base client stays `httpx`-only.
+`--emit-summary` writes an aggregate-only JSON file with no text.
+
+## 0.1.0
+
+Initial release: thin `httpx`-based client for `/v1/verify`, `/v1/verify/batch`,
+`/v1/feedback`(`/batch`), fine-tune jobs + dry run, drift and gray-zone-threshold
+monitoring, and usage/savings. GPTCache `SimilarityEvaluation` adapter under
+`cacheverifier.integrations.gptcache`.

{cacheverifier-0.1.0 → cacheverifier-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cacheverifier
-Version: 0.1.0
+Version: 0.3.0
 Summary: Python client for the hosted CacheVerifier semantic-cache verification API
 Project-URL: Homepage, https://www.cacheverifier.com
 Project-URL: Documentation, https://www.cacheverifier.com/docs
@@ -24,6 +24,10 @@ Requires-Dist: pytest>=7; extra == 'dev'
 Requires-Dist: ruff>=0.4; extra == 'dev'
 Provides-Extra: gptcache
 Requires-Dist: gptcache>=0.1.30; extra == 'gptcache'
+Provides-Extra: healthcheck
+Requires-Dist: numpy<2,>=1.26; extra == 'healthcheck'
+Requires-Dist: sentence-transformers<4,>=3.0; extra == 'healthcheck'
+Requires-Dist: torch>=2.2; extra == 'healthcheck'
 Description-Content-Type: text/markdown
 
 # cacheverifier
@@ -51,9 +55,12 @@ zone", where a plain threshold match might be wrong.
 pip install cacheverifier
 # with the GPTCache adapter:
 pip install "cacheverifier[gptcache]"
+# with the offline Health Check (adds torch + sentence-transformers):
+pip install "cacheverifier[healthcheck]"
 ```
 
-Requires Python 3.9+. The only runtime dependency is `httpx`.
+Requires Python 3.9+. The only runtime dependency is `httpx` — the extras above
+are opt-in.
 
 ## Quickstart
 
@@ -84,8 +91,34 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86)
 |---|---|
 | `approved` | serve the cached answer (`True`) or fall through (`False`) |
 | `score` / `threshold` | `approved` is `score >= threshold` |
-| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), or `"cold_start_fail_closed"` |
+| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` |
 | `latency_ms` | server-side inference time |
+| `degraded` | `True` when the call failed and this result was synthesized client-side |
+
+## On your request path
+
+`verify()` runs inline with your traffic, so its defaults are conservative:
+
+- **1s timeout** (`verify_timeout=`), separate from the 10s `timeout=` used for
+  fine-tuning / feedback / monitoring calls. Warm verification is tens of
+  milliseconds server-side; 1s covers the network round trip and a cold model
+  load after a deploy without letting a stuck verifier stall your request.
+- **Fails closed.** On a timeout, connection error, or 5xx, `verify()` does not
+  raise — it logs a warning on the `cacheverifier` logger and returns a
+  `VerifyResult` with `degraded=True` and `approved=False`, so you fall through
+  to your LLM exactly as you would on a cache miss. A `4xx` (bad key, bad
+  request, rate limit) still raises `CacheVerifierError`.
+
+```python
+cv = CacheVerifier(api_key="cv_...", verify_timeout=1.0, fail_open=False)  # the defaults
+
+# fail_open=True instead returns approved=True on an outage — only if a
+# stale-or-near-miss answer is acceptable for that traffic:
+cv = CacheVerifier(api_key="cv_...", fail_open=True)
+```
+
+There is no formal uptime SLA yet, which is the other reason the fallback path
+is a built-in default rather than left to you.
 
 ## GPTCache
 
@@ -123,6 +156,41 @@ if job.get("result_model_version"):
 `cv.dry_run([...])` reports the same baseline-vs-tuned AUC on examples you pass directly,
 without writing anything or deploying a model.
 
+## Local Health Check (offline)
+
+`cv.dry_run()` still uploads your examples to the API. If that's a blocker — a
+compliance review, or just not wanting production traffic to leave your network —
+run the identical stock-vs-fine-tuned evaluation entirely on your own machine:
+
+```bash
+pip install "cacheverifier[healthcheck]"
+
+cacheverifier healthcheck traffic.jsonl
+cacheverifier healthcheck traffic.jsonl --emit-summary summary.json
+```
+
+`traffic.jsonl` is a JSON array or JSONL of `{"query", "candidate_answer", "was_correct"}`
+rows **in arrival order** (the train/calibrate/test split is chronological, matching the
+hosted service so the numbers are comparable). Optional per row: `"stale": true`.
+
+Nothing is sent anywhere — the base model downloads once from Hugging Face, then it's
+fully offline. `--emit-summary` writes an aggregate-only JSON file (AUCs, counts, rates —
+no query or answer text) that's safe to share for a human read.
+
+```
+results
+------------------------------------------------------------------
+  train / calibrate / test:         3349 / 419 / 419
+  stock verifier   held-out AUC:    0.6120
+  fine-tuned       held-out AUC:    0.7080   (delta +0.0960)
+  label-noise proxy (disagreement): 11.4%
+  ceiling status:                   still_improvable
+
+verdict
+------------------------------------------------------------------
+  IMPROVED   -- fine-tuning on your own data helps this traffic
+```
+
 ## API surface
 
 | method | endpoint |

{cacheverifier-0.1.0 → cacheverifier-0.3.0}/README.md

@@ -23,9 +23,12 @@ zone", where a plain threshold match might be wrong.
 pip install cacheverifier
 # with the GPTCache adapter:
 pip install "cacheverifier[gptcache]"
+# with the offline Health Check (adds torch + sentence-transformers):
+pip install "cacheverifier[healthcheck]"
 ```
 
-Requires Python 3.9+. The only runtime dependency is `httpx`.
+Requires Python 3.9+. The only runtime dependency is `httpx` — the extras above
+are opt-in.
 
 ## Quickstart
 
@@ -56,8 +59,34 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86)
 |---|---|
 | `approved` | serve the cached answer (`True`) or fall through (`False`) |
 | `score` / `threshold` | `approved` is `score >= threshold` |
-| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), or `"cold_start_fail_closed"` |
+| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` |
 | `latency_ms` | server-side inference time |
+| `degraded` | `True` when the call failed and this result was synthesized client-side |
+
+## On your request path
+
+`verify()` runs inline with your traffic, so its defaults are conservative:
+
+- **1s timeout** (`verify_timeout=`), separate from the 10s `timeout=` used for
+  fine-tuning / feedback / monitoring calls. Warm verification is tens of
+  milliseconds server-side; 1s covers the network round trip and a cold model
+  load after a deploy without letting a stuck verifier stall your request.
+- **Fails closed.** On a timeout, connection error, or 5xx, `verify()` does not
+  raise — it logs a warning on the `cacheverifier` logger and returns a
+  `VerifyResult` with `degraded=True` and `approved=False`, so you fall through
+  to your LLM exactly as you would on a cache miss. A `4xx` (bad key, bad
+  request, rate limit) still raises `CacheVerifierError`.
+
+```python
+cv = CacheVerifier(api_key="cv_...", verify_timeout=1.0, fail_open=False)  # the defaults
+
+# fail_open=True instead returns approved=True on an outage — only if a
+# stale-or-near-miss answer is acceptable for that traffic:
+cv = CacheVerifier(api_key="cv_...", fail_open=True)
+```
+
+There is no formal uptime SLA yet, which is the other reason the fallback path
+is a built-in default rather than left to you.
 
 ## GPTCache
 
@@ -95,6 +124,41 @@ if job.get("result_model_version"):
 `cv.dry_run([...])` reports the same baseline-vs-tuned AUC on examples you pass directly,
 without writing anything or deploying a model.
 
+## Local Health Check (offline)
+
+`cv.dry_run()` still uploads your examples to the API. If that's a blocker — a
+compliance review, or just not wanting production traffic to leave your network —
+run the identical stock-vs-fine-tuned evaluation entirely on your own machine:
+
+```bash
+pip install "cacheverifier[healthcheck]"
+
+cacheverifier healthcheck traffic.jsonl
+cacheverifier healthcheck traffic.jsonl --emit-summary summary.json
+```
+
+`traffic.jsonl` is a JSON array or JSONL of `{"query", "candidate_answer", "was_correct"}`
+rows **in arrival order** (the train/calibrate/test split is chronological, matching the
+hosted service so the numbers are comparable). Optional per row: `"stale": true`.
+
+Nothing is sent anywhere — the base model downloads once from Hugging Face, then it's
+fully offline. `--emit-summary` writes an aggregate-only JSON file (AUCs, counts, rates —
+no query or answer text) that's safe to share for a human read.
+
+```
+results
+------------------------------------------------------------------
+  train / calibrate / test:         3349 / 419 / 419
+  stock verifier   held-out AUC:    0.6120
+  fine-tuned       held-out AUC:    0.7080   (delta +0.0960)
+  label-noise proxy (disagreement): 11.4%
+  ceiling status:                   still_improvable
+
+verdict
+------------------------------------------------------------------
+  IMPROVED   -- fine-tuning on your own data helps this traffic
+```
+
 ## API surface
 
 | method | endpoint |

{cacheverifier-0.1.0 → cacheverifier-0.3.0}/cacheverifier/__init__.py

@@ -18,5 +18,5 @@ Research behind it: https://github.com/imxinchengyou/CacheVerifier
 
 from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"
 __all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"]

cacheverifier-0.3.0/cacheverifier/__main__.py

@@ -0,0 +1,44 @@
+"""`python -m cacheverifier` / the `cacheverifier` console script.
+
+A thin argparse dispatcher. The base install (httpx only) provides
+`--version` and `--help`; the `healthcheck` subcommand additionally needs
+the `healthcheck` extra and imports nothing heavy until it runs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from cacheverifier import __version__
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="cacheverifier",
+        description="CacheVerifier -- hosted semantic-cache verification (https://www.cacheverifier.com).",
+    )
+    parser.add_argument("--version", action="version", version=f"cacheverifier {__version__}")
+    subparsers = parser.add_subparsers(dest="command", metavar="<command>")
+
+    # Import lazily and defensively: a missing healthcheck extra must not
+    # break `cacheverifier --help` or `--version`. cli.add_subparser itself
+    # only touches argparse.
+    from cacheverifier._healthcheck import cli as healthcheck_cli
+
+    healthcheck_cli.add_subparser(subparsers)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "command", None):
+        parser.print_help()
+        return 1
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cacheverifier-0.3.0/cacheverifier/_healthcheck/__init__.py

@@ -0,0 +1,7 @@
+"""`cacheverifier healthcheck` -- run the hosted Health Check Report's
+stock-vs-fine-tuned AUC evaluation entirely on your own machine.
+
+Everything under here needs the `healthcheck` extra
+(`pip install "cacheverifier[healthcheck]"`: torch, sentence-transformers,
+scikit-learn, numpy) and is imported only when the subcommand runs.
+"""

cacheverifier-0.3.0/cacheverifier/_healthcheck/_finetune.py

@@ -0,0 +1,274 @@
+"""Local fine-tune + held-out AUC evaluation for `cacheverifier healthcheck`.
+
+This is a trimmed, self-contained port of the hosted service's
+`verifier_core.finetune` / `verifier_core.cross_encoder` (the code behind
+`POST /v1/finetune/dry-run`). It keeps the parts a Health Check needs --
+the chronological train/calibrate/test split, held-out AUC for the stock
+vs. fine-tuned model, a Youden's-J operating point, and the label-noise
+and ceiling diagnostics -- and drops everything specific to running a
+production tenant (Conformal Risk Control certification, cost-ratio
+threshold grids, drift monitoring, operating-point-jump detection).
+
+Nothing here is imported unless the `healthcheck` extra is installed and
+the subcommand is actually run -- see `cacheverifier.__main__`.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Any, cast
+
+import numpy as np
+import torch
+from sentence_transformers import CrossEncoder, InputExample
+from torch.utils.data import DataLoader
+
+# All four imports above are from the `healthcheck` extra. This module is
+# only ever imported from inside `cacheverifier._healthcheck.cli.run()`,
+# which catches the ImportError and points the user at
+# `pip install "cacheverifier[healthcheck]"` -- so a plain
+# `import cacheverifier` / `cacheverifier --help` never pays this cost.
+
+DEFAULT_BASE_MODEL = "cross-encoder/ms-marco-MiniLM-L6-v2"
+"""The stock off-the-shelf verifier every tenant starts on -- the same base
+model the hosted service fine-tunes from."""
+
+MAX_SEQUENCE_LENGTH = 128
+"""Token cap applied identically at train and score time. Matches the
+hosted service; BERT-style attention memory is O(n^2) in sequence length,
+and this keeps a CPU run well-behaved."""
+
+_CONTENT_TOKEN_BUDGET = MAX_SEQUENCE_LENGTH - 3  # [CLS] + 2x [SEP]
+_MIN_QUERY_TOKENS = 32
+
+TRAIN_BATCH_SIZE = 4
+MIN_TRAIN_EXAMPLES = 20
+"""Hard floor -- below this there aren't enough rows for one meaningful
+epoch plus a held-out split."""
+
+COLD_START_WARNING_THRESHOLD = 1000
+"""The hosted service's research (Research page, Q2) found fine-tuning only
+reliably positive across every dataset above roughly this many rows. Below
+it a result is real but noisy."""
+
+_MIN_TEST_FOR_THRESHOLD = 10
+_MIN_TEST_FOR_CEILING_DIAGNOSIS = 50
+_CEILING_TAIL_MARGIN = 0.15
+_CEILING_AUC_CAP = 0.75
+
+NOISE_WARNING_DISAGREEMENT_RATE = 0.20
+"""`train_label_disagreement_rate` above this: worth auditing how the
+correctness signal is produced."""
+NOISE_HOLD_BACK_DISAGREEMENT_RATE = 0.30
+"""Above this the hosted service holds a new model back for review."""
+
+
+@dataclass(frozen=True)
+class GrayZoneExample:
+    query: str
+    candidate_answer: str
+    was_correct: bool
+
+
+@dataclass(frozen=True)
+class HealthCheckResult:
+    n_train: int
+    n_calibrate: int
+    n_test: int
+    train_positive_rate: float
+    test_positive_rate: float
+    auc_baseline: float
+    auc_tuned: float
+    train_time_seconds: float
+    threshold: float | None
+    threshold_hit_rate: float | None
+    threshold_error_rate: float | None
+    train_label_disagreement_rate: float | None
+    ceiling_status: str | None
+    model_path: str
+
+    @property
+    def auc_delta(self) -> float:
+        return self.auc_tuned - self.auc_baseline
+
+
+def _head_tail_truncate(ids: list[int], budget: int, head_ratio: float = 0.6) -> list[int]:
+    if len(ids) <= budget:
+        return ids
+    head_n = int(budget * head_ratio)
+    tail_n = budget - head_n
+    return ids[:head_n] + (ids[-tail_n:] if tail_n > 0 else [])
+
+
+def smart_truncate_pair(query: str, candidate_answer: str, tokenizer: Any) -> tuple[str, str]:
+    """Tokenizer-aware head+tail truncation, applied identically at train
+    and score time (a model trained on one truncation policy and scored
+    with another sees a different token window than it was optimized
+    against). A no-op whenever the pair already fits."""
+    query_ids = tokenizer.encode(query, add_special_tokens=False)
+    answer_ids = tokenizer.encode(candidate_answer, add_special_tokens=False)
+    if len(query_ids) + len(answer_ids) <= _CONTENT_TOKEN_BUDGET:
+        return query, candidate_answer
+
+    query_budget = min(len(query_ids), max(_MIN_QUERY_TOKENS, _CONTENT_TOKEN_BUDGET // 2))
+    answer_budget = _CONTENT_TOKEN_BUDGET - query_budget
+    query_ids = _head_tail_truncate(query_ids, query_budget)
+    answer_ids = _head_tail_truncate(answer_ids, answer_budget)
+    return (
+        tokenizer.decode(query_ids, skip_special_tokens=True),
+        tokenizer.decode(answer_ids, skip_special_tokens=True),
+    )
+
+
+def roc_auc(scores: np.ndarray, labels: np.ndarray) -> float:
+    """Rank-based AUC, no sklearn dependency. NaN if either class is absent."""
+    order = np.argsort(scores)
+    ranks = np.empty_like(order, dtype=float)
+    ranks[order] = np.arange(len(scores))
+    pos_ranks = ranks[labels == 1]
+    n_pos, n_neg = int((labels == 1).sum()), int((labels == 0).sum())
+    if n_pos == 0 or n_neg == 0:
+        return float("nan")
+    return float((pos_ranks.sum() - n_pos * (n_pos - 1) / 2) / (n_pos * n_neg))
+
+
+def select_threshold(scores: np.ndarray, labels: np.ndarray) -> float | None:
+    """Youden's J: the threshold maximizing (TPR - FPR). None when the
+    held-out split is too small or single-class to calibrate from -- the
+    caller decides the fallback."""
+    n_pos = int((labels == 1).sum())
+    n_neg = int((labels == 0).sum())
+    if n_pos == 0 or n_neg == 0 or (n_pos + n_neg) < _MIN_TEST_FOR_THRESHOLD:
+        return None
+
+    order = np.argsort(-scores)
+    sorted_scores = scores[order]
+    sorted_labels = labels[order]
+    tp = np.cumsum(sorted_labels == 1)
+    fp = np.cumsum(sorted_labels == 0)
+    youden_j = tp / n_pos - fp / n_neg
+    return float(sorted_scores[int(np.argmax(youden_j))])
+
+
+def _hit_and_error_rate(scores: np.ndarray, labels: np.ndarray, threshold: float) -> tuple[float, float | None]:
+    approved = scores >= threshold
+    n_approved = int(approved.sum())
+    hit_rate = n_approved / len(scores) if len(scores) else 0.0
+    if n_approved == 0:
+        return hit_rate, None
+    return hit_rate, float(np.mean(labels[approved] == 0))
+
+
+def _diagnose_ceiling(scores: np.ndarray, labels: np.ndarray) -> str | None:
+    """"insufficient_data" | "possible_ceiling" | "still_improvable" -- is
+    the fine-tuned verifier plausibly undertrained (more data would help)
+    or capped by the domain's own ambiguity? Precision in the top score
+    decile vs. the base rate, same shape of check as the hosted service."""
+    n = len(labels)
+    if n < _MIN_TEST_FOR_CEILING_DIAGNOSIS:
+        return "insufficient_data"
+    base_rate = float(labels.mean())
+    top_decile_n = max(1, n // 10)
+    top_decile_idx = np.argsort(-scores)[:top_decile_n]
+    precision_at_top_decile = float(labels[top_decile_idx].mean())
+    auc = roc_auc(scores, labels)
+    if precision_at_top_decile - base_rate >= _CEILING_TAIL_MARGIN and auc < _CEILING_AUC_CAP:
+        return "possible_ceiling"
+    return "still_improvable"
+
+
+def run_healthcheck(
+    examples: list[GrayZoneExample],
+    output_dir: str,
+    *,
+    base_model: str = DEFAULT_BASE_MODEL,
+    epochs: int = 1,
+) -> HealthCheckResult:
+    """Fine-tune `base_model` on a chronological prefix of `examples` and
+    measure held-out AUC for the stock vs. fine-tuned model.
+
+    `examples` must be in arrival (stream) order -- the train/calibrate/test
+    split is positional, not shuffled, matching the hosted service so the
+    numbers are comparable.
+    """
+    if len(examples) < MIN_TRAIN_EXAMPLES:
+        raise ValueError(f"need at least {MIN_TRAIN_EXAMPLES} examples, got {len(examples)}")
+
+    split = max(1, int(len(examples) * 0.8))
+    train_rows, holdout_rows = examples[:split], examples[split:]
+    if not holdout_rows:
+        train_rows, holdout_rows = examples[:-1], examples[-1:]
+    calib_split = max(1, len(holdout_rows) // 2)
+    calib_rows, test_rows = holdout_rows[:calib_split], holdout_rows[calib_split:]
+    if not test_rows:
+        calib_rows, test_rows = holdout_rows[:-1], holdout_rows[-1:]
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    test_pairs = [(e.query, e.candidate_answer) for e in test_rows]
+    test_labels = np.array([1 if e.was_correct else 0 for e in test_rows])
+    calib_pairs = [(e.query, e.candidate_answer) for e in calib_rows]
+    calib_labels = np.array([1 if e.was_correct else 0 for e in calib_rows])
+    train_positive_rate = float(np.mean([1 if e.was_correct else 0 for e in train_rows]))
+    test_positive_rate = float(test_labels.mean()) if len(test_labels) else 0.0
+
+    baseline = CrossEncoder(base_model, device=device, max_length=MAX_SEQUENCE_LENGTH)
+    test_pairs_scored = [smart_truncate_pair(q, a, baseline.tokenizer) for q, a in test_pairs]
+    baseline_scores = np.array(baseline.predict(test_pairs_scored, batch_size=32, show_progress_bar=False))
+    auc_baseline = roc_auc(baseline_scores, test_labels)
+    del baseline
+
+    tuned = CrossEncoder(base_model, device=device, max_length=MAX_SEQUENCE_LENGTH)
+    train_examples = [
+        InputExample(
+            texts=list(smart_truncate_pair(e.query, e.candidate_answer, tuned.tokenizer)),
+            label=1.0 if e.was_correct else 0.0,
+        )
+        for e in train_rows
+    ]
+    # A list[InputExample] is a valid PyTorch map-style dataset at runtime;
+    # sentence-transformers' pre-4.0 training API is built around exactly
+    # this, but the two libraries' stubs don't compose (InputExample isn't a
+    # typed Dataset). cast rather than `# type: ignore` so this stays clean
+    # whether or not torch's stubs are installed (base CI has no torch).
+    loader: DataLoader[Any] = DataLoader(cast(Any, train_examples), shuffle=True, batch_size=TRAIN_BATCH_SIZE)
+    t0 = time.time()
+    tuned.fit(train_dataloader=loader, epochs=epochs, show_progress_bar=False)
+    train_time_seconds = time.time() - t0
+    tuned.save(output_dir)
+
+    calib_scored = [smart_truncate_pair(q, a, tuned.tokenizer) for q, a in calib_pairs]
+    calib_scores = np.array(tuned.predict(calib_scored, batch_size=32, show_progress_bar=False))
+    threshold = select_threshold(calib_scores, calib_labels)
+
+    tuned_scores = np.array(tuned.predict(test_pairs_scored, batch_size=32, show_progress_bar=False))
+    auc_tuned = roc_auc(tuned_scores, test_labels)
+    ceiling_status = _diagnose_ceiling(tuned_scores, test_labels)
+
+    threshold_hit_rate: float | None = None
+    threshold_error_rate: float | None = None
+    train_label_disagreement_rate: float | None = None
+    if threshold is not None:
+        threshold_hit_rate, threshold_error_rate = _hit_and_error_rate(tuned_scores, test_labels, threshold)
+        train_pairs_scored = [smart_truncate_pair(e.query, e.candidate_answer, tuned.tokenizer) for e in train_rows]
+        train_scores = np.array(tuned.predict(train_pairs_scored, batch_size=32, show_progress_bar=False))
+        train_labels = np.array([1 if e.was_correct else 0 for e in train_rows])
+        train_pred = (train_scores >= threshold).astype(int)
+        train_label_disagreement_rate = float(np.mean(train_pred != train_labels))
+
+    return HealthCheckResult(
+        n_train=len(train_rows),
+        n_calibrate=len(calib_rows),
+        n_test=len(test_rows),
+        train_positive_rate=train_positive_rate,
+        test_positive_rate=test_positive_rate,
+        auc_baseline=auc_baseline,
+        auc_tuned=auc_tuned,
+        train_time_seconds=train_time_seconds,
+        threshold=threshold,
+        threshold_hit_rate=threshold_hit_rate,
+        threshold_error_rate=threshold_error_rate,
+        train_label_disagreement_rate=train_label_disagreement_rate,
+        ceiling_status=ceiling_status,
+        model_path=output_dir,
+    )