cacheverifier 0.1.0__tar.gz → 0.2.0__tar.gz

cacheverifier-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e ".[dev]"
+      - run: ruff check .
+      - run: pytest -q
+      - run: mypy cacheverifier
+        if: matrix.python-version != '3.9'  # mypy config targets 3.10+
+
+  # `cacheverifier healthcheck` -- the base `test` job above installs only
+  # `.[dev]`, so it exercises the CLI dispatch and the missing-extra path;
+  # this job installs `.[healthcheck,dev]` and runs the real fine-tune
+  # end-to-end. One Python version is enough (the training path isn't
+  # version-sensitive) and CPU-only torch keeps the install lean -- the
+  # default wheel drags in ~3 GB of unused CUDA libraries whose import-time
+  # mmap alone can OOM a small runner.
+  healthcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: python -m pip install --upgrade pip
+      - run: pip install torch --index-url https://download.pytorch.org/whl/cpu
+      - run: pip install -e ".[healthcheck,dev]"
+      - run: ruff check .
+      - run: mypy cacheverifier
+      - run: pytest -q tests/test_healthcheck.py
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build twine
+      - run: python -m build
+      - run: twine check dist/*

cacheverifier-0.2.0/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+## 0.2.0
+
+Add a `cacheverifier` console script with a `healthcheck` subcommand:
+`cacheverifier healthcheck traffic.jsonl` runs the hosted Health Check
+Report's stock-vs-fine-tuned held-out AUC evaluation (`POST /v1/finetune/dry-run`)
+entirely offline — no queries or answers leave the machine. Needs the new
+`healthcheck` extra (`pip install "cacheverifier[healthcheck]"`: torch,
+sentence-transformers, numpy); the base client stays `httpx`-only.
+`--emit-summary` writes an aggregate-only JSON file with no text.
+
+## 0.1.0
+
+Initial release: thin `httpx`-based client for `/v1/verify`, `/v1/verify/batch`,
+`/v1/feedback`(`/batch`), fine-tune jobs + dry run, drift and gray-zone-threshold
+monitoring, and usage/savings. GPTCache `SimilarityEvaluation` adapter under
+`cacheverifier.integrations.gptcache`.

{cacheverifier-0.1.0 → cacheverifier-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cacheverifier
-Version: 0.1.0
+Version: 0.2.0
 Summary: Python client for the hosted CacheVerifier semantic-cache verification API
 Project-URL: Homepage, https://www.cacheverifier.com
 Project-URL: Documentation, https://www.cacheverifier.com/docs
@@ -24,6 +24,10 @@ Requires-Dist: pytest>=7; extra == 'dev'
 Requires-Dist: ruff>=0.4; extra == 'dev'
 Provides-Extra: gptcache
 Requires-Dist: gptcache>=0.1.30; extra == 'gptcache'
+Provides-Extra: healthcheck
+Requires-Dist: numpy<2,>=1.26; extra == 'healthcheck'
+Requires-Dist: sentence-transformers<4,>=3.0; extra == 'healthcheck'
+Requires-Dist: torch>=2.2; extra == 'healthcheck'
 Description-Content-Type: text/markdown
 
 # cacheverifier
@@ -51,9 +55,12 @@ zone", where a plain threshold match might be wrong.
 pip install cacheverifier
 # with the GPTCache adapter:
 pip install "cacheverifier[gptcache]"
+# with the offline Health Check (adds torch + sentence-transformers):
+pip install "cacheverifier[healthcheck]"
 ```
 
-Requires Python 3.9+. The only runtime dependency is `httpx`.
+Requires Python 3.9+. The only runtime dependency is `httpx` — the extras above
+are opt-in.
 
 ## Quickstart
 
@@ -123,6 +130,41 @@ if job.get("result_model_version"):
 `cv.dry_run([...])` reports the same baseline-vs-tuned AUC on examples you pass directly,
 without writing anything or deploying a model.
 
+## Local Health Check (offline)
+
+`cv.dry_run()` still uploads your examples to the API. If that's a blocker — a
+compliance review, or just not wanting production traffic to leave your network —
+run the identical stock-vs-fine-tuned evaluation entirely on your own machine:
+
+```bash
+pip install "cacheverifier[healthcheck]"
+
+cacheverifier healthcheck traffic.jsonl
+cacheverifier healthcheck traffic.jsonl --emit-summary summary.json
+```
+
+`traffic.jsonl` is a JSON array or JSONL of `{"query", "candidate_answer", "was_correct"}`
+rows **in arrival order** (the train/calibrate/test split is chronological, matching the
+hosted service so the numbers are comparable). Optional per row: `"stale": true`.
+
+Nothing is sent anywhere — the base model downloads once from Hugging Face, then it's
+fully offline. `--emit-summary` writes an aggregate-only JSON file (AUCs, counts, rates —
+no query or answer text) that's safe to share for a human read.
+
+```
+results
+------------------------------------------------------------------
+  train / calibrate / test:         3349 / 419 / 419
+  stock verifier   held-out AUC:    0.6120
+  fine-tuned       held-out AUC:    0.7080   (delta +0.0960)
+  label-noise proxy (disagreement): 11.4%
+  ceiling status:                   still_improvable
+
+verdict
+------------------------------------------------------------------
+  IMPROVED   -- fine-tuning on your own data helps this traffic
+```
+
 ## API surface
 
 | method | endpoint |

{cacheverifier-0.1.0 → cacheverifier-0.2.0}/README.md

@@ -23,9 +23,12 @@ zone", where a plain threshold match might be wrong.
 pip install cacheverifier
 # with the GPTCache adapter:
 pip install "cacheverifier[gptcache]"
+# with the offline Health Check (adds torch + sentence-transformers):
+pip install "cacheverifier[healthcheck]"
 ```
 
-Requires Python 3.9+. The only runtime dependency is `httpx`.
+Requires Python 3.9+. The only runtime dependency is `httpx` — the extras above
+are opt-in.
 
 ## Quickstart
 
@@ -95,6 +98,41 @@ if job.get("result_model_version"):
 `cv.dry_run([...])` reports the same baseline-vs-tuned AUC on examples you pass directly,
 without writing anything or deploying a model.
 
+## Local Health Check (offline)
+
+`cv.dry_run()` still uploads your examples to the API. If that's a blocker — a
+compliance review, or just not wanting production traffic to leave your network —
+run the identical stock-vs-fine-tuned evaluation entirely on your own machine:
+
+```bash
+pip install "cacheverifier[healthcheck]"
+
+cacheverifier healthcheck traffic.jsonl
+cacheverifier healthcheck traffic.jsonl --emit-summary summary.json
+```
+
+`traffic.jsonl` is a JSON array or JSONL of `{"query", "candidate_answer", "was_correct"}`
+rows **in arrival order** (the train/calibrate/test split is chronological, matching the
+hosted service so the numbers are comparable). Optional per row: `"stale": true`.
+
+Nothing is sent anywhere — the base model downloads once from Hugging Face, then it's
+fully offline. `--emit-summary` writes an aggregate-only JSON file (AUCs, counts, rates —
+no query or answer text) that's safe to share for a human read.
+
+```
+results
+------------------------------------------------------------------
+  train / calibrate / test:         3349 / 419 / 419
+  stock verifier   held-out AUC:    0.6120
+  fine-tuned       held-out AUC:    0.7080   (delta +0.0960)
+  label-noise proxy (disagreement): 11.4%
+  ceiling status:                   still_improvable
+
+verdict
+------------------------------------------------------------------
+  IMPROVED   -- fine-tuning on your own data helps this traffic
+```
+
 ## API surface
 
 | method | endpoint |

{cacheverifier-0.1.0 → cacheverifier-0.2.0}/cacheverifier/__init__.py

@@ -18,5 +18,5 @@ Research behind it: https://github.com/imxinchengyou/CacheVerifier
 
 from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 __all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"]

cacheverifier-0.2.0/cacheverifier/__main__.py

@@ -0,0 +1,44 @@
+"""`python -m cacheverifier` / the `cacheverifier` console script.
+
+A thin argparse dispatcher. The base install (httpx only) provides
+`--version` and `--help`; the `healthcheck` subcommand additionally needs
+the `healthcheck` extra and imports nothing heavy until it runs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from cacheverifier import __version__
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="cacheverifier",
+        description="CacheVerifier -- hosted semantic-cache verification (https://www.cacheverifier.com).",
+    )
+    parser.add_argument("--version", action="version", version=f"cacheverifier {__version__}")
+    subparsers = parser.add_subparsers(dest="command", metavar="<command>")
+
+    # Import lazily and defensively: a missing healthcheck extra must not
+    # break `cacheverifier --help` or `--version`. cli.add_subparser itself
+    # only touches argparse.
+    from cacheverifier._healthcheck import cli as healthcheck_cli
+
+    healthcheck_cli.add_subparser(subparsers)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "command", None):
+        parser.print_help()
+        return 1
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cacheverifier-0.2.0/cacheverifier/_healthcheck/__init__.py

@@ -0,0 +1,7 @@
+"""`cacheverifier healthcheck` -- run the hosted Health Check Report's
+stock-vs-fine-tuned AUC evaluation entirely on your own machine.
+
+Everything under here needs the `healthcheck` extra
+(`pip install "cacheverifier[healthcheck]"`: torch, sentence-transformers,
+scikit-learn, numpy) and is imported only when the subcommand runs.
+"""

cacheverifier-0.2.0/cacheverifier/_healthcheck/_finetune.py

@@ -0,0 +1,274 @@
+"""Local fine-tune + held-out AUC evaluation for `cacheverifier healthcheck`.
+
+This is a trimmed, self-contained port of the hosted service's
+`verifier_core.finetune` / `verifier_core.cross_encoder` (the code behind
+`POST /v1/finetune/dry-run`). It keeps the parts a Health Check needs --
+the chronological train/calibrate/test split, held-out AUC for the stock
+vs. fine-tuned model, a Youden's-J operating point, and the label-noise
+and ceiling diagnostics -- and drops everything specific to running a
+production tenant (Conformal Risk Control certification, cost-ratio
+threshold grids, drift monitoring, operating-point-jump detection).
+
+Nothing here is imported unless the `healthcheck` extra is installed and
+the subcommand is actually run -- see `cacheverifier.__main__`.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Any, cast
+
+import numpy as np
+import torch
+from sentence_transformers import CrossEncoder, InputExample
+from torch.utils.data import DataLoader
+
+# All four imports above are from the `healthcheck` extra. This module is
+# only ever imported from inside `cacheverifier._healthcheck.cli.run()`,
+# which catches the ImportError and points the user at
+# `pip install "cacheverifier[healthcheck]"` -- so a plain
+# `import cacheverifier` / `cacheverifier --help` never pays this cost.
+
+DEFAULT_BASE_MODEL = "cross-encoder/ms-marco-MiniLM-L6-v2"
+"""The stock off-the-shelf verifier every tenant starts on -- the same base
+model the hosted service fine-tunes from."""
+
+MAX_SEQUENCE_LENGTH = 128
+"""Token cap applied identically at train and score time. Matches the
+hosted service; BERT-style attention memory is O(n^2) in sequence length,
+and this keeps a CPU run well-behaved."""
+
+_CONTENT_TOKEN_BUDGET = MAX_SEQUENCE_LENGTH - 3  # [CLS] + 2x [SEP]
+_MIN_QUERY_TOKENS = 32
+
+TRAIN_BATCH_SIZE = 4
+MIN_TRAIN_EXAMPLES = 20
+"""Hard floor -- below this there aren't enough rows for one meaningful
+epoch plus a held-out split."""
+
+COLD_START_WARNING_THRESHOLD = 1000
+"""The hosted service's research (Research page, Q2) found fine-tuning only
+reliably positive across every dataset above roughly this many rows. Below
+it a result is real but noisy."""
+
+_MIN_TEST_FOR_THRESHOLD = 10
+_MIN_TEST_FOR_CEILING_DIAGNOSIS = 50
+_CEILING_TAIL_MARGIN = 0.15
+_CEILING_AUC_CAP = 0.75
+
+NOISE_WARNING_DISAGREEMENT_RATE = 0.20
+"""`train_label_disagreement_rate` above this: worth auditing how the
+correctness signal is produced."""
+NOISE_HOLD_BACK_DISAGREEMENT_RATE = 0.30
+"""Above this the hosted service holds a new model back for review."""
+
+
+@dataclass(frozen=True)
+class GrayZoneExample:
+    query: str
+    candidate_answer: str
+    was_correct: bool
+
+
+@dataclass(frozen=True)
+class HealthCheckResult:
+    n_train: int
+    n_calibrate: int
+    n_test: int
+    train_positive_rate: float
+    test_positive_rate: float
+    auc_baseline: float
+    auc_tuned: float
+    train_time_seconds: float
+    threshold: float | None
+    threshold_hit_rate: float | None
+    threshold_error_rate: float | None
+    train_label_disagreement_rate: float | None
+    ceiling_status: str | None
+    model_path: str
+
+    @property
+    def auc_delta(self) -> float:
+        return self.auc_tuned - self.auc_baseline
+
+
+def _head_tail_truncate(ids: list[int], budget: int, head_ratio: float = 0.6) -> list[int]:
+    if len(ids) <= budget:
+        return ids
+    head_n = int(budget * head_ratio)
+    tail_n = budget - head_n
+    return ids[:head_n] + (ids[-tail_n:] if tail_n > 0 else [])
+
+
+def smart_truncate_pair(query: str, candidate_answer: str, tokenizer: Any) -> tuple[str, str]:
+    """Tokenizer-aware head+tail truncation, applied identically at train
+    and score time (a model trained on one truncation policy and scored
+    with another sees a different token window than it was optimized
+    against). A no-op whenever the pair already fits."""
+    query_ids = tokenizer.encode(query, add_special_tokens=False)
+    answer_ids = tokenizer.encode(candidate_answer, add_special_tokens=False)
+    if len(query_ids) + len(answer_ids) <= _CONTENT_TOKEN_BUDGET:
+        return query, candidate_answer
+
+    query_budget = min(len(query_ids), max(_MIN_QUERY_TOKENS, _CONTENT_TOKEN_BUDGET // 2))
+    answer_budget = _CONTENT_TOKEN_BUDGET - query_budget
+    query_ids = _head_tail_truncate(query_ids, query_budget)
+    answer_ids = _head_tail_truncate(answer_ids, answer_budget)
+    return (
+        tokenizer.decode(query_ids, skip_special_tokens=True),
+        tokenizer.decode(answer_ids, skip_special_tokens=True),
+    )
+
+
+def roc_auc(scores: np.ndarray, labels: np.ndarray) -> float:
+    """Rank-based AUC, no sklearn dependency. NaN if either class is absent."""
+    order = np.argsort(scores)
+    ranks = np.empty_like(order, dtype=float)
+    ranks[order] = np.arange(len(scores))
+    pos_ranks = ranks[labels == 1]
+    n_pos, n_neg = int((labels == 1).sum()), int((labels == 0).sum())
+    if n_pos == 0 or n_neg == 0:
+        return float("nan")
+    return float((pos_ranks.sum() - n_pos * (n_pos - 1) / 2) / (n_pos * n_neg))
+
+
+def select_threshold(scores: np.ndarray, labels: np.ndarray) -> float | None:
+    """Youden's J: the threshold maximizing (TPR - FPR). None when the
+    held-out split is too small or single-class to calibrate from -- the
+    caller decides the fallback."""
+    n_pos = int((labels == 1).sum())
+    n_neg = int((labels == 0).sum())
+    if n_pos == 0 or n_neg == 0 or (n_pos + n_neg) < _MIN_TEST_FOR_THRESHOLD:
+        return None
+
+    order = np.argsort(-scores)
+    sorted_scores = scores[order]
+    sorted_labels = labels[order]
+    tp = np.cumsum(sorted_labels == 1)
+    fp = np.cumsum(sorted_labels == 0)
+    youden_j = tp / n_pos - fp / n_neg
+    return float(sorted_scores[int(np.argmax(youden_j))])
+
+
+def _hit_and_error_rate(scores: np.ndarray, labels: np.ndarray, threshold: float) -> tuple[float, float | None]:
+    approved = scores >= threshold
+    n_approved = int(approved.sum())
+    hit_rate = n_approved / len(scores) if len(scores) else 0.0
+    if n_approved == 0:
+        return hit_rate, None
+    return hit_rate, float(np.mean(labels[approved] == 0))
+
+
+def _diagnose_ceiling(scores: np.ndarray, labels: np.ndarray) -> str | None:
+    """"insufficient_data" | "possible_ceiling" | "still_improvable" -- is
+    the fine-tuned verifier plausibly undertrained (more data would help)
+    or capped by the domain's own ambiguity? Precision in the top score
+    decile vs. the base rate, same shape of check as the hosted service."""
+    n = len(labels)
+    if n < _MIN_TEST_FOR_CEILING_DIAGNOSIS:
+        return "insufficient_data"
+    base_rate = float(labels.mean())
+    top_decile_n = max(1, n // 10)
+    top_decile_idx = np.argsort(-scores)[:top_decile_n]
+    precision_at_top_decile = float(labels[top_decile_idx].mean())
+    auc = roc_auc(scores, labels)
+    if precision_at_top_decile - base_rate >= _CEILING_TAIL_MARGIN and auc < _CEILING_AUC_CAP:
+        return "possible_ceiling"
+    return "still_improvable"
+
+
+def run_healthcheck(
+    examples: list[GrayZoneExample],
+    output_dir: str,
+    *,
+    base_model: str = DEFAULT_BASE_MODEL,
+    epochs: int = 1,
+) -> HealthCheckResult:
+    """Fine-tune `base_model` on a chronological prefix of `examples` and
+    measure held-out AUC for the stock vs. fine-tuned model.
+
+    `examples` must be in arrival (stream) order -- the train/calibrate/test
+    split is positional, not shuffled, matching the hosted service so the
+    numbers are comparable.
+    """
+    if len(examples) < MIN_TRAIN_EXAMPLES:
+        raise ValueError(f"need at least {MIN_TRAIN_EXAMPLES} examples, got {len(examples)}")
+
+    split = max(1, int(len(examples) * 0.8))
+    train_rows, holdout_rows = examples[:split], examples[split:]
+    if not holdout_rows:
+        train_rows, holdout_rows = examples[:-1], examples[-1:]
+    calib_split = max(1, len(holdout_rows) // 2)
+    calib_rows, test_rows = holdout_rows[:calib_split], holdout_rows[calib_split:]
+    if not test_rows:
+        calib_rows, test_rows = holdout_rows[:-1], holdout_rows[-1:]
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    test_pairs = [(e.query, e.candidate_answer) for e in test_rows]
+    test_labels = np.array([1 if e.was_correct else 0 for e in test_rows])
+    calib_pairs = [(e.query, e.candidate_answer) for e in calib_rows]
+    calib_labels = np.array([1 if e.was_correct else 0 for e in calib_rows])
+    train_positive_rate = float(np.mean([1 if e.was_correct else 0 for e in train_rows]))
+    test_positive_rate = float(test_labels.mean()) if len(test_labels) else 0.0
+
+    baseline = CrossEncoder(base_model, device=device, max_length=MAX_SEQUENCE_LENGTH)
+    test_pairs_scored = [smart_truncate_pair(q, a, baseline.tokenizer) for q, a in test_pairs]
+    baseline_scores = np.array(baseline.predict(test_pairs_scored, batch_size=32, show_progress_bar=False))
+    auc_baseline = roc_auc(baseline_scores, test_labels)
+    del baseline
+
+    tuned = CrossEncoder(base_model, device=device, max_length=MAX_SEQUENCE_LENGTH)
+    train_examples = [
+        InputExample(
+            texts=list(smart_truncate_pair(e.query, e.candidate_answer, tuned.tokenizer)),
+            label=1.0 if e.was_correct else 0.0,
+        )
+        for e in train_rows
+    ]
+    # A list[InputExample] is a valid PyTorch map-style dataset at runtime;
+    # sentence-transformers' pre-4.0 training API is built around exactly
+    # this, but the two libraries' stubs don't compose (InputExample isn't a
+    # typed Dataset). cast rather than `# type: ignore` so this stays clean
+    # whether or not torch's stubs are installed (base CI has no torch).
+    loader: DataLoader[Any] = DataLoader(cast(Any, train_examples), shuffle=True, batch_size=TRAIN_BATCH_SIZE)
+    t0 = time.time()
+    tuned.fit(train_dataloader=loader, epochs=epochs, show_progress_bar=False)
+    train_time_seconds = time.time() - t0
+    tuned.save(output_dir)
+
+    calib_scored = [smart_truncate_pair(q, a, tuned.tokenizer) for q, a in calib_pairs]
+    calib_scores = np.array(tuned.predict(calib_scored, batch_size=32, show_progress_bar=False))
+    threshold = select_threshold(calib_scores, calib_labels)
+
+    tuned_scores = np.array(tuned.predict(test_pairs_scored, batch_size=32, show_progress_bar=False))
+    auc_tuned = roc_auc(tuned_scores, test_labels)
+    ceiling_status = _diagnose_ceiling(tuned_scores, test_labels)
+
+    threshold_hit_rate: float | None = None
+    threshold_error_rate: float | None = None
+    train_label_disagreement_rate: float | None = None
+    if threshold is not None:
+        threshold_hit_rate, threshold_error_rate = _hit_and_error_rate(tuned_scores, test_labels, threshold)
+        train_pairs_scored = [smart_truncate_pair(e.query, e.candidate_answer, tuned.tokenizer) for e in train_rows]
+        train_scores = np.array(tuned.predict(train_pairs_scored, batch_size=32, show_progress_bar=False))
+        train_labels = np.array([1 if e.was_correct else 0 for e in train_rows])
+        train_pred = (train_scores >= threshold).astype(int)
+        train_label_disagreement_rate = float(np.mean(train_pred != train_labels))
+
+    return HealthCheckResult(
+        n_train=len(train_rows),
+        n_calibrate=len(calib_rows),
+        n_test=len(test_rows),
+        train_positive_rate=train_positive_rate,
+        test_positive_rate=test_positive_rate,
+        auc_baseline=auc_baseline,
+        auc_tuned=auc_tuned,
+        train_time_seconds=train_time_seconds,
+        threshold=threshold,
+        threshold_hit_rate=threshold_hit_rate,
+        threshold_error_rate=threshold_error_rate,
+        train_label_disagreement_rate=train_label_disagreement_rate,
+        ceiling_status=ceiling_status,
+        model_path=output_dir,
+    )

cacheverifier-0.2.0/cacheverifier/_healthcheck/cli.py

@@ -0,0 +1,209 @@
+"""Argument parsing, IO, and the printed report for `cacheverifier healthcheck`.
+
+The heavy work (torch, sentence-transformers) lives in `._finetune` and is
+imported only inside `run()`, so `cacheverifier --help` stays light and
+works without the `healthcheck` extra installed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+_MISSING_EXTRA_HINT = (
+    "the healthcheck extra is not installed -- run:\n"
+    '    pip install "cacheverifier[healthcheck]"\n'
+    "(this pulls in torch + sentence-transformers; it is deliberately not a\n"
+    "dependency of the base client)"
+)
+
+
+def add_subparser(subparsers: argparse._SubParsersAction) -> None:
+    p = subparsers.add_parser(
+        "healthcheck",
+        help="run the Health Check evaluation locally (no data leaves your machine)",
+        description=(
+            "Fine-tune the stock verifier on a chronological prefix of your own "
+            "gray-zone cache hits and measure held-out AUC before and after -- the "
+            "same evaluation as POST /v1/finetune/dry-run, but entirely offline. "
+            "Your queries and answers never leave this host; only the optional "
+            "--emit-summary file (floats and counts, no text) can."
+        ),
+    )
+    p.add_argument(
+        "input",
+        type=Path,
+        help="JSON array or JSONL of {query, candidate_answer, was_correct} rows, in arrival order",
+    )
+    p.add_argument(
+        "--emit-summary",
+        type=Path,
+        default=None,
+        metavar="PATH",
+        help="also write an aggregate-only JSON summary (no query/answer text) to PATH",
+    )
+    p.add_argument(
+        "--epochs",
+        type=int,
+        default=1,
+        help="fine-tuning epochs (default: 1, matching the hosted service)",
+    )
+    p.add_argument(
+        "--keep-model",
+        action="store_true",
+        help="keep the locally fine-tuned model directory instead of deleting it",
+    )
+    p.set_defaults(func=run)
+
+
+def _load_rows(path: Path) -> list[dict[str, Any]]:
+    text = path.read_text()
+    if text.lstrip().startswith("["):
+        rows = json.loads(text)
+    else:
+        rows = [json.loads(line) for line in text.splitlines() if line.strip()]
+    if not isinstance(rows, list):
+        # ValueError, not TypeError: this is bad input data, not a caller
+        # bug -- run() catches it alongside JSONDecodeError and prints it.
+        raise ValueError("input must be a JSON array or JSONL of objects")  # noqa: TRY004
+    return rows
+
+
+def _to_examples(rows: list[dict[str, Any]], example_cls: Any) -> tuple[list[Any], int]:
+    examples = []
+    n_stale = 0
+    for i, row in enumerate(rows):
+        missing = {"query", "candidate_answer", "was_correct"} - row.keys()
+        if missing:
+            raise ValueError(f"row {i}: missing key(s) {sorted(missing)}")
+        if row.get("stale"):
+            n_stale += 1
+            continue
+        examples.append(
+            example_cls(
+                query=str(row["query"]),
+                candidate_answer=str(row["candidate_answer"]),
+                was_correct=bool(row["was_correct"]),
+            )
+        )
+    return examples, n_stale
+
+
+def _fmt(x: float | None, *, pct: bool = False) -> str:
+    if x is None:
+        return "n/a"
+    return f"{x * 100:.1f}%" if pct else f"{x:.4f}"
+
+
+def _verdict(delta: float) -> str:
+    if delta >= 0.02:
+        return "IMPROVED   -- fine-tuning on your own data helps this traffic"
+    if delta <= -0.02:
+        return "WORSE      -- fine-tuning hurt; your signal may be too noisy or too sparse"
+    return "NO CHANGE   -- fine-tuning neither helped nor hurt measurably"
+
+
+def run(args: argparse.Namespace) -> int:
+    try:
+        from cacheverifier._healthcheck import _finetune
+    except ImportError as e:  # torch / sentence-transformers / numpy absent
+        print(f"error: {_MISSING_EXTRA_HINT}\n\n(import failed: {e})", file=sys.stderr)
+        return 2
+
+    input_path: Path = args.input
+    if not input_path.exists():
+        print(f"error: input file not found: {input_path}", file=sys.stderr)
+        return 2
+
+    try:
+        rows = _load_rows(input_path)
+        examples, n_stale = _to_examples(rows, _finetune.GrayZoneExample)
+    except (ValueError, json.JSONDecodeError, OSError) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+    n = len(examples)
+    print(f"\nCacheVerifier local Health Check  --  {input_path}")
+    print("=" * 66)
+    print(f"rows read:            {len(rows)}")
+    print(f"usable (non-stale):   {n}")
+    if n_stale:
+        print(f"stale, excluded:      {n_stale}")
+
+    if n < _finetune.MIN_TRAIN_EXAMPLES:
+        print(f"\nNeed at least {_finetune.MIN_TRAIN_EXAMPLES} non-stale rows to run; have {n}.", file=sys.stderr)
+        return 2
+    if n < _finetune.COLD_START_WARNING_THRESHOLD:
+        print(
+            f"\nNote: {n} rows is below the ~{_finetune.COLD_START_WARNING_THRESHOLD} the research "
+            "(cacheverifier.com/research, Q2) found fine-tuning reliably positive across every\n"
+            "      dataset. A small-sample result here is real but noisy -- re-run as feedback grows."
+        )
+
+    import tempfile
+
+    out_dir = tempfile.mkdtemp(prefix="cacheverifier_healthcheck_")
+    print(f"\nfine-tuning locally (nothing sent) -> {out_dir}")
+    print("this takes a few minutes on CPU; the base model downloads once on first run...\n")
+
+    result = _finetune.run_healthcheck(examples, out_dir, base_model=_finetune.DEFAULT_BASE_MODEL, epochs=args.epochs)
+
+    print("results")
+    print("-" * 66)
+    print(f"  train / calibrate / test:         {result.n_train} / {result.n_calibrate} / {result.n_test}")
+    print(f"  stock verifier   held-out AUC:    {_fmt(result.auc_baseline)}")
+    print(f"  fine-tuned       held-out AUC:    {_fmt(result.auc_tuned)}   (delta {result.auc_delta:+.4f})")
+    print(f"  label-noise proxy (disagreement): {_fmt(result.train_label_disagreement_rate, pct=True)}")
+    if result.train_label_disagreement_rate is not None:
+        if result.train_label_disagreement_rate > _finetune.NOISE_HOLD_BACK_DISAGREEMENT_RATE:
+            print("      -> HIGH: the hosted service would hold this model back for review")
+        elif result.train_label_disagreement_rate > _finetune.NOISE_WARNING_DISAGREEMENT_RATE:
+            print("      -> elevated: worth auditing how your correctness signal is produced")
+    print(f"  train / test positive rate:       {_fmt(result.train_positive_rate, pct=True)} / {_fmt(result.test_positive_rate, pct=True)}")
+    print(f"  ceiling status:                   {result.ceiling_status or 'n/a'}")
+    if result.threshold is not None:
+        print(
+            f"  operating point @ picked threshold: hit rate {_fmt(result.threshold_hit_rate, pct=True)}, "
+            f"error rate {_fmt(result.threshold_error_rate, pct=True)}"
+        )
+    print("\nverdict")
+    print("-" * 66)
+    print(f"  {_verdict(result.auc_delta)}\n")
+
+    if args.emit_summary is not None:
+        summary = {
+            "n_rows_read": len(rows),
+            "n_usable": n,
+            "n_stale_excluded": n_stale,
+            "n_train": result.n_train,
+            "n_calibrate": result.n_calibrate,
+            "n_test": result.n_test,
+            "auc_stock": round(result.auc_baseline, 4),
+            "auc_finetuned": round(result.auc_tuned, 4),
+            "auc_delta": round(result.auc_delta, 4),
+            "train_label_disagreement_rate": (
+                round(result.train_label_disagreement_rate, 4)
+                if result.train_label_disagreement_rate is not None
+                else None
+            ),
+            "train_positive_rate": round(result.train_positive_rate, 4),
+            "test_positive_rate": round(result.test_positive_rate, 4),
+            "ceiling_status": result.ceiling_status,
+            "threshold_hit_rate": result.threshold_hit_rate,
+            "threshold_error_rate": result.threshold_error_rate,
+        }
+        args.emit_summary.write_text(json.dumps(summary, indent=2) + "\n")
+        print(f"aggregate summary written to {args.emit_summary}")
+        print("(floats and counts only -- no query or answer text -- safe to share for a human read)\n")
+
+    if args.keep_model:
+        print(f"fine-tuned model kept at {result.model_path}")
+    else:
+        import shutil
+
+        shutil.rmtree(out_dir, ignore_errors=True)
+
+    return 0

{cacheverifier-0.1.0 → cacheverifier-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cacheverifier"
-version = "0.1.0"
+version = "0.2.0"
 description = "Python client for the hosted CacheVerifier semantic-cache verification API"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -29,6 +29,9 @@ classifiers = [
 ]
 dependencies = ["httpx>=0.24"]
 
+[project.scripts]
+cacheverifier = "cacheverifier.__main__:main"
+
 [project.urls]
 Homepage = "https://www.cacheverifier.com"
 Documentation = "https://www.cacheverifier.com/docs"
@@ -37,6 +40,20 @@ Research = "https://github.com/imxinchengyou/CacheVerifier"
 
 [project.optional-dependencies]
 gptcache = ["gptcache>=0.1.30"]
+# `cacheverifier healthcheck` -- runs the hosted Health Check Report's
+# stock-vs-fine-tuned evaluation locally. Deliberately NOT a base
+# dependency: torch + sentence-transformers dwarf the httpx-only client.
+# sentence-transformers is pinned <4 for the pre-4.0 CrossEncoder.fit()
+# API this uses (the >=4 line needs torch>=2.4, which drops older Pythons).
+# Effectively wants Python >=3.10.
+healthcheck = [
+    # numpy <2: the torch builds these versions pull are compiled against
+    # the numpy 1.x ABI ("Failed to initialize NumPy: _ARRAY_API not found"
+    # on numpy 2). Same constraint the hosted service uses.
+    "numpy>=1.26,<2",
+    "torch>=2.2",
+    "sentence-transformers>=3.0,<4",
+]
 dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4"]
 
 [tool.hatch.build.targets.wheel]
@@ -60,3 +77,11 @@ warn_redundant_casts = true
 [[tool.mypy.overrides]]
 module = "gptcache.*"
 ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+# The healthcheck extra's deps: not installed in the base test/type-check
+# env (CI runs `pip install -e ".[dev]"`, not `[dev,healthcheck]`), and the
+# `_healthcheck` subpackage is only importable when that extra is present
+# anyway. numpy ships inline types but only when installed.
+module = ["torch.*", "sentence_transformers.*", "numpy.*"]
+ignore_missing_imports = true

cacheverifier-0.2.0/tests/test_healthcheck.py

@@ -0,0 +1,98 @@
+"""CLI dispatch + Health Check runner.
+
+The dispatch/parsing tests run everywhere (base install). The end-to-end
+training test needs the `healthcheck` extra and is skipped when torch is
+absent -- CI's base matrix skips it; the dedicated `healthcheck` job runs
+it.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+from dataclasses import dataclass
+
+import pytest
+
+from cacheverifier.__main__ import main
+from cacheverifier._healthcheck import cli
+
+HAS_TORCH = importlib.util.find_spec("torch") is not None
+
+
+@dataclass(frozen=True)
+class _DummyExample:
+    query: str
+    candidate_answer: str
+    was_correct: bool
+
+
+class TestDispatch:
+    def test_version(self, capsys):
+        with pytest.raises(SystemExit) as e:
+            main(["--version"])
+        assert e.value.code == 0
+        assert "cacheverifier 0.2.0" in capsys.readouterr().out
+
+    def test_no_command_prints_help_and_returns_1(self, capsys):
+        assert main([]) == 1
+        assert "healthcheck" in capsys.readouterr().out
+
+    def test_healthcheck_help(self):
+        with pytest.raises(SystemExit) as e:
+            main(["healthcheck", "--help"])
+        assert e.value.code == 0
+
+    @pytest.mark.skipif(HAS_TORCH, reason="torch present -- the missing-extra path can't be exercised")
+    def test_healthcheck_without_extra_points_at_pip_install(self, capsys, tmp_path):
+        f = tmp_path / "t.jsonl"
+        f.write_text("")
+        assert main(["healthcheck", str(f)]) == 2
+        assert 'pip install "cacheverifier[healthcheck]"' in capsys.readouterr().err
+
+
+class TestParsing:
+    def test_load_rows_accepts_json_array_and_jsonl(self, tmp_path):
+        rows = [{"a": 1}, {"a": 2}]
+        arr = tmp_path / "a.json"
+        arr.write_text(json.dumps(rows))
+        jsonl = tmp_path / "a.jsonl"
+        jsonl.write_text("\n".join(json.dumps(r) for r in rows) + "\n")
+        assert cli._load_rows(arr) == rows
+        assert cli._load_rows(jsonl) == rows
+
+    def test_to_examples_skips_stale_and_counts_them(self):
+        rows = [
+            {"query": "q1", "candidate_answer": "a1", "was_correct": True},
+            {"query": "q2", "candidate_answer": "a2", "was_correct": False, "stale": True},
+        ]
+        examples, n_stale = cli._to_examples(rows, _DummyExample)
+        assert len(examples) == 1
+        assert n_stale == 1
+        assert examples[0] == _DummyExample("q1", "a1", True)
+
+    def test_to_examples_rejects_missing_keys(self):
+        with pytest.raises(ValueError, match="missing key"):
+            cli._to_examples([{"query": "q"}], _DummyExample)
+
+
+@pytest.mark.skipif(not HAS_TORCH, reason="needs the healthcheck extra (torch, sentence-transformers)")
+def test_run_healthcheck_end_to_end(tmp_path):
+    from cacheverifier._healthcheck._finetune import GrayZoneExample, HealthCheckResult, run_healthcheck
+
+    groups = [
+        ("how do I cancel my subscription", "Settings > Billing > Cancel.", "Settings > Billing > Pause a month."),
+        ("how do I get a refund", "Orders > Return > Request refund.", "Orders > Return > Exchange size."),
+        ("how do I reset my password", "Account > Security > Reset password.", "Account > Security > Enable 2FA."),
+    ]
+    examples = []
+    for i in range(36):
+        q, good, bad = groups[i % len(groups)]
+        correct = i % 2 == 0
+        examples.append(GrayZoneExample(q, good if correct else bad, correct))
+
+    result = run_healthcheck(examples, str(tmp_path / "model"), epochs=1)
+    assert isinstance(result, HealthCheckResult)
+    assert result.n_train + result.n_calibrate + result.n_test == 36
+    assert 0.0 <= result.auc_tuned <= 1.0
+    assert (tmp_path / "model").is_dir()

cacheverifier-0.1.0/.github/workflows/ci.yml

@@ -1,36 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: python -m pip install --upgrade pip
-      - run: pip install -e ".[dev]"
-      - run: ruff check .
-      - run: pytest -q
-      - run: mypy cacheverifier
-        if: matrix.python-version != '3.9'  # mypy config targets 3.10+
-
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - run: pip install build twine
-      - run: python -m build
-      - run: twine check dist/*

cacheverifier-0.1.0/CHANGELOG.md

@@ -1,8 +0,0 @@
-# Changelog
-
-## 0.1.0
-
-Initial release: thin `httpx`-based client for `/v1/verify`, `/v1/verify/batch`,
-`/v1/feedback`(`/batch`), fine-tune jobs + dry run, drift and gray-zone-threshold
-monitoring, and usage/savings. GPTCache `SimilarityEvaluation` adapter under
-`cacheverifier.integrations.gptcache`.