clef-evals 0.1.0__tar.gz

clef_evals-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,19 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+      - run: uv build
+      - run: uv publish

clef_evals-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+.env

clef_evals-0.1.0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.5
+Name: clef-evals
+Version: 0.1.0
+Summary: Calibration-first evaluation toolkit for Cloudflare's Clef decision models. Judge cheap, audit confidence.
+Author: Youssef Ouhaghi Ahmian
+License: Apache-2.0
+Keywords: brier,calibration,clef,cloudflare,ece,evaluation,judge,llm
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.25
+Requires-Dist: numpy>=1.24
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# clef-evals
+
+Calibration-first evaluation toolkit for Cloudflare's [Clef](https://blog.cloudflare.com/clef-decision-models/) decision models.
+
+## What it does
+
+- **Judge**: run Clef as an LLM-as-judge over eval sets with typed questions
+- **Calibrate**: compute Expected Calibration Error and Brier score
+- **Gate**: fail CI builds when accuracy or calibration regresses
+
+## Quick start
+
+```bash
+pip install clef-evals
+export CLEF_ACCOUNT_ID=your_id
+export CLEF_API_TOKEN=your_token
+```
+
+```python
+from clef_evals import ClefJudge, ece, brier_score
+
+judge = ClefJudge()
+result = judge.evaluate([
+    {"state": "Email: I need a refund", "question": "category",
+     "choices": ["billing", "technical", "sales"], "gold": "billing"},
+    # ... more items
+])
+print(f"accuracy: {result.accuracy:.4f}")
+print(f"ece:      {result.ece:.4f}")
+print(f"brier:    {result.brier:.4f}")
+```
+
+## License
+
+Apache 2.0

clef_evals-0.1.0/README.md

@@ -0,0 +1,35 @@
+# clef-evals
+
+Calibration-first evaluation toolkit for Cloudflare's [Clef](https://blog.cloudflare.com/clef-decision-models/) decision models.
+
+## What it does
+
+- **Judge**: run Clef as an LLM-as-judge over eval sets with typed questions
+- **Calibrate**: compute Expected Calibration Error and Brier score
+- **Gate**: fail CI builds when accuracy or calibration regresses
+
+## Quick start
+
+```bash
+pip install clef-evals
+export CLEF_ACCOUNT_ID=your_id
+export CLEF_API_TOKEN=your_token
+```
+
+```python
+from clef_evals import ClefJudge, ece, brier_score
+
+judge = ClefJudge()
+result = judge.evaluate([
+    {"state": "Email: I need a refund", "question": "category",
+     "choices": ["billing", "technical", "sales"], "gold": "billing"},
+    # ... more items
+])
+print(f"accuracy: {result.accuracy:.4f}")
+print(f"ece:      {result.ece:.4f}")
+print(f"brier:    {result.brier:.4f}")
+```
+
+## License
+
+Apache 2.0

clef_evals-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "clef-evals"
+version = "0.1.0"
+description = "Calibration-first evaluation toolkit for Cloudflare's Clef decision models. Judge cheap, audit confidence."
+readme = "README.md"
+license = { text = "Apache-2.0" }
+requires-python = ">=3.10"
+authors = [{ name = "Youssef Ouhaghi Ahmian" }]
+keywords = ["clef", "cloudflare", "evaluation", "calibration", "ece", "brier", "llm", "judge"]
+dependencies = ["httpx>=0.25", "numpy>=1.24"]
+
+[project.scripts]
+clef-eval = "clef_evals.cli:main"
+
+[project.optional-dependencies]
+dev = ["pytest>=7"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/clef_evals"]

clef_evals-0.1.0/src/clef_evals/__init__.py

@@ -0,0 +1,117 @@
+"""clef-evals: Calibration-first evaluation toolkit for Cloudflare's Clef."""
+
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+import httpx
+import numpy as np
+
+
+def ece(confidences: list[float], correct: list[bool], n_bins: int = 15) -> float:
+    """Expected Calibration Error."""
+    confs = np.array(confidences)
+    corr = np.array([1.0 if c else 0.0 for c in correct])
+    bins = np.linspace(0, 1, n_bins + 1)
+    ece_val = 0.0
+    for i in range(n_bins):
+        mask = (confs > bins[i]) & (confs <= bins[i + 1])
+        if mask.sum() == 0:
+            continue
+        bin_conf = confs[mask].mean()
+        bin_acc = corr[mask].mean()
+        ece_val += (mask.sum() / len(confs)) * abs(bin_acc - bin_conf)
+    return float(ece_val)
+
+
+def brier_score(confidences: list[float], correct: list[bool]) -> float:
+    """Brier score: mean squared error of probabilistic predictions."""
+    confs = np.array(confidences)
+    corr = np.array([1.0 if c else 0.0 for c in correct])
+    return float(np.mean((confs - corr) ** 2))
+
+
+@dataclass
+class ClefEvalResult:
+    accuracy: float
+    ece: float
+    brier: float
+    n_samples: int
+    correct: list[bool]
+    confidences: list[float]
+
+
+@dataclass
+class ClefJudge:
+    """Run Clef as an LLM-as-judge over an eval set."""
+
+    account_id: str = os.environ.get("CLEF_ACCOUNT_ID", "")
+    api_token: str = os.environ.get("CLEF_API_TOKEN", "")
+    model: str = "@cf/cloudflare/clef-flash"
+    base_url: str = "https://api.cloudflare.com/client/v4"
+    timeout: float = 60.0
+
+    def __post_init__(self):
+        if not self.account_id:
+            raise ValueError("Set CLEF_ACCOUNT_ID")
+
+    def judge_choice(self, state: str, question: str, choices: list[str]) -> tuple[str, float]:
+        """Ask Clef to choose one option. Returns (choice, confidence)."""
+        payload = {
+            "state": state,
+            "questions": {
+                question: {
+                    "type": "choice",
+                    "choices": choices,
+                },
+                "confidence": {
+                    "type": "score",
+                    "context": "How confident are you?",
+                    "levels": ["very_low", "low", "medium", "high", "very_high"],
+                },
+            },
+        }
+        resp = httpx.post(
+            f"{self.base_url}/accounts/{self.account_id}/ai/run/{self.model}",
+            json=payload,
+            headers={"Authorization": f"Bearer {self.api_token}"},
+            timeout=self.timeout,
+        )
+        resp.raise_for_status()
+        result = resp.json().get("result", {})
+        choice = result.get(question, choices[-1])
+        conf_map = {"very_low": 0.1, "low": 0.3, "medium": 0.5, "high": 0.7, "very_high": 0.9}
+        confidence = conf_map.get(result.get("confidence", "low"), 0.3)
+        return choice, confidence
+
+    def evaluate(
+        self,
+        eval_set: list[dict],
+        state_key: str = "state",
+        question_key: str = "question",
+        choices_key: str = "choices",
+        gold_key: str = "gold",
+    ) -> ClefEvalResult:
+        """Run judge over an eval set and compute calibration metrics."""
+        correct_flags, confidence_list = [], []
+        for item in eval_set:
+            pred, conf = self.judge_choice(
+                item[state_key], item[question_key], item[choices_key]
+            )
+            is_correct = pred == item[gold_key]
+            correct_flags.append(is_correct)
+            confidence_list.append(conf)
+
+        accuracy = sum(correct_flags) / len(correct_flags) if correct_flags else 0.0
+        return ClefEvalResult(
+            accuracy=accuracy,
+            ece=ece(confidence_list, correct_flags),
+            brier=brier_score(confidence_list, correct_flags),
+            n_samples=len(correct_flags),
+            correct=correct_flags,
+            confidences=confidence_list,
+        )
+
+
+__version__ = "0.1.0"
+__all__ = ["ClefJudge", "ClefEvalResult", "ece", "brier_score"]

clef_evals-0.1.0/src/clef_evals/cli.py

@@ -0,0 +1,16 @@
+"""CLI for clef-evals."""
+import argparse
+import json
+
+from . import ClefJudge
+
+
+def main():
+    p = argparse.ArgumentParser(prog="clef-eval", description="Evaluate Clef as judge")
+    p.add_argument("--json", action="store_true")
+    args = p.parse_args()
+    print("See README for usage examples.")
+
+
+if __name__ == "__main__":
+    main()