cafe-core 0.0.1__py3-none-any.whl

cafe/__init__.py

@@ -0,0 +1,184 @@
+"""CAFE core — a design-of-experiments evaluation engine for compound AI systems.
+
+CAFE *measures* compound AI systems; it does not implement them. You provide a
+black box ``run(config, item) -> output`` and declare the factors to vary; CAFE
+generates the design, executes every configuration with replication, and judges
+and attributes the results.
+
+Quick start (no API keys needed)::
+
+    from cafe.examples import build_example_study
+    results = build_example_study().run()
+    print(results.summary())
+
+Or define your own::
+
+    import cafe
+
+    async def my_system(config, item):
+        return f"answer for {item} using {config['model']}"
+
+    study = cafe.Study(
+        name="my-study",
+        system=my_system,
+        factors=[cafe.Factor("model", ["a", "b"])],
+        dataset=["question 1", "question 2"],
+    )
+    results = study.run()
+"""
+
+from cafe import (
+    datasets,
+    design,
+    evaluation,
+    execution,
+    judging,
+    llm,
+    rubrics,
+    stats,
+    study,
+    system,
+    techniques,
+)
+from cafe.design import (
+    FractionalDesign,
+    fractional_factorial,
+    fractional_factorial_design,
+    full_factorial,
+    generate,
+    single,
+    size,
+)
+from cafe.techniques import (
+    Pipeline,
+    PipelineView,
+    pipeline,
+    stage_report,
+)
+from cafe.evaluation import Evaluation, Preflight, evaluate, preflight
+from cafe.execution import estimate, run_study
+from cafe.judging import (
+    JUDGE_PRESETS,
+    Judge,
+    JudgeOutput,
+    LLMJudge,
+    Rating,
+    Ratings,
+    build_judge_prompt,
+    judge_results,
+)
+from cafe.llm import LLMError, complete, set_model_cost
+from cafe.execution.results import Observation, Results, config_id, config_label
+from cafe.judging.rubric import ANSWER_QUALITY_1_5, Level, Rubric, ScaleType
+from cafe.stats import (
+    Attribution,
+    CLMMResult,
+    Effects,
+    HumanRatings,
+    JudgeStability,
+    Logistic,
+    ParetoResult,
+    Reliability,
+    answer_sheet,
+    attribute,
+    check_r,
+    fit_clmm,
+    fit_effects,
+    fit_logistic,
+    human_evaluation,
+    human_ratings,
+    judge_stability,
+    krippendorff_alpha,
+    pareto,
+    ratings_from_human,
+    reliability,
+)
+from cafe.study import Factor, FactorType, Study
+from cafe.system import System, as_system
+
+__version__ = "0.0.1"
+
+__all__ = [
+    # ── data ──
+    "datasets",
+    "rubrics",
+    # ── public submodules (for `cafe.<tab>`, `from cafe import *`, API docs) ──
+    "design",
+    "evaluation",
+    "execution",
+    "judging",
+    "llm",
+    "stats",
+    "study",
+    "system",
+    "techniques",
+    # ── define an experiment ──
+    "Study",
+    "Factor",
+    "FactorType",
+    "System",
+    "as_system",
+    # ── composed pipelines (techniques) ──
+    "Pipeline",
+    "PipelineView",
+    "stage_report",
+    "pipeline",
+    # ── run it ──
+    "evaluate",       # the complete pipeline: answers -> judge -> attribution
+    "preflight",      # cheap pre-run check + cost estimate
+    "run_study",      # lower-level: answers only
+    "Evaluation",
+    "Preflight",
+    "Results",
+    "Observation",
+    "estimate",
+    # ── designs ──
+    "generate",
+    "size",
+    "full_factorial",
+    "single",
+    "fractional_factorial",
+    "fractional_factorial_design",
+    "FractionalDesign",
+    "config_id",
+    "config_label",
+    # ── judging ──
+    "complete",
+    "set_model_cost",
+    "LLMError",
+    "Rubric",
+    "Level",
+    "ScaleType",
+    "ANSWER_QUALITY_1_5",
+    "LLMJudge",
+    "Judge",
+    "JudgeOutput",
+    "JUDGE_PRESETS",
+    "judge_results",
+    "build_judge_prompt",
+    "Rating",
+    "Ratings",
+    # ── statistics ──
+    "attribute",
+    "Attribution",
+    "fit_effects",
+    "Effects",
+    "fit_clmm",
+    "CLMMResult",
+    "check_r",
+    "pareto",
+    "ParetoResult",
+    "judge_stability",
+    "JudgeStability",
+    "fit_logistic",
+    "Logistic",
+    "reliability",
+    "Reliability",
+    "human_ratings",
+    "HumanRatings",
+    "answer_sheet",
+    "krippendorff_alpha",
+    "ratings_from_human",
+    "human_evaluation",
+    "__version__",
+]

cafe/_async.py

@@ -0,0 +1,36 @@
+"""Run a coroutine to completion from sync code, in a script *or* a notebook.
+
+If no event loop is running (a plain script) we just ``asyncio.run``. If one is already
+running (e.g. a Jupyter cell), running another loop on the same thread would raise
+``asyncio.run() cannot be called from a running event loop`` — so we run it on a short-lived
+worker thread and join. This is what lets ``study.evaluate()`` / ``result.rejudge()`` be
+plain sync calls everywhere.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from typing import Any, Callable
+
+
+def run_blocking(make_coro: Callable[[], Any]) -> Any:
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(make_coro())
+
+    box: dict[str, Any] = {}
+
+    def _worker() -> None:
+        try:
+            box["result"] = asyncio.run(make_coro())
+        except BaseException as exc:  # noqa: BLE001 — re-raised on the caller's thread
+            box["error"] = exc
+
+    thread = threading.Thread(target=_worker, name="cafe-run")
+    thread.start()
+    thread.join()
+    if "error" in box:
+        raise box["error"]
+    return box["result"]

cafe/_env.py

@@ -0,0 +1,32 @@
+"""Tiny zero-dependency .env loader.
+
+Looks for the nearest ``.env`` file (current directory upward) and loads its
+``KEY=VALUE`` lines into ``os.environ`` without overwriting anything already set.
+Kept dependency-free on purpose — the engine never requires python-dotenv.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def load_env(start: str | None = None) -> str | None:
+    """Load the nearest ``.env`` into the environment. Returns the path used, or None.
+
+    Existing environment variables win (we only ``setdefault``), so real env vars
+    and CI secrets always override a local ``.env``.
+    """
+    base = Path(start or os.getcwd()).resolve()
+    for cur in [base, *base.parents]:
+        candidate = cur / ".env"
+        if candidate.is_file():
+            for raw in candidate.read_text(encoding="utf-8").splitlines():
+                line = raw.strip()
+                if not line or line.startswith("#") or "=" not in line:
+                    continue
+                key, value = line.split("=", 1)
+                value = value.strip().strip('"').strip("'")
+                os.environ.setdefault(key.strip(), value)
+            return str(candidate)
+    return None

cafe/cli.py

@@ -0,0 +1,323 @@
+"""cafe — a design-of-experiments evaluation engine for compound AI systems.
+
+Commands:
+  cafe run example            run the bundled toy study
+  cafe run path/to/study.py   run a study defined in a Python file
+  cafe validate [target]      expand the design and report its size (no execution)
+  cafe doctor                 check the environment (Python, R stats engine, LLM access)
+  cafe version                print the version
+
+Flags for `run`: --smoke (preflight), --reps, --concurrency,
+--checkpoint PATH (resumable), --out PATH (write results JSONL).
+
+The CLI is the headless/CI path; it shares the exact engine the library uses.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import importlib.util
+import statistics
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+from cafe import design, run_study
+from cafe.execution.results import Observation, Results, config_label
+from cafe.study import Study
+
+
+def _load_study(target: str) -> Study:
+    """Resolve a CLI target into a Study.
+
+    ``example`` -> the bundled toy study. Otherwise ``target`` is a path to a
+    .py file exposing a module-level ``study`` or a ``build_study()``/
+    ``build_example_study()`` callable.
+    """
+    if target == "example":
+        from cafe.examples import build_example_study
+
+        return build_example_study()
+
+    path = Path(target)
+    if not path.exists():
+        raise SystemExit(f"cafe: no such study target: {target!r}")
+
+    spec = importlib.util.spec_from_file_location("cafe_user_study", path)
+    if spec is None or spec.loader is None:
+        raise SystemExit(f"cafe: could not import {target!r}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    for attr in ("study", "STUDY"):
+        obj = getattr(module, attr, None)
+        if isinstance(obj, Study):
+            return obj
+    for attr in ("build_study", "build_example_study"):
+        fn = getattr(module, attr, None)
+        if callable(fn):
+            obj = fn()
+            if isinstance(obj, Study):
+                return obj
+    raise SystemExit(
+        f"cafe: {target!r} must define a `study` (Study) or a `build_study()` callable"
+    )
+
+
+def _progress(_obs: Observation, done: int, total: int) -> None:
+    bar_w = 24
+    filled = int(bar_w * done / total) if total else bar_w
+    bar = "█" * filled + "·" * (bar_w - filled)
+    sys.stderr.write(f"\r  running [{bar}] {done}/{total}")
+    sys.stderr.flush()
+    if done == total:
+        sys.stderr.write("\n")
+
+
+def _mean_meta(obs: list[Observation], key: str) -> float | None:
+    vals = [o.metadata[key] for o in obs if key in o.metadata and o.metadata[key] is not None]
+    return round(statistics.fmean(vals), 3) if vals else None
+
+
+def _print_results_table(results: Results) -> None:
+    by_config: dict[str, list[Observation]] = defaultdict(list)
+    for o in results.observations:
+        by_config[config_label(o.config)].append(o)
+
+    # Only show sim_q when some observation actually reports it (the toy example does;
+    # real Mode-A systems don't, and an always-"-" column is just clutter).
+    show_simq = any("sim_quality" in (o.metadata or {}) for o in results.observations)
+
+    rows: list[tuple[str, ...]] = []
+    header = ("configuration", "n", "err", "lat_s", "cost$") + (("sim_q",) if show_simq else ())
+    for label in sorted(by_config):
+        obs = by_config[label]
+        n_err = sum(1 for o in obs if not o.ok)
+        lat = _mean_meta(obs, "latency_s")
+        cost = _mean_meta(obs, "cost_usd")
+        row = (
+            label,
+            str(len(obs)),
+            str(n_err),
+            "-" if lat is None else f"{lat:.3f}",
+            "-" if cost is None else f"{cost:.4f}",
+        )
+        if show_simq:
+            simq = _mean_meta(obs, "sim_quality")
+            row += ("-" if simq is None else f"{simq:.3f}",)
+        rows.append(row)
+
+    widths = [max(len(header[i]), *(len(r[i]) for r in rows)) for i in range(len(header))]
+    fmt = "  ".join("{:<" + str(w) + "}" for w in widths)
+    print()
+    print("  " + fmt.format(*header))
+    print("  " + fmt.format(*("-" * w for w in widths)))
+    for r in rows:
+        print("  " + fmt.format(*r))
+    print()
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    study = _load_study(args.target)
+    n_inputs = 1 if args.smoke else max(1, len(study.dataset))
+    n_reps = 1 if args.smoke else (args.reps or study.replications)
+    n_cells = design.size(study) * n_inputs * n_reps
+    mode = "SMOKE (1 input, 1 rep)" if args.smoke else f"{n_reps} reps"
+    print(f"study   : {study.name}")
+    print(f"factors : {', '.join(f'{f.name}{f.levels}' for f in study.factors) or '(none)'}")
+    print(f"design  : {study.design} -> {design.size(study)} configs")
+    print(f"inputs  : {len(study.dataset)}   mode: {mode}")
+    print(f"cells   : {n_cells}")
+
+    results = asyncio.run(
+        run_study(
+            study,
+            replications=args.reps,
+            concurrency=args.concurrency,
+            checkpoint_path=args.checkpoint,
+            smoke=args.smoke,
+            on_progress=_progress,
+        )
+    )
+
+    s = results.summary()
+    _print_results_table(results)
+    print(
+        f"  done: {s['n_answers']} obs over {s['n_configs']} configs, "
+        f"{s['n_errors']} errors, {s['total_compute_s']}s compute"
+    )
+    if args.out:
+        results.to_jsonl(args.out)
+        print(f"  wrote {args.out}")
+    return 1 if s["n_errors"] else 0
+
+
+def _cmd_validate(args: argparse.Namespace) -> int:
+    study = _load_study(args.target)
+    configs = design.generate(study)
+    print(f"ok: study {study.name!r} is valid")
+    print(f"    design  : {study.design} -> {len(configs)} configurations")
+    print(f"    factors : {len(study.factors)}   inputs: {len(study.dataset)}")
+    for c in configs[: min(8, len(configs))]:
+        print(f"      - {config_label(c)}")
+    if len(configs) > 8:
+        print(f"      ... (+{len(configs) - 8} more)")
+    return 0
+
+
+def _cmd_doctor(args: argparse.Namespace) -> int:
+    """Check the CAFE environment: Python, the R statistics engine, and LLM access.
+
+    With ``--app``, also checks the web-app prerequisites (Docker + Docker Compose).
+    """
+    import os
+    import shutil
+    import subprocess
+
+    from cafe import __version__
+    from cafe.stats import check_glmer, check_r
+
+    def mark(ok: bool) -> str:
+        return "✓" if ok else "✗"
+
+    issues = 0
+    print("cafe doctor — environment check\n")
+
+    # --- Runtime -------------------------------------------------------------
+    py = sys.version_info
+    py_ok = py >= (3, 11)
+    if not py_ok:
+        issues += 1
+    print("Runtime")
+    print(f"  {mark(py_ok)}  Python {py.major}.{py.minor}.{py.micro}   (>= 3.11 required)")
+    print(f"  ✓  cafe {__version__}")
+    print()
+
+    # --- Statistics engine (R) -----------------------------------------------
+    print("Statistics engine  (required — the mixed-effects models run in R)")
+    if shutil.which("Rscript") is None:
+        print("  ✗  R not found")
+        print("       fix →  install R (e.g. `sudo apt install r-base` or `brew install r`),")
+        print("              then:  Rscript -e 'install.packages(c(\"ordinal\", \"lme4\"))'")
+        issues += 1
+    else:
+        try:
+            proc = subprocess.run(["Rscript", "-e", "cat(R.version.string)"],
+                                  capture_output=True, text=True, timeout=30)
+            ver = proc.stdout.strip() or "R (version unknown)"
+        except Exception:  # noqa: BLE001
+            ver = "R (version unknown)"
+        print(f"  ✓  {ver}   (Rscript on PATH)")
+        for pkg, (ok, _msg), label in (
+            ("ordinal", check_r(), "ordinal CLMM"),
+            ("lme4", check_glmer(), "binary logistic GLMM"),
+        ):
+            print(f"  {mark(ok)}  {pkg:<9}{label}")
+            if not ok:
+                print(f"       fix →  Rscript -e 'install.packages(\"{pkg}\")'")
+                issues += 1
+    print()
+
+    # --- LLM access ----------------------------------------------------------
+    print("LLM access  (required for real studies; the bundled example needs none)")
+    keys = ("OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY",
+            "GEMINI_API_KEY", "OLLAMA_API_KEY")
+    any_key = False
+    for k in keys:
+        present = bool(os.environ.get(k))
+        any_key = any_key or present
+        print(f"  {'✓' if present else '–'}  {k}{'' if present else '   not set'}")
+    if not any_key:
+        print("       set at least one to run studies with a live judge")
+    print()
+
+    # --- Containers (web app) — only with --app ------------------------------
+    if getattr(args, "app", False):
+        print("Containers  (for the self-hostable web app: `docker compose up`)")
+        if shutil.which("docker") is None:
+            print("  ✗  docker not found   — install Docker Engine / Docker Desktop")
+            issues += 1
+        else:
+            try:
+                proc = subprocess.run(["docker", "version", "--format", "{{.Server.Version}}"],
+                                      capture_output=True, text=True, timeout=10)
+                if proc.returncode == 0 and proc.stdout.strip():
+                    print(f"  ✓  docker {proc.stdout.strip()}   (daemon running)")
+                else:
+                    print("  ✗  docker installed but the daemon isn't responding   — start Docker")
+                    issues += 1
+            except Exception:  # noqa: BLE001
+                print("  ✗  docker present but not responding")
+                issues += 1
+            try:
+                proc = subprocess.run(["docker", "compose", "version", "--short"],
+                                      capture_output=True, text=True, timeout=10)
+                if proc.returncode == 0 and proc.stdout.strip():
+                    print(f"  ✓  docker compose {proc.stdout.strip()}")
+                else:
+                    print("  ✗  'docker compose' (v2) not available   — update Docker")
+                    issues += 1
+            except Exception:  # noqa: BLE001
+                print("  ✗  'docker compose' (v2) not available")
+                issues += 1
+        print()
+
+    # --- Summary -------------------------------------------------------------
+    if issues == 0:
+        print("All set — CAFE's statistics and execution layers are ready.")
+        return 0
+    word = "issue" if issues == 1 else "issues"
+    print(f"{issues} {word} — install the items marked ✗ above.")
+    return 1
+
+
+def _cmd_version(_args: argparse.Namespace) -> int:
+    from cafe import __version__
+
+    print(f"cafe {__version__}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    from cafe._env import load_env
+    load_env()  # load the nearest .env so `doctor` / `run` see the same keys a study does
+    parser = argparse.ArgumentParser(
+        prog="cafe", description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,  # keep the docstring's layout
+    )
+    parser.add_argument("--version", action="store_true", help="print version and exit")
+    sub = parser.add_subparsers(dest="command")
+
+    p_run = sub.add_parser("run", help="run a study (or `example`)")
+    p_run.add_argument("target", help="'example' or a path to a .py study file")
+    p_run.add_argument("--smoke", action="store_true", help="preflight: 1 input, 1 rep, no judging")
+    p_run.add_argument("--reps", type=int, default=None, help="replications per (config, input)")
+    p_run.add_argument("--concurrency", type=int, default=8, help="max cells in flight")
+    p_run.add_argument("--checkpoint", default=None, help="resumable checkpoint JSONL path")
+    p_run.add_argument("--out", default=None, help="write results to this JSONL path")
+    p_run.set_defaults(func=_cmd_run)
+
+    p_val = sub.add_parser("validate", help="expand a study's design without running it")
+    p_val.add_argument("target", nargs="?", default="example", help="'example' or a .py study file")
+    p_val.set_defaults(func=_cmd_validate)
+
+    p_doc = sub.add_parser("doctor", help="check the environment (Python, R stats engine, LLM access)")
+    p_doc.add_argument("--app", action="store_true",
+                       help="also check the web-app prerequisites (Docker + Docker Compose)")
+    p_doc.set_defaults(func=_cmd_doctor)
+
+    p_ver = sub.add_parser("version", help="print version and exit")
+    p_ver.set_defaults(func=_cmd_version)
+
+    args = parser.parse_args(argv)
+    if args.version:
+        return _cmd_version(args)
+    if not getattr(args, "command", None):
+        parser.print_help()
+        return 0
+    return args.func(args)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

cafe/datasets.py

@@ -0,0 +1,97 @@
+"""Load public benchmarks as CAFE datasets.
+
+Each loader returns a list of items shaped for ``Study(dataset=...)``:
+``{"id", "text", "reference", ...}`` — so the reference-guided judge has a gold
+answer to score against. Needs the ``datasets`` extra (HuggingFace ``datasets``):
+``pip install 'cafe-core[datasets]'``.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Any
+
+
+def _quiet_hf() -> None:
+    """Best-effort: silence HuggingFace's download chatter (the 'unauthenticated requests
+    to the HF Hub / set HF_TOKEN' warning, progress logs). Purely cosmetic for showcasing;
+    never let it break the actual load."""
+    import warnings
+
+    warnings.filterwarnings("ignore", message=r".*unauthenticated requests to the HF Hub.*")
+    for mod in ("huggingface_hub", "datasets"):
+        try:
+            __import__(mod).logging.set_verbosity_error()
+        except Exception:  # noqa: BLE001 — cosmetic only
+            pass
+    try:
+        import logging
+
+        logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
+    except Exception:  # noqa: BLE001
+        pass
+
+
+def _load_hf(name: str, *args: Any, **kwargs: Any):
+    try:
+        from datasets import load_dataset
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError(
+            "dataset loaders need the 'datasets' extra: pip install 'cafe-core[datasets]'"
+        ) from exc
+    _quiet_hf()
+    return load_dataset(name, *args, **kwargs)
+
+
+def _sample(rows: list[Any], n: int | None, *, shuffle: bool, seed: int) -> list[Any]:
+    rows = list(rows)
+    if shuffle:
+        random.Random(seed).shuffle(rows)
+    return rows if n is None else rows[:n]
+
+
+def load_truthfulqa(
+    n: int | None = 20,
+    *,
+    split: str = "train",
+    categories: list[str] | None = None,
+    shuffle: bool = True,
+    seed: int = 0,
+) -> list[dict[str, Any]]:
+    """TruthfulQA — adversarial questions that elicit common misconceptions.
+
+    Reference-based: each item carries the gold ``Best Answer``. Returns items
+    ``{id, text, reference, category}``. ``categories`` filters by Category
+    (e.g. ``["Misconceptions", "Law"]``).
+    """
+    ds = _load_hf("domenicrosati/TruthfulQA", split=split)
+    rows = list(ds)
+    if categories:
+        wanted = {c.lower() for c in categories}
+        rows = [r for r in rows if str(r.get("Category", "")).lower() in wanted]
+    rows = _sample(rows, n, shuffle=shuffle, seed=seed)
+    return [
+        {
+            "id": f"tq{i}",
+            "text": r["Question"],
+            "reference": r["Best Answer"],
+            "category": r.get("Category"),
+        }
+        for i, r in enumerate(rows)
+    ]
+
+
+def load_gsm8k(
+    n: int | None = 20,
+    *,
+    split: str = "test",
+    shuffle: bool = True,
+    seed: int = 0,
+) -> list[dict[str, Any]]:
+    """GSM8K — grade-school math word problems with an exact numeric reference."""
+    ds = _load_hf("openai/gsm8k", "main", split=split)
+    rows = _sample(list(ds), n, shuffle=shuffle, seed=seed)
+    return [
+        {"id": f"gsm{i}", "text": r["question"], "reference": r["answer"].split("####")[-1].strip()}
+        for i, r in enumerate(rows)
+    ]

cafe/design/__init__.py

@@ -0,0 +1,19 @@
+"""Design of experiments: turn factors into the configurations to run."""
+
+from cafe.design.factorial import Config, full_factorial, generate, single, size
+from cafe.design.fractional import (
+    FractionalDesign,
+    fractional_factorial,
+    fractional_factorial_design,
+)
+
+__all__ = [
+    "Config",
+    "generate",
+    "size",
+    "full_factorial",
+    "single",
+    "fractional_factorial",
+    "fractional_factorial_design",
+    "FractionalDesign",
+]