evalstone 0.1.0__py3-none-any.whl

evalstone/__init__.py

@@ -0,0 +1,131 @@
+"""Evalstone — the plug-and-play evals layer for LLM-based projects.
+
+The library computes; it owns no database. See https://evalstone.com/docs.
+"""
+
+import sys
+import types
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+__version__ = "0.1.0"
+
+# Exports resolve on first access (PEP 562) so that importing the package — which
+# the pytest plugin does in every pytest session on the interpreter — costs
+# nothing until a name is used.
+_EXPORTS: dict[str, str] = {
+    "check": "evalstone.checks",
+    "registry": "evalstone.checks",
+    "Dataset": "evalstone.dataset",
+    "Sample": "evalstone.dataset",
+    "Source": "evalstone.dataset",
+    "Gate": "evalstone.gate",
+    "GateFailed": "evalstone.gate",
+    "GateReport": "evalstone.gate",
+    "Output": "evalstone.output",
+    "Providers": "evalstone.providers",
+    "Comparison": "evalstone.run",
+    "Result": "evalstone.run",
+    "Run": "evalstone.run",
+    "RunSummary": "evalstone.run",
+    "compare": "evalstone.run",
+    "write_report": "evalstone.run",
+    "Check": "evalstone.score",
+    "CheckKind": "evalstone.score",
+    "Context": "evalstone.score",
+    "MemoryCache": "evalstone.score",
+    "Score": "evalstone.score",
+    "Suite": "evalstone.suite",
+    "CallableTarget": "evalstone.target",
+    "HttpTarget": "evalstone.target",
+    "PromptTarget": "evalstone.target",
+    "RecordedTarget": "evalstone.target",
+    "Target": "evalstone.target",
+    "target": "evalstone.target",
+}
+
+if TYPE_CHECKING:
+    from evalstone import checks
+    from evalstone.checks import check, registry
+    from evalstone.dataset import Dataset, Sample, Source
+    from evalstone.gate import Gate, GateFailed, GateReport
+    from evalstone.output import Output
+    from evalstone.providers import Providers
+    from evalstone.run import Comparison, Result, Run, RunSummary, compare, write_report
+    from evalstone.score import Check, CheckKind, Context, MemoryCache, Score
+    from evalstone.suite import Suite
+    from evalstone.target import (
+        CallableTarget,
+        HttpTarget,
+        PromptTarget,
+        RecordedTarget,
+        Target,
+        target,
+    )
+
+
+class _Package(types.ModuleType):
+    """``evalstone.target`` is the factory function, as it always was, even though a
+    submodule of the same name exists: importing that submodule makes the import
+    system bind it onto the package, which the setter below ignores."""
+
+    @property
+    def target(self) -> Any:
+        return import_module("evalstone.target").target
+
+    @target.setter
+    def target(self, value: Any) -> None:
+        pass
+
+
+sys.modules[__name__].__class__ = _Package
+
+
+def __getattr__(name: str) -> Any:
+    if name == "checks":
+        return import_module("evalstone.checks")
+    try:
+        module = _EXPORTS[name]
+    except KeyError:
+        raise AttributeError(f"module 'evalstone' has no attribute {name!r}") from None
+    value = getattr(import_module(module), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted(set(__all__) | {n for n in globals() if n.startswith("__")})
+
+
+__all__ = [
+    "CallableTarget",
+    "Check",
+    "CheckKind",
+    "Comparison",
+    "Context",
+    "Dataset",
+    "Gate",
+    "GateFailed",
+    "GateReport",
+    "HttpTarget",
+    "MemoryCache",
+    "Output",
+    "PromptTarget",
+    "Providers",
+    "RecordedTarget",
+    "Result",
+    "Run",
+    "RunSummary",
+    "Sample",
+    "Score",
+    "Source",
+    "Suite",
+    "Target",
+    "__version__",
+    "check",
+    "checks",
+    "compare",
+    "registry",
+    "target",
+    "write_report",
+]

evalstone/__main__.py

@@ -0,0 +1,3 @@
+from evalstone.cli import main
+
+main()

evalstone/_git.py

@@ -0,0 +1,22 @@
+"""Best-effort git facts for run provenance. Never raises."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+def head_sha(cwd: str | Path | None = None) -> str | None:
+    try:
+        out = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            cwd=cwd,
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    sha = out.stdout.strip()
+    return sha if out.returncode == 0 and sha else None

evalstone/_json.py

@@ -0,0 +1,59 @@
+"""Canonical JSON helpers: one stable serialisation so content hashes are reproducible.
+
+Only JSON-native values plus a few explicitly handled types (sets, tuples,
+bytes, dates, enums, pydantic models, dataclasses) are accepted. Anything else
+raises ``TypeError`` instead of being stringified — ``str(obj)`` would bake a
+memory address or a hash-randomised set order into a content hash.
+"""
+
+from __future__ import annotations
+
+import base64
+import dataclasses
+import datetime as dt
+import enum
+import hashlib
+import json
+from typing import Any
+
+
+def _encode(value: Any) -> Any:
+    """Convert the supported non-JSON types to a deterministic JSON-native form."""
+    if value is None or isinstance(value, str | int | float | bool):
+        return value
+    if isinstance(value, list):
+        return [_encode(v) for v in value]
+    if isinstance(value, dict):
+        if any(not isinstance(k, str) for k in value):
+            raise TypeError("dict keys must be strings to hash")
+        return {k: _encode(v) for k, v in value.items()}
+    if isinstance(value, enum.Enum):
+        return value.value
+    if isinstance(value, set | frozenset):
+        return sorted((_encode(v) for v in value), key=lambda v: json.dumps(v, sort_keys=True))
+    if isinstance(value, tuple):
+        return [_encode(v) for v in value]
+    if isinstance(value, bytes | bytearray):
+        return {"__bytes__": base64.b64encode(bytes(value)).decode("ascii")}
+    if isinstance(value, dt.datetime | dt.date | dt.time):
+        return value.isoformat()
+    if hasattr(value, "model_dump") and callable(value.model_dump):  # pydantic
+        return value.model_dump(mode="json")
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return dataclasses.asdict(value)
+    raise TypeError(f"not JSON-serialisable for hashing/text: {type(value).__name__}")
+
+
+def dumps(value: Any, **kwargs: Any) -> str:
+    """``json.dumps`` with the explicit encoder above (never ``default=str``)."""
+    return json.dumps(_encode(value), ensure_ascii=False, **kwargs)
+
+
+def canonical(value: Any) -> str:
+    """Serialise ``value`` deterministically (sorted keys, no whitespace, UTF-8 kept)."""
+    return dumps(_encode(value), sort_keys=True, separators=(",", ":"))
+
+
+def sha256(value: Any) -> str:
+    """sha256 hex digest of the canonical JSON form of ``value``."""
+    return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()

evalstone/checks/__init__.py

@@ -0,0 +1,41 @@
+"""Checks: the registry, the ``@check`` decorator and the built-in factories.
+
+Built-in checks and user-defined checks have the same shape — see ``_base``.
+"""
+
+from evalstone.checks import comparison, rag, safety, trajectory
+from evalstone.checks._base import CheckFn, FunctionCheck, Registry, check, registry
+from evalstone.checks.comparison import pairwise
+from evalstone.checks.deterministic import (
+    contains,
+    exact,
+    json_schema,
+    length,
+    numeric_tolerance,
+    python,
+    regex,
+)
+from evalstone.checks.human import human
+from evalstone.checks.judge import judge
+
+__all__ = [
+    "CheckFn",
+    "FunctionCheck",
+    "Registry",
+    "check",
+    "comparison",
+    "contains",
+    "exact",
+    "human",
+    "json_schema",
+    "judge",
+    "length",
+    "numeric_tolerance",
+    "pairwise",
+    "python",
+    "rag",
+    "regex",
+    "registry",
+    "safety",
+    "trajectory",
+]

evalstone/checks/_base.py

@@ -0,0 +1,153 @@
+"""The check registry, ``FunctionCheck`` and the ``@check`` decorator.
+
+Built-in checks and user-defined checks have the same shape. A decorated
+function may return a ``Score``, a ``bool`` or a ``float`` in 0..1; the
+decorator normalises it. Any exception raised inside a check becomes
+``Score.errored(...)`` — a crashing check never passes and never takes the
+run down.
+"""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import time
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from evalstone.score import Check, CheckKind, Context, Score
+
+if TYPE_CHECKING:
+    from evalstone.dataset import Sample
+    from evalstone.output import Output
+
+CheckFn = Callable[..., "Score | bool | float"]
+
+
+class Registry:
+    """Checks by key. Registering the same key twice is an error unless ``replace=True``."""
+
+    def __init__(self) -> None:
+        self._checks: dict[str, Check] = {}
+        self.generation = 0  # bumped by clear(); project modules re-import when it moves
+
+    def register(self, chk: Check, *, replace: bool = False) -> Check:
+        """Register ``chk`` under its key.
+
+        A second registration of the same key is an error, except when it is the
+        same function re-imported from a fresh module object (``evalstone.yaml``
+        project reloads, notebook re-runs): same ``__qualname__`` but different
+        module globals. Two distinct functions from one module — closures from a
+        factory, lambdas — share globals and are still rejected.
+        """
+        previous = self._checks.get(chk.key)
+        if previous is not None and not replace and not _reimported(previous, chk):
+            raise ValueError(f"check {chk.key!r} is already registered")
+        self._checks[chk.key] = chk
+        return chk
+
+    def get(self, key: str) -> Check:
+        try:
+            return self._checks[key]
+        except KeyError:
+            raise KeyError(f"unknown check {key!r}; registered: {sorted(self._checks)}") from None
+
+    def __contains__(self, key: str) -> bool:
+        return key in self._checks
+
+    def keys(self) -> list[str]:
+        return sorted(self._checks)
+
+    def clear(self) -> None:
+        self._checks.clear()
+        self.generation += 1
+
+
+def _reimported(previous: Check, current: Check) -> bool:
+    pf, cf = getattr(previous, "_fn", None), getattr(current, "_fn", None)
+    if pf is None or cf is None:
+        return False
+    same_name = (pf.__module__, pf.__qualname__) == (cf.__module__, cf.__qualname__)
+    fresh_module = getattr(pf, "__globals__", None) is not getattr(cf, "__globals__", None)
+    return same_name and fresh_module
+
+
+registry = Registry()
+
+
+def _normalise(result: Any) -> Score:
+    if isinstance(result, Score):
+        return result
+    if isinstance(result, bool):
+        return Score.from_bool(result)
+    if isinstance(result, int | float):
+        return Score.from_value(float(result))
+    raise TypeError(f"check must return Score, bool or float, got {type(result).__name__}")
+
+
+class FunctionCheck:
+    """A plain function promoted to a ``Check``."""
+
+    def __init__(
+        self,
+        key: str,
+        fn: CheckFn,
+        *,
+        kind: CheckKind = CheckKind.DETERMINISTIC,
+        version: int = 1,
+        description: str = "",
+    ) -> None:
+        self.key = key
+        self.kind = kind
+        self.version = version
+        self.description = description or (inspect.getdoc(fn) or "")
+        self._fn = fn
+        self._wants_ctx = "ctx" in inspect.signature(fn).parameters
+        functools.update_wrapper(self, fn)
+
+    def evaluate(self, sample: Sample, output: Output, ctx: Context) -> Score:
+        started = time.perf_counter()
+        try:
+            if self._wants_ctx:
+                result = self._fn(sample, output, ctx=ctx)
+            else:
+                result = self._fn(sample, output)
+            score = _normalise(result)
+        except Exception as exc:  # a crashing check is an errored score, never a pass
+            score = Score.errored(f"{type(exc).__name__}: {exc}")
+        elapsed = (time.perf_counter() - started) * 1000
+        if score.latency_ms == 0.0:
+            score = score.model_copy(update={"latency_ms": elapsed})
+        return score
+
+    def __call__(self, sample: Sample, output: Output, ctx: Context | None = None) -> Score:
+        return self.evaluate(sample, output, ctx or Context())
+
+    def __repr__(self) -> str:
+        return f"<check {self.key} kind={self.kind} v{self.version}>"
+
+
+def check(
+    key: str,
+    *,
+    kind: CheckKind | str = CheckKind.DETERMINISTIC,
+    version: int = 1,
+    register: bool = True,
+) -> Callable[[CheckFn], FunctionCheck]:
+    """Turn ``fn(sample, output[, ctx]) -> Score | bool | float`` into a registered check.
+
+    >>> @check("no_apology")
+    ... def no_apology(sample, output):
+    ...     return "sorry" not in output.text.lower()
+    """
+
+    def decorator(fn: CheckFn) -> FunctionCheck:
+        chk = FunctionCheck(key, fn, kind=CheckKind(kind), version=version)
+        if register:
+            registry.register(chk)
+        return chk
+
+    return decorator
+
+
+__all__ = ["CheckFn", "FunctionCheck", "Registry", "check", "registry"]

evalstone/checks/comparison.py

@@ -0,0 +1,154 @@
+"""Pairwise comparison: is the candidate output better than a baseline's?
+
+Judges are far more consistent at *preferring* than at scoring, and they prefer
+whichever answer comes first unless you shuffle — so by default both orders are
+judged and the verdicts combined (Evals 101, chapters 4 and 6).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from evalstone._json import sha256
+from evalstone.checks._base import FunctionCheck
+from evalstone.checks.judge import _json_text, extract_json
+from evalstone.dataset import Sample
+from evalstone.output import Output
+from evalstone.providers import ChatMessage, ProviderError
+from evalstone.score import CheckKind, Context, Score
+from evalstone.target import RecordedTarget, Target
+
+SYSTEM = """You are comparing two answers to the same INPUT.
+
+CRITERIA:
+{rubric}
+
+Decide which answer better satisfies the criteria. Ignore the order they are shown in,
+their length, and formatting. Reply with ONLY a JSON object:
+{{"winner": "A" | "B" | "tie", "reason": "<one sentence>"}}"""
+
+USER = """INPUT:
+{input}
+
+ANSWER A:
+{a}
+
+ANSWER B:
+{b}"""
+
+
+def parse_winner(text: str) -> tuple[str, str]:
+    data = extract_json(text)
+    winner = str(data.get("winner", "")).strip().upper()
+    if winner not in {"A", "B", "TIE"}:
+        raise ValueError(f"judge winner must be A, B or tie, got {winner!r}")
+    return winner, str(data.get("reason", "")).strip()
+
+
+def pairwise(
+    key: str,
+    rubric: str,
+    *,
+    baseline: Target | Mapping[str, Any],
+    model: str | None = None,
+    both_orders: bool = True,
+    max_tokens: int = 256,
+) -> FunctionCheck:
+    """The candidate output beats the ``baseline`` output for the same sample.
+
+    ``baseline`` is a Target (its outputs are produced per sample, cost counted) or a
+    mapping of sample id → output. Each judged order scores 1 (candidate preferred),
+    0.5 (tie) or 0 (baseline preferred); the value is their mean, so with both orders
+    it is one of 1.0 / 0.75 / 0.5 / 0.25 / 0.0. Passes at 0.5 or above (wins or ties).
+    """
+    base_target: Target = baseline if isinstance(baseline, Target) else RecordedTarget(baseline)
+    identity = {"v": 1, "rubric": rubric, "both_orders": both_orders}
+
+    def ask(ctx: Context, sample_input: str, a: str, b: str) -> tuple[str, str, float, float]:
+        user = USER.format(input=sample_input, a=a, b=b)
+        resp = ctx.providers.complete(
+            model,
+            {
+                "system": SYSTEM.format(rubric=rubric.strip()),
+                "messages": [ChatMessage(role="user", content=user)],
+                "max_tokens": max_tokens,
+            },
+        )
+        winner, reason = parse_winner(resp.text)
+        return winner, reason, resp.cost_usd or 0.0, resp.latency_ms
+
+    def fn(sample: Sample, output: Output, ctx: Context) -> Score:
+        if ctx.providers is None:
+            return Score.errored("pairwise: no providers configured")
+        try:
+            resolved = "/".join(ctx.providers.split(model))
+        except ProviderError as exc:
+            return Score.errored(str(exc))
+        version = sha256({**identity, "model": resolved})[:16]
+        base = base_target.produce(sample)
+        if base.error:
+            return Score.errored(f"pairwise: baseline: {base.error}")
+        text = _json_text(sample.input)
+        cache_key = sha256({"p": version, "i": text, "c": output.text, "b": base.text})
+        cached = ctx.cache.get(cache_key) if ctx.cache is not None else None
+        if cached is not None:
+            # The judge verdict is cached; the baseline was still produced (its cost counts).
+            raw = {**(cached.raw or {}), "cached": True}
+            return cached.model_copy(
+                update={"cost_usd": base.cost_usd, "latency_ms": base.latency_ms, "raw": raw}
+            )
+        cost = base.cost_usd
+        latency = 0.0
+        points: list[float] = []
+        reasons: list[str] = []
+        try:
+            orders = [("cand_first", output.text, base.text)]
+            if both_orders:
+                orders.append(("base_first", base.text, output.text))
+            for label, a, b in orders:
+                winner, reason, c, ms = ask(ctx, text, a, b)
+                cost += c
+                latency += ms
+                candidate_is_a = label == "cand_first"
+                if winner == "TIE":
+                    points.append(0.5)
+                elif (winner == "A") == candidate_is_a:
+                    points.append(1.0)
+                else:
+                    points.append(0.0)
+                reasons.append(reason)
+        except ProviderError as exc:
+            return Score.errored(str(exc), cost_usd=cost, latency_ms=latency)
+        except ValueError as exc:
+            return Score.errored(f"pairwise: {exc}", cost_usd=cost, latency_ms=latency)
+        value = sum(points) / len(points)
+        verdict = _label(value)
+        score = Score(
+            value=value,
+            passed=value >= 0.5,
+            reason=f"candidate {verdict}: " + " | ".join(r for r in reasons if r),
+            cost_usd=cost,
+            latency_ms=latency,
+            raw={"points": points, "orders": len(points), "judge_version": version},
+        )
+        if ctx.cache is not None:
+            ctx.cache.set(cache_key, score)
+        return score
+
+    return FunctionCheck(key, fn, kind=CheckKind.COMPARISON)
+
+
+def _label(value: float) -> str:
+    if value == 1.0:
+        return "wins"
+    if value > 0.5:
+        return "mostly wins"
+    if value == 0.5:
+        return "ties/split"
+    if value > 0.0:
+        return "mostly loses"
+    return "loses"
+
+
+__all__ = ["pairwise", "parse_winner"]