oryxeval 26.11.6__py3-none-any.whl

oryxeval/__init__.py

@@ -0,0 +1,107 @@
+"""LLM evaluation harness: one cached cell per arm of an eval matrix.
+
+Install it with::
+
+    pip install oryxeval
+
+It is a separate package from oryxflow, which it builds on. Import it explicitly::
+
+    import oryxeval as ev
+
+SEAM: the per-case DataFrame is the interface. Everything downstream of a run --
+``Metric``, the slices, the intervals, the verdict, the report -- reads only that
+frame and never touches a pydantic-evals object. The two override points on
+``TaskEval`` are the exit: ``_evaluate()`` returns a report, so swap it for a
+different runner, and ``_to_frame(report)`` returns the rows, so swap it for a
+different report shape. Nothing else has to change. The property that buys is
+available today rather than later: every metric, verdict and report test is
+written against a hand-built DataFrame, so everything downstream of a run is
+testable without pydantic-evals present.
+
+``cli`` and ``build_app`` are resolved on first use rather than at import, so
+``import oryxeval`` does not pay for Typer -- a sweep run from a notebook
+never needs a command line.
+"""
+
+import importlib
+
+from importlib.metadata import PackageNotFoundError, version as _version
+
+try:
+    __version__ = _version("oryxeval")
+except PackageNotFoundError:
+    __version__ = "unknown"
+
+from oryxflow import (Parameter, IntParameter, FloatParameter, BoolParameter,
+                      ChoiceParameter, ListParameter, DictParameter, DateParameter,
+                      EnumParameter, hash_files)
+
+from oryxeval.baseline import (
+    BaselineError,
+    PromptArm,
+    Variant,
+    VariantError,
+    git_sha,
+    git_tree,
+)
+from oryxeval.cases import RESERVED_METADATA, CaseList, load_cases
+from oryxeval.metric import Metric
+from oryxeval.stats import (DeltaCI, JudgeAlignment, RateCI, cluster_labels,
+                            corrected_rate, delta_ci, judge_alignment,
+                            rate_ci)
+from oryxeval.sweep import EvalResult, sweep
+from oryxeval.task import EvalFailureRateError, TaskEval
+
+__all__ = [
+    '__version__',
+    'BoolParameter',
+    'ChoiceParameter',
+    'DateParameter',
+    'DictParameter',
+    'EnumParameter',
+    'FloatParameter',
+    'IntParameter',
+    'ListParameter',
+    'Parameter',
+    'hash_files',
+    'BaselineError',
+    'CaseList',
+    'DeltaCI',
+    'EvalFailureRateError',
+    'EvalResult',
+    'JudgeAlignment',
+    'Metric',
+    'PromptArm',
+    'RESERVED_METADATA',
+    'RateCI',
+    'TaskEval',
+    'Variant',
+    'VariantError',
+    'build_app',
+    'cli',
+    'cluster_labels',
+    'corrected_rate',
+    'delta_ci',
+    'git_sha',
+    'git_tree',
+    'judge_alignment',
+    'load_cases',
+    'rate_ci',
+    'sweep',
+]
+
+_LAZY = ('cli', 'build_app')
+
+
+def __getattr__(name):
+    """Import the Typer-backed names only when one of them is actually used."""
+    if name in _LAZY:
+        module = importlib.import_module('oryxeval.cli')
+        value = module if name == 'cli' else getattr(module, name)
+        globals()[name] = value
+        return value
+    raise AttributeError("module {!r} has no attribute {!r}".format(__name__, name))
+
+
+def __dir__():
+    return sorted(set(list(globals()) + list(__all__)))

oryxeval/baseline.py

@@ -0,0 +1,260 @@
+"""The two ways to build a comparison arm, deliberately distinct.
+
+A baseline is *something that existed*, so it is checked out of git: byte-exact by
+construction, immune to the live files moving under it, and -- the reason this module
+does not simply ship a string-replacement helper -- able to restore a section that was
+DELETED. A probe is a rewrite that has not shipped anywhere, so there is no ref to check
+out and an anchored ``(old -> new)`` replacement is all there is.
+
+Both fail loudly. A comparison arm that silently becomes a no-op reports the old version
+as having scored well, which is indistinguishable from a real result and therefore the
+worst failure a comparison can have.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class BaselineError(RuntimeError):
+    """A baseline arm could not be built, or was built empty.
+
+    A ``RuntimeError``, because an empty or wrong baseline is not a recoverable
+    condition: whatever the arm went on to measure would not be the old version.
+    """
+
+
+class VariantError(ValueError):
+    """A variant's anchor text was not found in the rendered text.
+
+    A ``ValueError``, because the arguments no longer describe the text they are
+    applied to -- the anchor has been edited away, and applying the variant anyway
+    would report the unchanged prompt's numbers as the variant's.
+    """
+
+
+def _git(args, repo=None, binary=False):
+    """Run a git command, returning stdout (bytes when ``binary``). Raise on failure."""
+    cwd = str(repo) if repo is not None else os.getcwd()
+    proc = subprocess.run(['git'] + list(args), cwd=cwd,
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if proc.returncode != 0:
+        raise BaselineError("git {} failed in {}: {}".format(
+            ' '.join(args), cwd, proc.stderr.decode('utf-8', 'replace').strip()))
+    return proc.stdout if binary else proc.stdout.decode('utf-8', 'replace')
+
+
+def git_sha(ref, repo=None):
+    """Resolve ``ref`` to its full commit SHA.
+
+    An arm built from a git ref must report *this* as its ``code_version()``, never the
+    ref string: ``HEAD~1`` names different content every time something is committed, and
+    a cache key that moves under a stable-looking name reads stale numbers as fresh ones.
+    A resolved SHA is immutable, so the arm's cache key is stable forever.
+    """
+    try:
+        out = _git(['rev-parse', '--verify', '{}^{{commit}}'.format(ref)], repo=repo)
+    except BaselineError:
+        out = _git(['rev-parse', '--verify', str(ref)], repo=repo)
+    sha = out.strip()
+    if not sha:
+        raise BaselineError("ref '{}' did not resolve to a commit".format(ref))
+    return sha
+
+
+def _as_list(paths):
+    if paths is None:
+        return []
+    if isinstance(paths, (str, bytes, os.PathLike)):
+        return [paths]
+    return list(paths)
+
+
+def _ls_tree(ref, paths, repo=None):
+    """Filenames under ``paths`` at ``ref``, recursively, git-style forward slashes."""
+    args = ['ls-tree', '-r', '-z', '--name-only', str(ref), '--']
+    args += [str(p).replace('\\', '/') for p in paths]
+    out = _git(args, repo=repo, binary=True).decode('utf-8', 'surrogateescape')
+    return [n for n in out.split('\0') if n]
+
+
+def _default_dest(sha, repo=None):
+    base = Path(repo) if repo is not None else Path.cwd()
+    root = base / '.oryxflow' / 'baseline'
+    root.mkdir(parents=True, exist_ok=True)
+    # The materialized tree is generated, never source: ignore it wherever it lands,
+    # so a user whose repo predates this never has to notice it.
+    ignore = root / '.gitignore'
+    if not ignore.exists():
+        ignore.write_text('*\n')
+    return root / sha[:12]
+
+
+def git_tree(ref, paths, dest=None, expect_absent=None, repo=None):
+    """Materialize files from a git ref into a directory, for a BASELINE arm.
+
+    Use this for a prompt/template tree as it was before a change: it is
+    byte-exact by construction, cannot decay as the live files move, and -- the
+    reason it exists -- it restores DELETED files, which a string-replacement
+    variant cannot do at all.
+
+    ``ref`` is anything git resolves ('abc123^', 'v2.1', 'main~3'). Because a
+    resolved ref is immutable, an arm built from one has a cache key that is
+    stable forever.
+
+    Raises RuntimeError when the ref yields no files under ``paths`` -- a
+    silently empty baseline reads as "the old version already scored well",
+    which is the worst possible failure for a comparison.
+
+    ``expect_absent=`` names files that must NOT exist at this ref; their
+    presence means the ref is wrong and the arm is measuring nothing.
+
+    Returns the directory the files were written to. Point your own loader at it --
+    how your templates are loaded is yours to know, so nothing here is patched or
+    installed on your behalf. The default location is keyed by the resolved SHA,
+    beside the eval and gitignored, so a ref is materialized once and reused.
+    Pass ``repo=`` to run git somewhere other than the working directory, and use
+    ``git_sha(ref)`` for the arm's ``code_version()``.
+    """
+    paths = _as_list(paths)
+    if not paths:
+        raise BaselineError(
+            "git_tree(ref={!r}) needs at least one path; materializing nothing would "
+            "give an empty baseline".format(ref))
+    sha = git_sha(ref, repo=repo)
+
+    for name in _as_list(expect_absent):
+        found = _ls_tree(sha, [name], repo=repo)
+        if found:
+            raise BaselineError(
+                "ref '{}' ({}) still has {} -- expect_absent said it should not exist "
+                "there, so this is not the ref you meant and the arm would measure "
+                "nothing.".format(ref, sha[:10], ', '.join(sorted(found))))
+
+    names = _ls_tree(sha, paths, repo=repo)
+    if not names:
+        raise BaselineError(
+            "ref '{}' ({}) has no files under {} -- an empty baseline reads as 'the old "
+            "version already scored well', so it is refused rather than measured.".format(
+                ref, sha[:10], ', '.join(repr(str(p)) for p in paths)))
+
+    dest = Path(dest) if dest is not None else _default_dest(sha, repo=repo)
+    dest.mkdir(parents=True, exist_ok=True)
+    for name in names:
+        target = dest.joinpath(*name.split('/'))
+        blob = _git(['show', '{}:{}'.format(sha, name)], repo=repo, binary=True)
+        if target.exists() and target.read_bytes() == blob:
+            continue
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(blob)
+    manifest = dest / '_oryxflow_baseline.json'
+    manifest.write_text(json.dumps(
+        {'ref': str(ref), 'sha': sha, 'paths': [str(p) for p in paths], 'files': names},
+        indent=2, sort_keys=True))
+    return dest
+
+
+@dataclass
+class Variant:
+    """An exact (old -> new) replacement applied to already-rendered text, for a
+    PROBE arm -- a rewrite that has not shipped, so there is no ref to check out.
+
+    Raises when ``old`` is not present: a variant whose anchor text has been
+    edited away silently becomes a no-op, and then reports the unchanged prompt's
+    numbers as the variant's.
+
+    NOT for baselines. Use ``git_tree`` there: a replacement cannot restore a
+    deleted section, and cannot put one back in its original position.
+    """
+    old: str
+    new: str
+
+    def apply(self, text):
+        """Return ``text`` with ``old`` replaced by ``new``. Raise ``VariantError``
+        (a ``ValueError``) when ``old`` is absent, rather than returning the text
+        unchanged."""
+        if self.old not in text:
+            raise VariantError(
+                "variant anchor not found in the rendered text (anchor {!r}...): the "
+                "text it was written against has changed, so this variant would be a "
+                "no-op and its arm would report the unchanged prompt's numbers.".format(
+                    self.old[:80]))
+        return text.replace(self.old, self.new)
+
+    def __call__(self, text):
+        return self.apply(text)
+
+    def code_version(self):
+        """A token that changes when this replacement changes, for an arm's cache key."""
+        h = hashlib.md5()
+        h.update(self.old.encode('utf-8'))
+        h.update(b'\0')
+        h.update(self.new.encode('utf-8'))
+        return 'variant-' + h.hexdigest()[:10]
+
+
+@dataclass
+class PromptArm:
+    """Where one arm reads its prompt files, AND the cache key that matches them.
+
+    The two have to agree, and nothing else makes them. Written by hand it is two
+    places -- a ``prompt_root(arm)`` that returns a directory and a
+    ``code_version()`` that returns a token -- and the failure when they drift is
+    silent and bad: a baseline arm reading a checked-out tree but keyed on a hash
+    of the LIVE files re-runs when a live prompt is edited (merely wasteful) and
+    does NOT re-run when you point it at a different ref (stale, and reported as
+    fresh). One object owns both, so it cannot happen.
+
+        ARMS = {
+            'live':    ev.PromptArm(globs=['prompts/*.jinja2'], root=ROOT),
+            'baseline': ev.PromptArm(ref='v2.1', paths=['prompts/'], root=ROOT),
+        }
+
+        # in the agent:  ARMS[arm].dir()
+        # in the task:   def code_version(self): return ARMS[self.arm].code_version()
+
+    ``ref=None`` means the working tree, keyed on the CONTENT of ``globs``. A
+    ``ref`` means that commit, materialized with :py:func:`git_tree` and keyed on
+    the resolved sha -- immutable, so a baseline arm's cell stays valid for good.
+
+    ``subdir`` is applied to both, so the two arms hand the loader the same
+    layout and it never has to know which one it got.
+    """
+    ref: str = None
+    paths: tuple = ()
+    globs: tuple = ()
+    root: str = None
+    subdir: str = ''
+    expect_absent: tuple = ()
+    dest: str = None
+
+    def __post_init__(self):
+        if self.ref is None and not self.globs:
+            raise ValueError(
+                'PromptArm: the working-tree arm needs globs= (the files whose CONTENT '
+                'is its cache key) -- without them the arm is keyed on nothing and an '
+                'edited prompt is served from cache')
+        if self.ref is not None and not self.paths:
+            raise ValueError(
+                'PromptArm: a ref arm needs paths= (what to materialize from {!r})'.format(
+                    self.ref))
+
+    def dir(self):
+        """The directory this arm reads its prompts from."""
+        base = Path(self.root) if self.root is not None else Path.cwd()
+        if self.ref is not None:
+            base = git_tree(self.ref, self.paths, dest=self.dest,
+                            expect_absent=self.expect_absent, repo=self.root)
+        return base.joinpath(*self.subdir.split('/')) if self.subdir else base
+
+    def code_version(self):
+        """The cache token for the files :py:meth:`dir` will hand out."""
+        if self.ref is not None:
+            return git_sha(self.ref, repo=self.root)
+        from oryxflow.utils import hash_files
+        return hash_files(*self.globs, root=self.root)

oryxeval/cases.py

@@ -0,0 +1,267 @@
+"""Eval cases as a file a domain expert can edit, not constructors in Python.
+
+A case set buried in ``Case(...)`` calls stops being reviewable as a diff, cannot be
+edited by the person who actually has the examples, and turns "add fifty more" into code
+generation. This module reads the same cases from a spreadsheet, a YAML file or a JSONL
+file, and routes the columns by name with nothing to configure.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+
+import yaml
+from pydantic_evals import Case
+
+
+# Reserved metadata column names. Mostly documented rather than enforced -- a user may
+# have their own meaning for them, and the cost of being wrong is a slice column with a
+# surprising name, not a wrong number. Two exceptions: ``holdout`` is acted on, and
+# ``arm`` is REFUSED by ``sweep`` (it is the label sweep gives each cell, so a metadata
+# column of that name would be shadowed and any slice on it would silently report the
+# arm label instead).
+RESERVED_METADATA = ('synthetic', 'holdout', 'arm', 'expect_*')
+
+_BOOL_TRUE = ('true', 'yes')
+_BOOL_FALSE = ('false', 'no')
+_TRUTHY = ('true', 'yes', '1')
+
+_EXPECTED = 'expected'
+_EXPECTED_PREFIX = 'expected.'
+
+
+class CaseList(list):
+    """The list of cases, plus how many rows were withheld.
+
+    ``excluded`` counts the ``holdout`` rows that were dropped, so a report can state
+    the number. A case whose text is embedded in the prompt under test is scoring
+    against its own answer key, and a silent exclusion is as misleading as none at all.
+    """
+
+    excluded = 0
+
+
+def load_cases(path, inputs=None, expected=None, name_col='name'):
+    """Load eval cases from a .csv / .yaml / .jsonl file into pydantic-evals ``Case`` objects.
+
+    Column routing, by name, with no configuration:
+      * ``name``                      -> the case name
+      * a field of the ``inputs`` model -> that input field (validated by pydantic)
+      * ``expected`` / ``expected.*``  -> the expected output
+      * everything else                -> case metadata, which becomes a DataFrame
+                                          column and is therefore available as a report slice
+
+    A value of the form ``@some/path.md`` is replaced by that file's text, resolved
+    relative to the case file. This is what keeps a flat CSV usable with realistic
+    inputs -- a long fixture document lives in one file instead of being repeated
+    down a column.
+
+    ``inputs`` is a pydantic model class. Pass it: a column your model has no field for
+    lands in metadata, so a misspelled one leaves its real field unset and pydantic
+    raises a ``ValidationError`` naming that field. That check is the whole reason
+    moving cases out of Python is safe. Without a model there is nothing to tell an
+    input column from a metadata one, so every column except ``name``, ``expected*``
+    and the reserved names below becomes an input.
+
+    ``expected`` is an optional model class for the expected output, built from the
+    ``expected.*`` columns the same way.
+
+    Reserved metadata names: ``synthetic`` (kept apart from real cases, because a
+    synthetic set written by the mind that wrote the prompt flatters it), ``holdout``,
+    ``arm``, and ``expect_*`` for per-case expectations. A truthy ``holdout``
+    **excludes** the case from the returned list; the number dropped is on the returned
+    list's ``.excluded`` attribute. ``arm`` is the one name a sweep will not accept --
+    it labels each cell of the matrix, so a column of that name is refused up front
+    rather than shadowed (name yours ``surface``, ``write_arm``, ``variant``...).
+
+    ``path`` may also be a list of row dicts -- the same routing, for a probe that keeps
+    its handful of cases inline. ``@file`` references then resolve against the working
+    directory.
+
+    Returns a ``CaseList`` -- a list of ``Case`` carrying that count.
+    """
+    if isinstance(path, (list, tuple)):
+        rows = [dict(r) if isinstance(r, dict) else r for r in path]
+        path = Path.cwd() / '<inline cases>'
+    else:
+        path = Path(path)
+        rows = _read(path)
+    field_names = _model_field_names(inputs) if inputs is not None else None
+
+    cases = CaseList()
+    excluded = 0
+    for i, row in enumerate(rows):
+        if not isinstance(row, dict):
+            raise ValueError(
+                "{}: case {} is a {}, expected one mapping per case".format(
+                    path, i + 1, type(row).__name__))
+        row = {(k.strip() if isinstance(k, str) else k): v for k, v in row.items()}
+        name = row.pop(name_col, None)
+        name = str(name) if name not in (None, '') else 'case_{}'.format(i + 1)
+        row = {k: _resolve_at(v, path, name, k) for k, v in row.items()}
+
+        kwargs, expected_map, expected_scalar, metadata = {}, {}, None, {}
+        has_expected_scalar = False
+        for col, value in row.items():
+            if col == _EXPECTED:
+                expected_scalar, has_expected_scalar = value, True
+            elif col.startswith(_EXPECTED_PREFIX):
+                expected_map[col[len(_EXPECTED_PREFIX):]] = value
+            elif field_names is None:
+                if _is_reserved(col):
+                    metadata[col] = _coerce_metadata(col, value)
+                else:
+                    kwargs[col] = value
+            elif col in field_names:
+                kwargs[col] = value
+            else:
+                metadata[col] = _coerce_metadata(col, value)
+
+        if has_expected_scalar and expected_map:
+            raise ValueError(
+                "case '{}' in {}: has both an 'expected' column and expected.* columns "
+                "-- use one or the other".format(name, path))
+        if has_expected_scalar and isinstance(expected_scalar, dict):
+            expected_map, has_expected_scalar = dict(expected_scalar), False
+
+        case_inputs = _drop_empty(kwargs)
+        if inputs is not None:
+            case_inputs = _build(inputs, case_inputs, name, path)
+
+        if expected is not None:
+            if has_expected_scalar:
+                raise ValueError(
+                    "case '{}' in {}: expected= is a model, so the file needs "
+                    "expected.<field> columns, not a bare 'expected' column".format(name, path))
+            case_expected = _build(expected, _drop_empty(expected_map), name, path)
+        elif expected_map:
+            case_expected = _drop_empty(expected_map)
+        else:
+            case_expected = expected_scalar if expected_scalar != '' else None
+
+        if _truthy(metadata.get('holdout')):
+            excluded += 1
+            continue
+        cases.append(Case(name=name, inputs=case_inputs, metadata=metadata or None,
+                          expected_output=case_expected))
+
+    cases.excluded = excluded
+    return cases
+
+
+# ---- readers ---------------------------------------------------------------------
+
+def _read(path):
+    suffix = path.suffix.lower()
+    if suffix == '.csv':
+        return _read_csv(path)
+    if suffix in ('.yaml', '.yml'):
+        return _read_yaml(path)
+    if suffix == '.jsonl':
+        return _read_jsonl(path)
+    raise ValueError(
+        "{}: unsupported case file '{}' -- use .csv, .yaml or .jsonl".format(path, suffix))
+
+
+def _read_csv(path):
+    # utf-8-sig: a CSV saved by a spreadsheet carries a BOM, which would otherwise
+    # become part of the first column's name and route that column to metadata.
+    with open(path, newline='', encoding='utf-8-sig') as f:
+        return [dict(row) for row in csv.DictReader(f)]
+
+
+def _read_yaml(path):
+    with open(path, encoding='utf-8') as f:
+        raw = yaml.safe_load(f)
+    if isinstance(raw, dict) and 'cases' in raw:
+        raw = raw['cases']
+    if not isinstance(raw, list):
+        raise ValueError(
+            "{}: expected a list of cases, or a 'cases:' key holding one".format(path))
+    return raw
+
+
+def _read_jsonl(path):
+    rows = []
+    with open(path, encoding='utf-8') as f:
+        for i, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rows.append(json.loads(line))
+            except ValueError as e:
+                raise ValueError("{} line {}: not valid JSON -- {}".format(path, i, e))
+    return rows
+
+
+# ---- helpers ---------------------------------------------------------------------
+
+def _model_field_names(model):
+    fields = getattr(model, 'model_fields', None) or getattr(model, '__fields__', None)
+    if not fields:
+        raise TypeError(
+            "{!r} is not a pydantic model class -- load_cases needs one so a misspelled "
+            "column is a ValidationError rather than a silent default".format(model))
+    return list(fields)
+
+
+def _build(model, kwargs, name, path):
+    try:
+        return model(**kwargs)
+    except Exception as e:
+        # Keep the exception type -- a ValidationError already names the offending
+        # field -- and add the case, which the file knows and the model does not.
+        if hasattr(e, 'add_note'):
+            e.add_note("while building case '{}' from {}".format(name, path))
+        raise
+
+
+def _resolve_at(value, path, name, col):
+    """``@some/path.md`` -> that file's text, resolved relative to the case file."""
+    if not isinstance(value, str) or not value.startswith('@'):
+        return value
+    ref = value[1:].strip()
+    target = (path.parent / ref).resolve()
+    if not target.is_file():
+        raise FileNotFoundError(
+            "case '{}', column '{}': '{}' points at {}, which does not exist".format(
+                name, col, value, target))
+    return target.read_text(encoding='utf-8')
+
+
+def _drop_empty(kwargs):
+    """An empty CSV cell means 'not provided', so the model's own default applies."""
+    return {k: v for k, v in kwargs.items() if v not in (None, '')}
+
+
+def _is_reserved(col):
+    return col in ('synthetic', 'holdout') or col.startswith('expect_')
+
+
+def _coerce_metadata(col, value):
+    """CSV gives every cell as a string; recover booleans where the target is unambiguous.
+
+    ``true/false/yes/no`` are booleans in any column. ``1/0`` are booleans only in a
+    column that is boolean by convention (``holdout``, ``synthetic``, ``expect_*``) --
+    elsewhere they are as likely to be a count, and guessing wrong would put a bool in
+    a column someone means to average.
+    """
+    if not isinstance(value, str):
+        return value
+    text = value.strip().lower()
+    if text in _BOOL_TRUE:
+        return True
+    if text in _BOOL_FALSE:
+        return False
+    if _is_reserved(col) and text in ('1', '0'):
+        return text == '1'
+    return None if value == '' else value
+
+
+def _truthy(value):
+    if isinstance(value, str):
+        return value.strip().lower() in _TRUTHY
+    return bool(value)