ezvals 0.2.0__py3-none-win_amd64.whl

ezvals/__init__.py

@@ -0,0 +1,23 @@
+from typing import Any, Callable, TypedDict
+
+from ezvals.cli import run
+from ezvals.context import EvalContext, EvalResult, TraceData
+from ezvals.evals import eval, run_evals
+
+
+class EvalCase(TypedDict, total=False):
+    """An item of @eval(cases=[...]). Fields override the decorator's values for that case."""
+    id: str
+    input: Any
+    reference: Any
+    metadata: dict
+    dataset: str
+    labels: list
+    default_score_key: str
+    timeout: float
+    target: Callable
+    evaluators: list
+    trials: int
+
+
+__all__ = ["eval", "EvalContext", "EvalResult", "TraceData", "EvalCase", "run_evals", "run"]

ezvals/cli.py

@@ -0,0 +1,33 @@
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+BINARY = Path(__file__).parent / "bin" / ("ezvals.exe" if os.name == "nt" else "ezvals")
+
+
+def main():
+    """`ezvals` console script: hand off to the bundled host binary."""
+    env = {**os.environ, "EZVALS_PYTHON": sys.executable}
+    if os.name == "nt":
+        sys.exit(subprocess.run([str(BINARY), *sys.argv[1:]], env=env).returncode)
+    os.execve(BINARY, [str(BINARY), *sys.argv[1:]], env)
+
+
+def run(path: str, dataset: Optional[str] = None, labels: Optional[list] = None, limit: Optional[int] = None,
+        output: Optional[str] = None, concurrency: Optional[int] = None, timeout: Optional[float] = None,
+        trials: Optional[int] = None, session: Optional[str] = None, run_name: Optional[str] = None, no_save: bool = False,
+        config: Optional[str] = None) -> dict:
+    """Programmatic `ezvals run`. Returns the run (results, totals, ids) plus `saved_path`."""
+    flags = {"--dataset": dataset, "--limit": limit, "--output": output, "--concurrency": concurrency,
+             "--timeout": timeout, "--trials": trials, "--session": session, "--run-name": run_name, "--config": config}
+    args = [str(BINARY), "run", path, "--json"]
+    args += [arg for flag, value in flags.items() if value is not None for arg in (flag, str(value))]
+    args += [arg for label in labels or [] for arg in ("--label", label)]
+    args += ["--no-save"] if no_save else []
+    proc = subprocess.run(args, capture_output=True, text=True, env={**os.environ, "EZVALS_PYTHON": sys.executable})
+    if proc.returncode not in (0, 4):  # 4: no evals matched, which still returns the (empty) run
+        raise ValueError(proc.stderr.strip())
+    return json.loads(proc.stdout)

ezvals/context.py

@@ -0,0 +1,134 @@
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+
+class TraceData(dict):
+    """Trace/debug data. A dict with attribute access; `messages` and `trace_url` are always readable."""
+
+    def __getattr__(self, key):
+        if key == "messages":
+            return self.setdefault("messages", [])
+        if key == "trace_url":
+            return self.get("trace_url")
+        try:
+            return self[key]
+        except KeyError:
+            raise AttributeError(key) from None
+
+    def __setattr__(self, key, value):
+        self[key] = list(value) if key == "messages" else value
+
+    def add_messages(self, messages: list) -> "TraceData":
+        """Replace messages (does not append)."""
+        self["messages"] = list(messages)
+        return self
+
+
+def normalize_score(score: Any, default_key: Optional[str]) -> dict:
+    if isinstance(score, bool):
+        score = {"passed": score}
+    elif isinstance(score, (int, float)):
+        score = {"value": score}
+    elif not isinstance(score, dict) or score.get("value") is None and score.get("passed") is None:
+        raise ValueError(f"Invalid score {score!r}: use True/False, a number, "
+                         "or {'key': ..., 'passed'/'value': ..., 'notes': ...}")
+    score = dict(score)
+    if "key" not in score:
+        if default_key is None:
+            raise ValueError("Must specify score key or set default_score_key")
+        score["key"] = default_key
+    return score
+
+
+def normalize_scores(scores: Any, default_key: Optional[str]) -> list:
+    """A score, a list of scores, or None, as a list of score dicts."""
+    scores = scores if isinstance(scores, list) else [] if scores is None else [scores]
+    normalized = [normalize_score(s, default_key) for s in scores]
+    if sum(not (isinstance(s, dict) and "key" in s) for s in scores) > 1:
+        raise ValueError(f"Scores {scores!r} would share the key {default_key!r}: give each a 'key'")
+    return normalized
+
+
+@dataclass
+class EvalResult:
+    input: Any
+    output: Any
+    reference: Any = None
+    scores: list = field(default_factory=list)
+    error: Optional[str] = None
+    latency: Optional[float] = None
+    metadata: dict = field(default_factory=dict)
+    trace_data: TraceData = field(default_factory=TraceData)
+
+    def __post_init__(self):
+        self.scores = normalize_scores(self.scores, "pass")
+        self.metadata = self.metadata or {}
+        self.trace_data = TraceData(self.trace_data or {})
+
+
+class EvalContext:
+    """Mutable builder for an EvalResult. Injected into any eval parameter annotated as EvalContext."""
+
+    def __init__(self, input=None, output=None, reference=None, default_score_key: Optional[str] = "pass",
+                 metadata=None, trace_data=None, latency=None, function_name=None, dataset=None, labels=None,
+                 **run_info):
+        self.input = input
+        self.output = output
+        self.reference = reference
+        self.default_score_key = default_score_key
+        self.metadata = dict(metadata or {})
+        self.trace_data = TraceData(trace_data or {})
+        self.latency = latency
+        self.scores: list = []
+        self.error: Optional[str] = None
+        self.function_name = function_name
+        self.dataset = dataset
+        self.labels = labels
+        self._run = run_info
+
+    def store(self, input=None, output=None, reference=None, latency=None, scores=None, messages=None,
+              trace_url=None, metadata=None, trace_data=None) -> "EvalContext":
+        """Set any fields passed. Scores with an existing key overwrite it; metadata and trace_data merge."""
+        if input is not None:
+            self.input = input
+        if output is not None:
+            self.output = output
+        if reference is not None:
+            self.reference = reference
+        if latency is not None:
+            self.latency = latency
+        if trace_data is not None:
+            self.trace_data.update(trace_data)
+        if messages is not None:
+            self.trace_data.messages = messages
+        if trace_url is not None:
+            self.trace_data.trace_url = trace_url
+        if metadata is not None:
+            self.metadata.update(metadata)
+        for score in normalize_scores(scores, self.default_score_key):
+            keys = [s["key"] for s in self.scores]
+            if score["key"] in keys:
+                self.scores[keys.index(score["key"])] = score
+            else:
+                self.scores.append(score)
+        return self
+
+    def build(self) -> EvalResult:
+        scores = self.scores or ([] if self.error else [{"key": self.default_score_key or "pass", "passed": True}])
+        return EvalResult(input=self.input, output=self.output, reference=self.reference, scores=scores,
+                          error=self.error, latency=self.latency, metadata=self.metadata, trace_data=self.trace_data)
+
+    def build_with_error(self, error: str) -> EvalResult:
+        self.error = error
+        return self.build()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+for _name in ("run_id", "session_name", "run_name", "eval_path"):
+    setattr(EvalContext, _name, property(lambda self, n=_name: self._run.get(n)))
+EvalContext.config = property(lambda self: self._run.get("config") or {})

ezvals/evals.py

@@ -0,0 +1,287 @@
+import asyncio
+import contextlib
+import contextvars
+import functools
+import importlib.util
+import inspect
+import os
+import sys
+import threading
+import time
+import traceback
+import typing
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+from ezvals.context import EvalContext, EvalResult, normalize_scores
+
+# Parameters that file defaults and individual cases can set.
+PARAMS = {"input", "reference", "dataset", "labels", "metadata", "default_score_key", "timeout", "target", "evaluators",
+          "trials"}
+_UNSET = object()
+
+
+class EvalFunction:
+    def __init__(self, func: Callable, params: dict):
+        self.func = func
+        self.params = params
+        self.ctx_param = next((name for name, hint in _hints(func).items() if _is_context(hint)), None)
+        needs_ctx = {"target", "input_loader", "cases"} & params.keys()
+        if needs_ctx and self.ctx_param is None:
+            raise ValueError(f"{sorted(needs_ctx)[0]} requires the evaluation function to accept a context parameter")
+        if "input_loader" in params and {"input", "reference", "cases"} & params.keys():
+            raise ValueError("input_loader cannot be used with input=, reference= or cases=")
+        functools.update_wrapper(self, func)
+
+    def __call__(self):
+        return _run_sync(self.call_async())
+
+    async def call_async(self):
+        results = [r for e in expand(self) for r in await e.run()]
+        return results[0] if len(results) == 1 else results
+
+
+def eval(func=None, *, input=_UNSET, reference=_UNSET, dataset=_UNSET, labels=_UNSET, metadata=_UNSET,
+         default_score_key=_UNSET, timeout=_UNSET, target=_UNSET, evaluators=_UNSET, input_loader=_UNSET,
+         cases=_UNSET, trials=_UNSET):
+    """Mark a function as an evaluation. Usable as @eval or @eval(...). `ezvals run` runs it `trials` times."""
+    params = {k: v for k, v in locals().items() if k != "func" and v is not _UNSET}
+    if func is not None:
+        return EvalFunction(func, params)
+    return lambda f: EvalFunction(f, params)
+
+
+@dataclass
+class Eval:
+    """One runnable evaluation: an EvalFunction resolved against file defaults and a single case."""
+    id: str
+    name: str
+    func: Callable
+    ctx_param: Optional[str]
+    params: dict
+    error: Optional[str] = None
+
+    async def run(self, run_info: Optional[dict] = None, grade: Optional[dict] = None) -> list:
+        """Run the eval. With `grade` (a stored result), skip the target and score that result's output again."""
+        run_info = run_info or {}
+        p = self.params
+        if self.error:
+            return [EvalResult(input=None, output=None, error=self.error)]
+        ctx = EvalContext(input=p.get("input"), reference=p.get("reference"), metadata=p.get("metadata"),
+                          default_score_key=p["default_score_key"], function_name=self.name,
+                          dataset=p["dataset"], labels=p["labels"], **run_info) if self.ctx_param else None
+        if grade:
+            ctx.store(input=grade.get("input"), output=grade.get("output"), latency=grade.get("latency"),
+                      metadata=grade.get("metadata"), trace_data=grade.get("trace_data"))
+        timeout = run_info.get("timeout") or p.get("timeout")
+        start = time.perf_counter()
+        try:
+            returned = await asyncio.wait_for(self._call_with_target(ctx, skip_target=bool(grade)), timeout)
+            if returned is None and ctx is not None:
+                returned = ctx
+            if isinstance(returned, EvalContext):
+                returned = returned.build()
+            results = returned if isinstance(returned, list) else [returned]
+            if not all(isinstance(r, EvalResult) for r in results):
+                raise ValueError("Evaluation function must return EvalResult, List[EvalResult], EvalContext, "
+                                 f"or None (with a context parameter), got {type(returned).__name__}")
+        except asyncio.TimeoutError:
+            results = [_errored(ctx, f"TimeoutError: Evaluation timed out after {float(timeout)}s")]
+        except AssertionError as e:
+            if ctx is None:
+                results = [_errored(None, describe(e))]
+            else:  # a bare `assert` has no message, so its source line explains the failure
+                notes = str(e) or traceback.extract_tb(e.__traceback__)[-1].line or "Assertion failed"
+                ctx.store(scores={"passed": False, "notes": notes})
+                results = [ctx.build()]
+        except Exception as e:
+            results = [_errored(ctx, describe(e))]
+        if ctx is not None and inspect.iscoroutine(ctx.output):
+            ctx.output.close()
+            ctx.output = None
+            results = [ctx.build_with_error("TypeError: ctx.output was never awaited. Did you forget await?")]
+        try:  # evaluators score every finished result, including ones a failed assertion scored
+            for i in range(len(results)):
+                for evaluator in [] if results[i].error else p.get("evaluators") or []:
+                    scored = await _call(evaluator, results[i])
+                    if isinstance(scored, EvalResult):
+                        results[i] = scored
+                    elif scored is not None:
+                        results[i].scores += normalize_scores(scored, "pass")
+        except Exception as e:
+            results = [_errored(ctx, describe(e))]
+        for result in results:
+            if not result.scores and not result.error:
+                result.scores = [{"key": p["default_score_key"], "passed": True}]
+            if result.latency is None:
+                result.latency = (time.perf_counter() - start) / len(results)
+        return results
+
+    async def _call_with_target(self, ctx, skip_target):
+        if self.params.get("target") and not skip_target:
+            out = await _call(self.params["target"], ctx)
+            if isinstance(out, EvalResult):
+                ctx.store(output=out.output, latency=out.latency, trace_data=out.trace_data, metadata=out.metadata)
+            elif out is not None and not isinstance(out, EvalContext):
+                ctx.store(output=out)
+        return await _call(self.func, **({self.ctx_param: ctx} if self.ctx_param else {}))
+
+
+def expand(fn: EvalFunction, file_defaults: Optional[dict] = None, file: Optional[str] = None) -> list:
+    """Resolve an EvalFunction into runnable Evals: builtins < file defaults < decorator < each case."""
+    file = file or inspect.getfile(fn.func)
+    unknown = set(file_defaults or {}) - PARAMS
+    if unknown:
+        raise ValueError(f"Unknown keys in ezvals_defaults: {', '.join(sorted(unknown))}")
+    builtins = {"dataset": Path(file).stem, "labels": [], "metadata": {}, "default_score_key": "pass"}
+    base = _layer(_layer(builtins, file_defaults or {}, merge_labels=False),
+                  {k: v for k, v in fn.params.items() if k in PARAMS}, merge_labels=False)
+    name = fn.func.__name__
+
+    def make(case_name, params, error=None):
+        return Eval(f"{file}::{case_name}", case_name, fn.func, fn.ctx_param, params, error)
+
+    if "input_loader" in fn.params:
+        try:
+            examples = _run_sync(_call(fn.params["input_loader"]))
+        except Exception as e:
+            return [make(name, base, describe(e, f"Input loader failed: {e}"))]
+        cases = [ex if isinstance(ex, dict) else {k: getattr(ex, k) for k in PARAMS | {"id"} if hasattr(ex, k)} for ex in examples]
+    elif "cases" in fn.params:
+        cases = fn.params["cases"]
+    else:
+        return [make(name, base)]
+
+    evals = []
+    for i, case in enumerate(cases):
+        unknown = set(case) - PARAMS - {"id"}
+        if unknown:
+            raise ValueError(f"Unknown case keys: {', '.join(sorted(unknown))}")
+        overrides = {k: v for k, v in case.items() if k != "id"}
+        evals.append(make(f"{name}[{case.get('id', i)}]", _layer(base, overrides, merge_labels=True)))
+    return evals
+
+
+def discover(files: list) -> list:
+    """Import eval files and return their Evals in source order."""
+    evals = []
+    for file in files:
+        sys.path.insert(0, str(Path(file).resolve().parent))
+        spec = importlib.util.spec_from_file_location(Path(file).stem, file)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[spec.name] = module
+        spec.loader.exec_module(module)
+        found = [obj for obj in vars(module).values()
+                 if isinstance(obj, EvalFunction) and obj.func.__module__ == module.__name__]
+        for fn in sorted(found, key=lambda f: f.func.__code__.co_firstlineno):
+            evals += expand(fn, getattr(module, "ezvals_defaults", None), file)
+    return evals
+
+
+def run_evals(evals: list, concurrency: int = 1, timeout: Optional[float] = None, dataset: Optional[str] = None,
+              labels: Optional[list] = None, limit: Optional[int] = None) -> list:
+    """Run eval functions and/or paths in-process and return their EvalResults (nothing is saved)."""
+    resolved = []
+    for item in evals:
+        resolved += expand(item) if isinstance(item, EvalFunction) else discover(_eval_files(item))
+    if dataset:
+        resolved = [e for e in resolved if e.params["dataset"] in dataset.split(",")]
+    if labels:
+        resolved = [e for e in resolved if set(labels) & set(e.params["labels"])]
+    semaphore = asyncio.Semaphore(concurrency)
+
+    async def one(e):
+        async with semaphore:
+            return await e.run({"timeout": timeout})
+
+    async def all_():
+        return await asyncio.gather(*(one(e) for e in resolved[:limit]))
+
+    return [r for results in _run_sync(all_()) for r in results]
+
+
+def _eval_files(path: str) -> list:
+    p = Path(path)
+    if p.is_file():
+        return [str(p)]
+    return [str(f) for f in sorted(p.rglob("*.py"))
+            if not f.name.startswith("_") and not any(part.startswith((".", "__")) for part in f.relative_to(p).parts)]
+
+
+def _layer(base: dict, over: dict, merge_labels: bool) -> dict:
+    """Overlay params. Metadata merges (None clears); case labels merge with base labels (None/[] clears)."""
+    out = {**base, **over}
+    if over.get("metadata") is not None:
+        out["metadata"] = {**(base.get("metadata") or {}), **over["metadata"]}
+    if merge_labels and "labels" in over:
+        out["labels"] = base["labels"] + [l for l in over["labels"] or [] if l not in base["labels"]] if over["labels"] else []
+    return out
+
+
+_INTERNAL = (os.path.dirname(__file__), os.path.dirname(asyncio.__file__), threading.__file__, "<frozen")
+
+
+def describe(error: BaseException, headline: Optional[str] = None) -> str:
+    """The error, then its traceback through the user's own code only (not ezvals', asyncio's or threading's)."""
+    frames = [f for f in traceback.extract_tb(error.__traceback__) if not f.filename.startswith(_INTERNAL)]
+    for f in frames:
+        if f.filename.startswith(os.getcwd() + os.sep):
+            f.filename = os.path.relpath(f.filename)
+    where = "Traceback (most recent call last):\n" + "".join(traceback.format_list(frames)) if frames else ""
+    return (headline + "\n" if headline else "".join(traceback.format_exception_only(error))) + where
+
+
+def _errored(ctx: Optional[EvalContext], error: str) -> EvalResult:
+    return ctx.build_with_error(error) if ctx else EvalResult(input=None, output=None, error=error)
+
+
+async def _call(fn, *args, **kwargs):
+    """Await async callables; run sync ones on a daemon thread so a timeout can abandon them."""
+    if inspect.iscoroutinefunction(fn):
+        return await fn(*args, **kwargs)
+    loop = asyncio.get_running_loop()
+    future = loop.create_future()
+    call = functools.partial(contextvars.copy_context().run, fn, *args, **kwargs)
+
+    def settle(result=None, error=None):
+        if not future.done():
+            future.set_exception(error) if error else future.set_result(result)
+
+    def target():
+        try:
+            outcome = (call(), None)
+        except BaseException as e:
+            outcome = (None, e)
+        with contextlib.suppress(RuntimeError):  # the loop is gone if this call was abandoned after a timeout
+            loop.call_soon_threadsafe(settle, *outcome)
+
+    threading.Thread(target=target, daemon=True).start()
+    return await future
+
+
+def _run_sync(coro):
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+    with ThreadPoolExecutor(1) as pool:
+        return pool.submit(asyncio.run, coro).result()
+
+
+def _hints(func) -> dict:
+    params = inspect.signature(func).parameters
+    try:
+        hints = typing.get_type_hints(func)
+    except Exception:
+        hints = {}
+    return {name: hints.get(name, p.annotation) for name, p in params.items()}
+
+
+def _is_context(hint) -> bool:
+    if hint is EvalContext:
+        return True
+    if isinstance(hint, (str, typing.ForwardRef)):
+        return str(getattr(hint, "__forward_arg__", hint)).split(".")[-1] == "EvalContext"
+    return any(_is_context(arg) for arg in typing.get_args(hint))

ezvals/worker.py

@@ -0,0 +1,66 @@
+"""Worker process the ezvals host spawns to run Python evals.
+
+Protocol (JSON lines): the worker writes {"type": "evals"} once discovery finishes, then reads
+{"id", "run", "grade"?} requests from stdin and answers each with {"type": "result", "id"}. Evals run
+concurrently; the worker exits once stdin closes and in-flight evals finish. User output goes to stderr.
+"""
+import asyncio
+import dataclasses
+import json
+import os
+import sys
+
+from ezvals.evals import describe, discover
+
+
+def main():
+    protocol = os.fdopen(os.dup(1), "w", buffering=1)
+    os.dup2(2, 1)
+    sys.stdout.reconfigure(line_buffering=True)  # user prints show up as they happen with --verbose
+
+    def send(message):
+        try:
+            line = json.dumps(message, default=_jsonable, allow_nan=False)
+        except ValueError:  # NaN/Infinity aren't JSON: send them as null
+            line = json.dumps(json.loads(json.dumps(message, default=_jsonable), parse_constant=lambda _: None))
+        protocol.write(line + "\n")
+
+    run_info = json.loads(os.environ.get("EZVALS_RUN") or "{}")
+    try:
+        evals = {e.id: e for e in discover(sys.argv[1:])}
+    except Exception as error:
+        send({"type": "error", "error": describe(error)})
+        sys.exit(1)
+    send({"type": "evals", "evals": [{
+        "id": e.id, "function": e.name, "dataset": e.params["dataset"], "labels": e.params["labels"],
+        "input": e.params.get("input"), "reference": e.params.get("reference"), "metadata": e.params.get("metadata"),
+        "trials": e.params.get("trials"), "target": bool(e.params.get("target")),
+    } for e in evals.values()]})
+
+    async def run(request):
+        try:
+            results = await evals[request["run"]].run(run_info, request.get("grade"))
+            send({"type": "result", "id": request["id"], "results": [vars(r) for r in results]})
+        except Exception as error:  # the host waits for every result, so one that can't be reported becomes an error
+            send({"type": "result", "id": request["id"], "results": [{"error": describe(error)}]})
+
+    async def serve():
+        loop = asyncio.get_running_loop()
+        tasks = set()
+        while line := await loop.run_in_executor(None, sys.stdin.readline):
+            tasks.add(asyncio.create_task(run(json.loads(line))))
+        await asyncio.gather(*tasks)
+
+    asyncio.run(serve())
+
+
+def _jsonable(obj):
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump()
+    if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
+        return dataclasses.asdict(obj)
+    return str(obj)
+
+
+if __name__ == "__main__":
+    main()

ezvals-0.2.0.dist-info/METADATA

@@ -0,0 +1,94 @@
+Metadata-Version: 2.5
+Name: ezvals
+Version: 0.2.0
+Summary: A lightweight, code-first evaluation framework for testing AI agents and LLM applications
+Project-URL: Homepage, https://github.com/camronh/EZVals
+Project-URL: Repository, https://github.com/camronh/EZVals
+License: MIT
+Keywords: agents,benchmark,cli,evaluation,llm,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# EZVals
+
+Unit testing for AI agents and LLM apps. Write evals like tests, in Python or TypeScript; EZVals runs them, stores every result, and gives you a web UI to review, annotate and compare runs.
+
+![UI screenshot](https://raw.githubusercontent.com/camronh/EZVals/main/assets/ui.png)
+
+## Install
+
+EZVals is a development dependency.
+
+```bash
+uv add --dev ezvals          # or: pip install ezvals
+npm install --save-dev ezvals  # TypeScript (Node >= 22.18)
+```
+
+## Write an eval
+
+```python
+# evals/support.py
+from ezvals import eval, EvalContext
+
+@eval(input="I want a refund", dataset="customer_service")
+async def test_refund(ctx: EvalContext):
+    ctx.output = await run_agent(ctx.input)
+    assert "refund" in ctx.output.lower(), "Should acknowledge refund"
+```
+
+```ts
+// evals/support.eval.ts
+import assert from "node:assert";
+import { evaluate } from "ezvals";
+
+evaluate("refund", { input: "I want a refund", dataset: "customer_service" }, async (ctx) => {
+  ctx.output = await runAgent(ctx.input);
+  assert(String(ctx.output).toLowerCase().includes("refund"), "Should acknowledge refund");
+});
+```
+
+A failed assertion is a failing score, not an error, and whatever the eval stored before it is kept. Use `cases=` to run one eval over many inputs, `ctx.store(scores=...)` for named or numeric scores, and `target=` to separate calling your agent from scoring it. See the [docs](https://github.com/camronh/EZVals/tree/main/docs) and [examples](https://github.com/camronh/EZVals/tree/main/examples).
+
+## Run it
+
+```bash
+ezvals run evals/                  # headless; saves to .ezvals/sessions/
+ezvals run evals/ --json           # ...and print the results as JSON (handy for agents)
+ezvals serve evals/                # web UI: run, filter, annotate, compare runs
+ezvals run evals/support.py::test_refund -c 8 --timeout 30
+ezvals run evals/ --trials 5         # run each eval 5 times; reports pass@5 and pass^5
+ezvals regrade a1b2c3d4              # re-score a run's outputs after changing a grader, without re-running the agent
+ezvals query "SELECT run_name, total_passed FROM runs"   # SQL over every saved run (see: ezvals query --schema)
+```
+
+TypeScript projects run the same commands through `npx ezvals`. A directory with both languages runs as one.
+
+## Agent skill
+
+EZVals ships a skill that teaches coding agents to write and analyze evals:
+
+```bash
+ezvals skills add --claude   # or --codex, --cursor, --windsurf, --kiro, --roo, --agents
+```
+
+## How it works
+
+`ezvals` is a single Go binary (`cmd/ezvals/`) that owns the CLI, the web UI (`ui/`, embedded at build time) and storage. To run evals it spawns a small worker from the language SDK (`python/`, `typescript/`) that imports your files and executes evals on request; results stream back and are appended to a per-run event log. Adding a language means writing an SDK against [the SDK spec](https://github.com/camronh/EZVals/blob/main/docs/.spec/EXPERIENCE_SPEC_SDK.md) and passing the shared `conformance/` fixtures.
+
+## Development
+
+Requires Go, Node, and uv.
+
+```bash
+make build                         # UI + host binary (into python/ezvals/bin) + TypeScript SDK
+make test                          # Go (incl. cross-language conformance), Python and TypeScript tests
+make test-e2e                      # browser tests for the web UI against a real server
+make storybook                     # every UI component and page in its states (also run as tests by `make test`)
+uv run --project python ezvals serve examples
+```
+
+Specs in `docs/.spec/` are the source of truth for behavior.

ezvals-0.2.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+ezvals/__init__.py,sha256=ANt0TZK8B8JuwGB_gSC1gYTTq7fJ_VNUnWORhhVWHfY,610
+ezvals/cli.py,sha256=XAzIrl7qZ6u88qJlYW2gZjTKACiLQrZUARLW9R90I6M,1775
+ezvals/context.py,sha256=jz_26oBUMu2pI1LXJ0WsjlgALpfssNCpkh_7B646L3Q,5317
+ezvals/evals.py,sha256=2kHIMQuSn_20FpelO1hVElyUIzY6bHBaiB0rmS-dPGw,13096
+ezvals/worker.py,sha256=TuL38t8Jg69zH3IYt4PUKwSX3iHqEEMGdcvuGJ_wCXw,2609
+ezvals/bin/ezvals.exe,sha256=lKAtvh4zDlUPjnrVW6NQ_Ck6RS2F3IbP0MdXv-qsnJk,18387456
+ezvals-0.2.0.dist-info/METADATA,sha256=0O1_3Vm7EcLFLYxhGjduezJASU5Fh9Xsp6uKPKjOSYY,4145
+ezvals-0.2.0.dist-info/WHEEL,sha256=A7YmR1UrJozxyeLCzQFwQUdR7CRUwkFKasEDmnrjVxY,94
+ezvals-0.2.0.dist-info/entry_points.txt,sha256=HgjFjE8IGlHpiVID1_NF3vqwcwxcml_MW4u7OH1cCqo,43
+ezvals-0.2.0.dist-info/RECORD,,

ezvals-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: false
+Tag: py3-none-win_amd64

ezvals-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ezvals = ezvals.cli:main