evals-viewer-io 0.0.3__tar.gz

evals_viewer_io-0.0.3/.gitignore

@@ -0,0 +1,7 @@
+node_modules/
+dist/
+.DS_Store
+*.log
+.venv/
+__pycache__/
+*.egg-info/

evals_viewer_io-0.0.3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Dan Lester
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evals_viewer_io-0.0.3/PKG-INFO

@@ -0,0 +1,216 @@
+Metadata-Version: 2.4
+Name: evals-viewer-io
+Version: 0.0.3
+Summary: Pydantic schemas and a writer for the evals-viewer on-disk format — the Python writer side of the evals-viewer framework.
+Project-URL: Homepage, https://github.com/ideonate/evals-viewer
+Project-URL: Repository, https://github.com/ideonate/evals-viewer
+Project-URL: Issues, https://github.com/ideonate/evals-viewer/issues
+Project-URL: Documentation, https://github.com/ideonate/evals-viewer/blob/main/docs/data-layout.md
+Author: Dan Lester
+License-Expression: MIT
+License-File: LICENSE
+Keywords: evals,evaluation,llm,pydantic,viewer
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: pydantic>=2.0
+Provides-Extra: pytest
+Requires-Dist: pytest>=7; extra == 'pytest'
+Description-Content-Type: text/markdown
+
+# evals-viewer-io
+
+Pydantic schemas and a writer for the [**evals-viewer**](https://github.com/ideonate/evals-viewer) on-disk format. This is the Python writer side of the framework — it produces the JSON tree that [`@ideonate/evals-viewer-server`](https://www.npmjs.com/package/@ideonate/evals-viewer-server) reads and the Vue frontend [`@ideonate/evals-viewer-core`](https://www.npmjs.com/package/@ideonate/evals-viewer-core) renders.
+
+## Install
+
+```sh
+pip install evals-viewer-io
+```
+
+Requires Python 3.10+ and Pydantic 2.
+
+## What's in the box
+
+| Symbol | Purpose |
+| --- | --- |
+| `RunMetadata`, `EvalSummary`, `CaseSummary`, `AggregateStats` | Pydantic models matching the on-disk format |
+| `TokenUsage` | Token / cost model with addition, `from_pydantic_ai` adapter, per-model breakdown |
+| `save_run_metadata`, `save_eval_results` | Filesystem writers — given models and dicts, write JSON in the layout the viewer expects |
+| `compute_aggregates(cases)` | Group `case.scores[evaluator]` across cases → `{evaluator: {mean, min, max}}` |
+| `compute_token_totals(cases)` | Sum token usage / cost / per-model breakdown across cases |
+| `eval_run_dir` (pytest fixture) | Optional fixture creating a fresh run directory under `EVALS_RESULTS_DIR` |
+
+## Quickstart: minimal end-to-end
+
+```python
+from evals_viewer_io import (
+    RunMetadata, EvalSummary, CaseSummary, TokenUsage,
+    compute_aggregates, compute_token_totals,
+    save_eval_results,
+)
+
+# 1. Build per-case rows. The output_summary dict is a free-form bag of
+#    fields the viewer can show in the eval-detail table; token fields
+#    use the canonical input_tokens / output_tokens / cost_usd / usage_by_model.
+cases = [
+    CaseSummary(
+        name="case_001",
+        scores={"Accuracy": 0.9, "Coverage": 0.8},
+        judge_reasons={"Accuracy": "All key facts present."},
+        output_summary={
+            "input_tokens": 1234,
+            "output_tokens": 567,
+            "cost_usd": 0.012,
+        },
+    ),
+    CaseSummary(
+        name="case_002",
+        scores={"Accuracy": 0.7, "Coverage": 0.9},
+        output_summary={"input_tokens": 980, "output_tokens": 440, "cost_usd": 0.009},
+    ),
+    CaseSummary(name="case_003", success=False, error="Timeout"),
+]
+
+# 2. Compute the per-eval aggregates and write the run.
+summary = EvalSummary(
+    timestamp="2026-04-07T10:30:00Z",
+    aggregates=compute_aggregates(cases),
+    cases=cases,
+)
+
+save_eval_results(
+    results_dir="./tests/test-results/evals",
+    run_id="2026-04-07_103000",
+    eval_name="my_eval",
+    summary=summary,
+    outputs={
+        "case_001": {"answer": "...", "input_tokens": 1234, "output_tokens": 567, "cost_usd": 0.012},
+        "case_002": {"answer": "...", "input_tokens": 980, "output_tokens": 440, "cost_usd": 0.009},
+    },
+    run=RunMetadata(timestamp="2026-04-07T10:30:00Z", git_commit="abc1234"),
+)
+```
+
+That writes:
+
+```
+tests/test-results/evals/2026-04-07_103000/
+├── run.json
+└── my_eval/
+    ├── summary.json
+    └── outputs/
+        ├── case_001.json
+        └── case_002.json
+```
+
+Open the viewer and the run shows up.
+
+## Token usage
+
+`TokenUsage` is a normal Pydantic model with `__add__` so you can sum across cases or across model calls:
+
+```python
+from evals_viewer_io import TokenUsage
+
+opus_call = TokenUsage(input_tokens=1200, output_tokens=300, cost_usd=0.018)
+haiku_call = TokenUsage(input_tokens=800, output_tokens=200, cost_usd=0.0009)
+
+# Per-model breakdown for one case
+case_total = TokenUsage(
+    input_tokens=opus_call.input_tokens + haiku_call.input_tokens,
+    output_tokens=opus_call.output_tokens + haiku_call.output_tokens,
+    cost_usd=(opus_call.cost_usd or 0) + (haiku_call.cost_usd or 0),
+    usage_by_model={"opus": opus_call, "haiku": haiku_call},
+)
+
+# Or just use sum() across multiple cases:
+total = sum([case1_usage, case2_usage, case3_usage])
+```
+
+The viewer reads `input_tokens`, `output_tokens`, `cost_usd`, and `usage_by_model` from both each case's full output JSON and from the per-case row in `summary.json`'s `output_summary`.
+
+### Pydantic-AI adapter
+
+If you use [pydantic-ai](https://ai.pydantic.dev), there's a one-liner to convert its `Usage` / `RunUsage` objects (which use `request_tokens` / `response_tokens` rather than `input` / `output`):
+
+```python
+from evals_viewer_io import TokenUsage
+
+usage = TokenUsage.from_pydantic_ai(result.usage(), cost_usd=my_cost_calc(result))
+```
+
+The adapter uses `getattr` so this package never imports pydantic-ai itself. Other frameworks (OpenAI SDK, Anthropic SDK, …) can be mapped just as easily — `TokenUsage(input_tokens=resp.usage.prompt_tokens, output_tokens=resp.usage.completion_tokens)` etc.
+
+Cost is the caller's responsibility. Pricing tables go stale fast and don't belong in this package.
+
+## Aggregating tokens across cases
+
+```python
+from evals_viewer_io import compute_token_totals
+
+totals = compute_token_totals(cases)
+print(totals.input_tokens, totals.output_tokens, totals.cost_usd)
+print(totals.usage_by_model)  # per-model breakdown summed across all cases
+```
+
+The function reads `input_tokens` / `output_tokens` / `cost_usd` / `usage_by_model` from each case's `output_summary`. Cases that don't have those fields contribute zero.
+
+## pytest fixture
+
+```python
+# tests/conftest.py
+from evals_viewer_io.pytest import eval_run_dir  # noqa: F401
+```
+
+```python
+# tests/test_my_eval.py
+def test_my_eval(eval_run_dir):
+    # eval_run_dir is a pathlib.Path under EVALS_RESULTS_DIR (or a tmp dir),
+    # and run.json has already been written.
+    ...
+    save_eval_results(
+        results_dir=eval_run_dir.parent,
+        run_id=eval_run_dir.name,
+        eval_name="my_eval",
+        summary=summary,
+        outputs=outputs,
+    )
+```
+
+Set `EVALS_RESULTS_DIR=tests/test-results/evals` (or wherever your project keeps them) so the run lands somewhere the viewer can find.
+
+## What this package deliberately does *not* do
+
+This is intentionally a small package — schemas plus the smallest set of helpers that every consumer would need to write themselves. It does **not** include:
+
+- **Token field extraction from arbitrary model outputs.** Different LLM SDKs name fields differently; the caller knows their own output schema.
+- **A pricing table.** Costs are pricing × tokens; pricing changes weekly. You compute it, you pass it in via `cost_usd`.
+- **Pydantic→dict serialization.** If your case output is a Pydantic model, call `.model_dump()` yourself before passing it to `save_eval_results`. Hiding that behind a wrapper would just suppress errors.
+- **Coupling to a specific eval framework** like `pydantic-evals` or `inspect_ai`. The writer takes plain dicts. Frameworks can be added as adapters when there's demand.
+- **Schema versioning.** The on-disk format is forward-compatible by design (`extra="allow"` everywhere). If a breaking change ever lands, that's the time for a `schema_version` field, not now.
+
+## On-disk contract
+
+See [`docs/data-layout.md`](https://github.com/ideonate/evals-viewer/blob/main/docs/data-layout.md) in the monorepo for the full directory tree and per-file schemas. The TL;DR:
+
+```
+{results_dir}/{run_id}/
+├── run.json                       (RunMetadata)
+└── {eval_name}/
+    ├── summary.json               (EvalSummary: aggregates + per-case rows)
+    ├── outputs/{case_name}.json   (full per-case output)
+    ├── inputs/{case_name}.json    (optional; saved input fixture)
+    └── case-scores/{case_name}.json (optional; per-question scores)
+```
+
+## License
+
+MIT

evals_viewer_io-0.0.3/README.md

@@ -0,0 +1,189 @@
+# evals-viewer-io
+
+Pydantic schemas and a writer for the [**evals-viewer**](https://github.com/ideonate/evals-viewer) on-disk format. This is the Python writer side of the framework — it produces the JSON tree that [`@ideonate/evals-viewer-server`](https://www.npmjs.com/package/@ideonate/evals-viewer-server) reads and the Vue frontend [`@ideonate/evals-viewer-core`](https://www.npmjs.com/package/@ideonate/evals-viewer-core) renders.
+
+## Install
+
+```sh
+pip install evals-viewer-io
+```
+
+Requires Python 3.10+ and Pydantic 2.
+
+## What's in the box
+
+| Symbol | Purpose |
+| --- | --- |
+| `RunMetadata`, `EvalSummary`, `CaseSummary`, `AggregateStats` | Pydantic models matching the on-disk format |
+| `TokenUsage` | Token / cost model with addition, `from_pydantic_ai` adapter, per-model breakdown |
+| `save_run_metadata`, `save_eval_results` | Filesystem writers — given models and dicts, write JSON in the layout the viewer expects |
+| `compute_aggregates(cases)` | Group `case.scores[evaluator]` across cases → `{evaluator: {mean, min, max}}` |
+| `compute_token_totals(cases)` | Sum token usage / cost / per-model breakdown across cases |
+| `eval_run_dir` (pytest fixture) | Optional fixture creating a fresh run directory under `EVALS_RESULTS_DIR` |
+
+## Quickstart: minimal end-to-end
+
+```python
+from evals_viewer_io import (
+    RunMetadata, EvalSummary, CaseSummary, TokenUsage,
+    compute_aggregates, compute_token_totals,
+    save_eval_results,
+)
+
+# 1. Build per-case rows. The output_summary dict is a free-form bag of
+#    fields the viewer can show in the eval-detail table; token fields
+#    use the canonical input_tokens / output_tokens / cost_usd / usage_by_model.
+cases = [
+    CaseSummary(
+        name="case_001",
+        scores={"Accuracy": 0.9, "Coverage": 0.8},
+        judge_reasons={"Accuracy": "All key facts present."},
+        output_summary={
+            "input_tokens": 1234,
+            "output_tokens": 567,
+            "cost_usd": 0.012,
+        },
+    ),
+    CaseSummary(
+        name="case_002",
+        scores={"Accuracy": 0.7, "Coverage": 0.9},
+        output_summary={"input_tokens": 980, "output_tokens": 440, "cost_usd": 0.009},
+    ),
+    CaseSummary(name="case_003", success=False, error="Timeout"),
+]
+
+# 2. Compute the per-eval aggregates and write the run.
+summary = EvalSummary(
+    timestamp="2026-04-07T10:30:00Z",
+    aggregates=compute_aggregates(cases),
+    cases=cases,
+)
+
+save_eval_results(
+    results_dir="./tests/test-results/evals",
+    run_id="2026-04-07_103000",
+    eval_name="my_eval",
+    summary=summary,
+    outputs={
+        "case_001": {"answer": "...", "input_tokens": 1234, "output_tokens": 567, "cost_usd": 0.012},
+        "case_002": {"answer": "...", "input_tokens": 980, "output_tokens": 440, "cost_usd": 0.009},
+    },
+    run=RunMetadata(timestamp="2026-04-07T10:30:00Z", git_commit="abc1234"),
+)
+```
+
+That writes:
+
+```
+tests/test-results/evals/2026-04-07_103000/
+├── run.json
+└── my_eval/
+    ├── summary.json
+    └── outputs/
+        ├── case_001.json
+        └── case_002.json
+```
+
+Open the viewer and the run shows up.
+
+## Token usage
+
+`TokenUsage` is a normal Pydantic model with `__add__` so you can sum across cases or across model calls:
+
+```python
+from evals_viewer_io import TokenUsage
+
+opus_call = TokenUsage(input_tokens=1200, output_tokens=300, cost_usd=0.018)
+haiku_call = TokenUsage(input_tokens=800, output_tokens=200, cost_usd=0.0009)
+
+# Per-model breakdown for one case
+case_total = TokenUsage(
+    input_tokens=opus_call.input_tokens + haiku_call.input_tokens,
+    output_tokens=opus_call.output_tokens + haiku_call.output_tokens,
+    cost_usd=(opus_call.cost_usd or 0) + (haiku_call.cost_usd or 0),
+    usage_by_model={"opus": opus_call, "haiku": haiku_call},
+)
+
+# Or just use sum() across multiple cases:
+total = sum([case1_usage, case2_usage, case3_usage])
+```
+
+The viewer reads `input_tokens`, `output_tokens`, `cost_usd`, and `usage_by_model` from both each case's full output JSON and from the per-case row in `summary.json`'s `output_summary`.
+
+### Pydantic-AI adapter
+
+If you use [pydantic-ai](https://ai.pydantic.dev), there's a one-liner to convert its `Usage` / `RunUsage` objects (which use `request_tokens` / `response_tokens` rather than `input` / `output`):
+
+```python
+from evals_viewer_io import TokenUsage
+
+usage = TokenUsage.from_pydantic_ai(result.usage(), cost_usd=my_cost_calc(result))
+```
+
+The adapter uses `getattr` so this package never imports pydantic-ai itself. Other frameworks (OpenAI SDK, Anthropic SDK, …) can be mapped just as easily — `TokenUsage(input_tokens=resp.usage.prompt_tokens, output_tokens=resp.usage.completion_tokens)` etc.
+
+Cost is the caller's responsibility. Pricing tables go stale fast and don't belong in this package.
+
+## Aggregating tokens across cases
+
+```python
+from evals_viewer_io import compute_token_totals
+
+totals = compute_token_totals(cases)
+print(totals.input_tokens, totals.output_tokens, totals.cost_usd)
+print(totals.usage_by_model)  # per-model breakdown summed across all cases
+```
+
+The function reads `input_tokens` / `output_tokens` / `cost_usd` / `usage_by_model` from each case's `output_summary`. Cases that don't have those fields contribute zero.
+
+## pytest fixture
+
+```python
+# tests/conftest.py
+from evals_viewer_io.pytest import eval_run_dir  # noqa: F401
+```
+
+```python
+# tests/test_my_eval.py
+def test_my_eval(eval_run_dir):
+    # eval_run_dir is a pathlib.Path under EVALS_RESULTS_DIR (or a tmp dir),
+    # and run.json has already been written.
+    ...
+    save_eval_results(
+        results_dir=eval_run_dir.parent,
+        run_id=eval_run_dir.name,
+        eval_name="my_eval",
+        summary=summary,
+        outputs=outputs,
+    )
+```
+
+Set `EVALS_RESULTS_DIR=tests/test-results/evals` (or wherever your project keeps them) so the run lands somewhere the viewer can find.
+
+## What this package deliberately does *not* do
+
+This is intentionally a small package — schemas plus the smallest set of helpers that every consumer would need to write themselves. It does **not** include:
+
+- **Token field extraction from arbitrary model outputs.** Different LLM SDKs name fields differently; the caller knows their own output schema.
+- **A pricing table.** Costs are pricing × tokens; pricing changes weekly. You compute it, you pass it in via `cost_usd`.
+- **Pydantic→dict serialization.** If your case output is a Pydantic model, call `.model_dump()` yourself before passing it to `save_eval_results`. Hiding that behind a wrapper would just suppress errors.
+- **Coupling to a specific eval framework** like `pydantic-evals` or `inspect_ai`. The writer takes plain dicts. Frameworks can be added as adapters when there's demand.
+- **Schema versioning.** The on-disk format is forward-compatible by design (`extra="allow"` everywhere). If a breaking change ever lands, that's the time for a `schema_version` field, not now.
+
+## On-disk contract
+
+See [`docs/data-layout.md`](https://github.com/ideonate/evals-viewer/blob/main/docs/data-layout.md) in the monorepo for the full directory tree and per-file schemas. The TL;DR:
+
+```
+{results_dir}/{run_id}/
+├── run.json                       (RunMetadata)
+└── {eval_name}/
+    ├── summary.json               (EvalSummary: aggregates + per-case rows)
+    ├── outputs/{case_name}.json   (full per-case output)
+    ├── inputs/{case_name}.json    (optional; saved input fixture)
+    └── case-scores/{case_name}.json (optional; per-question scores)
+```
+
+## License
+
+MIT

evals_viewer_io-0.0.3/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "evals-viewer-io"
+version = "0.0.3"
+description = "Pydantic schemas and a writer for the evals-viewer on-disk format — the Python writer side of the evals-viewer framework."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+    { name = "Dan Lester" },
+]
+keywords = ["evals", "llm", "pydantic", "evaluation", "viewer"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pydantic>=2.0",
+]
+
+[project.optional-dependencies]
+pytest = ["pytest>=7"]
+
+[project.urls]
+Homepage = "https://github.com/ideonate/evals-viewer"
+Repository = "https://github.com/ideonate/evals-viewer"
+Issues = "https://github.com/ideonate/evals-viewer/issues"
+Documentation = "https://github.com/ideonate/evals-viewer/blob/main/docs/data-layout.md"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evals_viewer_io"]

evals_viewer_io-0.0.3/src/evals_viewer_io/__init__.py

@@ -0,0 +1,23 @@
+"""Pydantic schemas and writer for the evals-viewer on-disk format."""
+
+from .aggregates import compute_aggregates, compute_token_totals
+from .schema import (
+    AggregateStats,
+    CaseSummary,
+    EvalSummary,
+    RunMetadata,
+    TokenUsage,
+)
+from .writer import save_eval_results, save_run_metadata
+
+__all__ = [
+    "AggregateStats",
+    "CaseSummary",
+    "EvalSummary",
+    "RunMetadata",
+    "TokenUsage",
+    "compute_aggregates",
+    "compute_token_totals",
+    "save_eval_results",
+    "save_run_metadata",
+]

evals_viewer_io-0.0.3/src/evals_viewer_io/aggregates.py

@@ -0,0 +1,88 @@
+"""Helpers for computing per-eval aggregates from a list of cases.
+
+These are deliberately small and unambiguous: any consumer with multiple
+cases per eval will need them, and the math has no domain knowledge.
+Anything fancier (token cost tables, framework-specific extraction) lives
+in caller code, not here.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+from .schema import AggregateStats, CaseSummary, TokenUsage
+
+
+def compute_aggregates(cases: Iterable[CaseSummary]) -> dict[str, AggregateStats]:
+    """Group ``case.scores[evaluator]`` across cases and return mean/min/max.
+
+    Failed cases (``success=False``) and cases with no scores are skipped.
+    Evaluators are listed in first-seen order across the input.
+    """
+    grouped: dict[str, list[float]] = {}
+    for case in cases:
+        if case.success is False or not case.scores:
+            continue
+        for evaluator, value in case.scores.items():
+            if value is None:
+                continue
+            grouped.setdefault(evaluator, []).append(float(value))
+
+    return {
+        evaluator: AggregateStats(
+            mean=sum(values) / len(values),
+            min=min(values),
+            max=max(values),
+        )
+        for evaluator, values in grouped.items()
+        if values
+    }
+
+
+def compute_token_totals(cases: Iterable[CaseSummary]) -> TokenUsage:
+    """Sum token usage across cases.
+
+    Reads ``input_tokens``, ``output_tokens``, ``cost_usd``, and
+    ``usage_by_model`` from each case's ``output_summary``. Cases without
+    those fields contribute zero. The returned ``TokenUsage.usage_by_model``
+    is the per-model breakdown summed across all cases that had one.
+    """
+    total = TokenUsage()
+    for case in cases:
+        summary = case.output_summary or {}
+        contribution = _token_usage_from_dict(summary)
+        if contribution is not None:
+            total = total + contribution
+    return total
+
+
+def _token_usage_from_dict(d: dict[str, Any]) -> TokenUsage | None:
+    """Best-effort extraction from a free-form output_summary dict."""
+    if not any(k in d for k in ("input_tokens", "output_tokens", "cost_usd", "usage_by_model")):
+        return None
+
+    raw_by_model = d.get("usage_by_model")
+    by_model: dict[str, TokenUsage] | None
+    if raw_by_model:
+        by_model = {}
+        for model, usage in raw_by_model.items():
+            if isinstance(usage, TokenUsage):
+                entry = usage
+            elif isinstance(usage, dict):
+                entry = TokenUsage(
+                    input_tokens=int(usage.get("input_tokens", 0) or 0),
+                    output_tokens=int(usage.get("output_tokens", 0) or 0),
+                    cost_usd=usage.get("cost_usd"),
+                )
+            else:
+                continue
+            by_model[model] = entry
+    else:
+        by_model = None
+
+    return TokenUsage(
+        input_tokens=int(d.get("input_tokens", 0) or 0),
+        output_tokens=int(d.get("output_tokens", 0) or 0),
+        cost_usd=d.get("cost_usd"),
+        usage_by_model=by_model,
+    )

evals_viewer_io-0.0.3/src/evals_viewer_io/pytest.py

@@ -0,0 +1,39 @@
+"""Optional pytest fixture helpers for evals-viewer-io.
+
+Apps can write their own fixtures, but this provides a sensible default:
+
+    # conftest.py
+    from evals_viewer_io.pytest import eval_run_dir  # noqa: F401
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pytest
+
+from .schema import RunMetadata
+from .writer import save_run_metadata
+
+
+def _now_run_id() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d_%H%M%S")
+
+
+@pytest.fixture(scope="session")
+def eval_run_dir(tmp_path_factory) -> Path:
+    """Create a fresh run directory for the test session and return its path.
+
+    Honours ``EVALS_RESULTS_DIR`` if set, otherwise uses a tmp dir. Writes a
+    minimal ``run.json`` so the viewer will list it.
+    """
+    base = Path(os.environ.get("EVALS_RESULTS_DIR", tmp_path_factory.mktemp("evals")))
+    run_id = _now_run_id()
+    save_run_metadata(
+        base,
+        run_id,
+        RunMetadata(timestamp=datetime.now(timezone.utc).isoformat()),
+    )
+    return base / run_id

evals_viewer_io-0.0.3/src/evals_viewer_io/schema.py

@@ -0,0 +1,141 @@
+"""Pydantic models matching the evals-viewer on-disk format.
+
+See ``docs/data-layout.md`` in the monorepo for the canonical contract.
+These models are intentionally permissive (extra fields allowed) so writers
+can include domain-specific extras without coordinating schema bumps.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class _Permissive(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+
+class AggregateStats(_Permissive):
+    """Aggregate stats for one evaluator across all cases in an eval."""
+
+    mean: float
+    min: float
+    max: float
+
+
+class TokenUsage(_Permissive):
+    """Token usage and cost for one model call, one case, or a whole eval.
+
+    Field names follow the Anthropic / viewer convention (input/output rather
+    than request/response). Adapter classmethods like ``from_pydantic_ai``
+    rename other frameworks' fields onto these.
+
+    Cost is the caller's responsibility — pricing tables go stale fast and
+    don't belong in this package.
+
+    The ``usage_by_model`` map is one level deep: entries inside it should
+    leave their own ``usage_by_model`` as None.
+    """
+
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cost_usd: float | None = None
+    usage_by_model: dict[str, "TokenUsage"] | None = None
+
+    def __add__(self, other: TokenUsage) -> TokenUsage:
+        if not isinstance(other, TokenUsage):
+            return NotImplemented
+
+        # Sum cost only if at least one side reports it; preserve None when
+        # nobody knows the cost.
+        if self.cost_usd is None and other.cost_usd is None:
+            cost = None
+        else:
+            cost = (self.cost_usd or 0.0) + (other.cost_usd or 0.0)
+
+        # Merge per-model breakdowns. Only one level deep — don't recurse.
+        merged: dict[str, TokenUsage] | None
+        if self.usage_by_model is None and other.usage_by_model is None:
+            merged = None
+        else:
+            merged = {}
+            for source in (self.usage_by_model or {}, other.usage_by_model or {}):
+                for model, usage in source.items():
+                    if model in merged:
+                        merged[model] = TokenUsage(
+                            input_tokens=merged[model].input_tokens + usage.input_tokens,
+                            output_tokens=merged[model].output_tokens + usage.output_tokens,
+                            cost_usd=_sum_optional(merged[model].cost_usd, usage.cost_usd),
+                        )
+                    else:
+                        merged[model] = TokenUsage(
+                            input_tokens=usage.input_tokens,
+                            output_tokens=usage.output_tokens,
+                            cost_usd=usage.cost_usd,
+                        )
+
+        return TokenUsage(
+            input_tokens=self.input_tokens + other.input_tokens,
+            output_tokens=self.output_tokens + other.output_tokens,
+            cost_usd=cost,
+            usage_by_model=merged,
+        )
+
+    def __radd__(self, other: Any) -> TokenUsage:
+        # Lets `sum([usage1, usage2, ...])` work without an explicit start value.
+        if other == 0:
+            return self
+        return NotImplemented
+
+    @classmethod
+    def from_pydantic_ai(cls, usage: Any, *, cost_usd: float | None = None) -> TokenUsage:
+        """Build a TokenUsage from a pydantic-ai Usage or RunUsage object.
+
+        Maps ``request_tokens`` → ``input_tokens`` and ``response_tokens`` →
+        ``output_tokens``. Tolerates None values (treats them as zero).
+        Cost is not derived from tokens — pass it in if you have it.
+
+        Uses ``getattr`` so this package never imports pydantic-ai itself.
+        """
+        return cls(
+            input_tokens=getattr(usage, "request_tokens", 0) or 0,
+            output_tokens=getattr(usage, "response_tokens", 0) or 0,
+            cost_usd=cost_usd,
+        )
+
+
+def _sum_optional(a: float | None, b: float | None) -> float | None:
+    if a is None and b is None:
+        return None
+    return (a or 0.0) + (b or 0.0)
+
+
+class CaseSummary(_Permissive):
+    """One row in ``summary.json``'s ``cases`` array."""
+
+    name: str
+    success: bool = True
+    error: str | None = None
+    scores: dict[str, float] | None = None
+    judge_reasons: dict[str, str] | None = None
+    integrity: dict[str, Any] | None = None
+    output_summary: dict[str, Any] | None = None
+
+
+class EvalSummary(_Permissive):
+    """Contents of ``summary.json`` for one eval inside one run."""
+
+    timestamp: str | None = None
+    aggregates: dict[str, AggregateStats] = Field(default_factory=dict)
+    cases: list[CaseSummary] = Field(default_factory=list)
+
+
+class RunMetadata(_Permissive):
+    """Contents of ``run.json`` at the run-id directory root."""
+
+    timestamp: str
+    git_commit: str | None = None
+    git_branch: str | None = None
+    git_dirty: bool = False
+    tags: list[str] = Field(default_factory=list)

evals_viewer_io-0.0.3/src/evals_viewer_io/writer.py

@@ -0,0 +1,64 @@
+"""Filesystem writer for the evals-viewer on-disk format."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Mapping
+
+from .schema import EvalSummary, RunMetadata
+
+
+def _dump(path: Path, data: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2, default=str))
+
+
+def save_run_metadata(
+    results_dir: str | Path,
+    run_id: str,
+    run: RunMetadata,
+) -> Path:
+    """Write ``{results_dir}/{run_id}/run.json`` and return its path."""
+    run_dir = Path(results_dir) / run_id
+    path = run_dir / "run.json"
+    _dump(path, run.model_dump(mode="json", exclude_none=True))
+    return path
+
+
+def save_eval_results(
+    results_dir: str | Path,
+    run_id: str,
+    eval_name: str,
+    summary: EvalSummary,
+    outputs: Mapping[str, Any],
+    *,
+    inputs: Mapping[str, Any] | None = None,
+    case_scores: Mapping[str, Any] | None = None,
+    run: RunMetadata | None = None,
+) -> Path:
+    """Write a complete eval result tree under ``{results_dir}/{run_id}/{eval_name}/``.
+
+    If ``run`` is supplied and ``run.json`` does not yet exist, it is written too.
+    """
+    base = Path(results_dir) / run_id / eval_name
+
+    _dump(base / "summary.json", summary.model_dump(mode="json", exclude_none=True))
+
+    for case_name, output in outputs.items():
+        _dump(base / "outputs" / f"{case_name}.json", output)
+
+    if inputs:
+        for case_name, value in inputs.items():
+            _dump(base / "inputs" / f"{case_name}.json", value)
+
+    if case_scores:
+        for case_name, value in case_scores.items():
+            _dump(base / "case-scores" / f"{case_name}.json", value)
+
+    if run is not None:
+        run_path = Path(results_dir) / run_id / "run.json"
+        if not run_path.exists():
+            save_run_metadata(results_dir, run_id, run)
+
+    return base