evalstand 0.0.0.dev0__tar.gz

evalstand-0.0.0.dev0/.gitignore

@@ -0,0 +1,28 @@
+__pycache__/
+*.py[cod]
+.Python
+build/
+dist/
+*.egg-info/
+.venv/
+venv/
+
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+coverage.xml
+
+*.db
+*.sqlite
+*.sqlite3
+.evalstand/
+
+.env
+.env.*
+!.env.example
+
+.DS_Store
+.idea/
+.vscode/

evalstand-0.0.0.dev0/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+
+All notable changes to this project are documented here.
+Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
+this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Phase 0: repository scaffold, tooling configuration, CI workflow, ADR process.
+- Name chosen: `evalstand` (ADR 0001).

evalstand-0.0.0.dev0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 evalstand contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalstand-0.0.0.dev0/PKG-INFO

@@ -0,0 +1,127 @@
+Metadata-Version: 2.5
+Name: evalstand
+Version: 0.0.0.dev0
+Summary: A local-first LLM evaluation tool for Python: write evals, run them like tests, watch results stream in.
+License: MIT License
+        
+        Copyright (c) 2026 evalstand contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: ai,eval,evaluation,llm,pytest,testing
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: litellm>=1.55
+Requires-Dist: pydantic>=2.10
+Requires-Dist: pytest>=8.3
+Requires-Dist: rapidfuzz>=3.11
+Requires-Dist: rich>=13.9
+Requires-Dist: tenacity>=9.0
+Requires-Dist: textual>=1.0
+Requires-Dist: typer>=0.15
+Requires-Dist: watchfiles>=1.0
+Provides-Extra: examples
+Requires-Dist: faker>=33.1; extra == 'examples'
+Requires-Dist: pypdfium2>=4.30; extra == 'examples'
+Requires-Dist: reportlab>=4.2; extra == 'examples'
+Description-Content-Type: text/markdown
+
+# evalstand
+
+> **Status: in development.** Phase 0 of 7. This release is a **name
+> reservation placeholder** and contains no working code. The example below
+> shows the intended API, which is not implemented yet.
+
+Evaluating an LLM application should feel like running a test suite.
+
+`evalstand` is a local-first LLM evaluation tool for Python. You write an eval
+file, run a watch command, and results stream into a live terminal UI — scores,
+nested call traces, token counts, latency, and cost. Everything runs on your
+machine and persists to a local SQLite database, so you can compare a run
+against the one before it.
+
+```python
+from evalstand import Case, evaluate
+from evalstand.scorers import exact, levenshtein
+
+
+def load_cases() -> list[Case]:
+    return [
+        Case(id="q1", input="What is the capital of France?", expected="Paris"),
+        Case(id="q2", input="What is 2 + 2?", expected="4"),
+    ]
+
+
+async def answer(question: str) -> str:
+    resp = await llm.acall("gpt-4o-mini", [{"role": "user", "content": question}])
+    return resp.text
+
+
+evaluate(
+    name="basic-qa",
+    cases=load_cases,
+    task=answer,
+    scorers=[exact, levenshtein],
+)
+```
+
+Save that as `qa_eval.py` and run it either way:
+
+```bash
+evalstand run qa_eval.py     # live TUI, watch mode, traces
+pytest qa_eval.py            # plain test runner, CI-friendly
+```
+
+## Why
+
+Existing Python options are either heavyweight platforms that push you toward a
+hosted service, or bare metric libraries with no runner, no persistence, and no
+live feedback loop. `evalstand` is the middle: a real runner with a real UI that
+stays on your machine.
+
+## Planned capabilities
+
+See [PLAN.md](PLAN.md) for the full build plan and the capability checklist that
+defines v1.
+
+## Limitations
+
+Stated up front, and kept accurate as the project grows:
+
+- Score differences between runs are reported as plain deltas. There is **no
+  statistical significance testing** in v1, so a delta is not evidence of a real
+  regression or improvement.
+- LLM-as-judge scorers are **unvalidated** — they have not been calibrated
+  against human labels.
+
+## Licence
+
+MIT. See [LICENSE](LICENSE).
+
+---
+
+<sub>Inspired by [evalite](https://github.com/mattpocock/evalite) (MIT), which
+showed that local LLM evals could feel like running tests. `evalstand` is an
+independent Python implementation.</sub>

evalstand-0.0.0.dev0/PLAN.md

@@ -0,0 +1,432 @@
+# Build Plan — `evalstand`
+
+> **Name:** `evalstand` — verified free on PyPI (404) and GitHub (0 name matches) on 2026-09-01.
+> **Audience:** an autonomous coding agent (Claude Code) executing this plan without access to the conversation that produced it.
+> **Owner profile:** mid-level software engineer, 3–5 years production experience, strong in Python, Docker, and CI; new to AI engineering.
+> **Budget:** ~15 hours per week. Total: 6–7 weeks for feature parity (Phases 0–7).
+> **Scope:** this project is entirely self-contained. It has no dependencies on, and makes no references to, any other project. Everything needed is in this document.
+
+---
+
+## 1. What we are building and why
+
+### The goal
+
+`evalstand` — a local-first LLM evaluation tool for Python.
+
+The core idea: evaluating an LLM application should feel exactly like running a test suite: you write an eval file, run a watch command, and a live UI updates as results stream in, showing scores, traces, token counts, and cost. Nothing in the Python ecosystem offers that experience. The closest Python tools are either heavyweight platforms that push you toward a hosted service, or bare metric libraries with no runner, no persistence, and no live feedback loop.
+
+The design is informed by `evalite` (TypeScript, MIT), which solved this problem well. `evalstand` is an independent Python implementation, not a transliteration: the capability set in Section 2 is the target, and every design decision is made for Python idiom on its own merits. Where the reference design does not carry into Python, we adopt the Python equivalent and record why in an ADR.
+
+### Explicit non-goals for this version
+
+These are deliberately out of scope. They are listed so the agent does not drift into them, and revisited in Section 8.
+
+- Statistical significance testing of run-over-run differences.
+- Validation or calibration of LLM-as-judge scorers.
+- Any hosted service, account system, or cloud sync.
+- Distributed or multi-machine execution.
+- Dataset generation or curation tooling.
+
+### Interface choice: TUI first
+
+`evalstand` ships a **Textual TUI first**, with a web UI as an optional stretch phase. Rationale: the TUI reaches parity of *function* much faster, keeps the entire stack in Python, and demos well in a terminal recording. Record this in an ADR.
+
+---
+
+## 2. Feature parity checklist
+
+This is the definition of "done" for v1. Every row must be satisfied or consciously waived with an ADR.
+
+| # | Capability | `evalstand` implementation | Phase |
+| --- | --- | --- | --- |
+| 1 | Eval files auto-collected by the test runner | `*_eval.py` collected by a pytest plugin | 2 |
+| 2 | One entry function for cases, task, scorers | `evaluate(cases=, task=, scorers=)` | 2 |
+| 3 | Case loader returning input/expected pairs | Sync or async callable returning `Case` objects | 2 |
+| 4 | Task receives input, returns output | Sync or async, auto-detected | 2 |
+| 5 | Built-in scorer library | Native scorer set, plus an adapter interface | 4 |
+| 6 | Custom scorers via a simple function signature | `@scorer` decorator | 4 |
+| 7 | Traces — nested LLM calls captured inside a task | `trace()` context manager + auto-capture | 3 |
+| 8 | Token and cost reporting per call and per run | Via LiteLLM cost lookup | 1, 3 |
+| 9 | Model response caching | SQLite-backed cache | 1 |
+| 10 | Run each case N times | `--repeat N` / `repeat=` argument | 3 |
+| 11 | SQLite result storage | stdlib `sqlite3` with migrations | 5 |
+| 12 | Score history across runs | `history` command and TUI history view | 5 |
+| 13 | Live-updating UI during a run | Textual TUI | 6 |
+| 14 | Watch mode with file-change re-runs | `watchfiles` | 6 |
+| 15 | Case detail view — input, output, expected, trace | TUI detail pane | 6 |
+| 16 | Custom result table columns | `columns=` argument on `evaluate()` | 6 |
+| 17 | `--threshold` for CI pass/fail | `--threshold` with documented exit codes | 7 |
+| 18 | Streaming task output | LiteLLM streaming, rendered live | 3 |
+| 19 | Runs under the plain test runner too | Works under bare `pytest` | 2 |
+
+---
+
+## 3. Architecture
+
+```
+evalstand/
+├── pyproject.toml
+├── README.md
+├── LICENSE                        # MIT
+├── CHANGELOG.md
+├── .github/workflows/
+│   ├── ci.yml                     # lint, type-check, test — no API keys
+│   └── release.yml                # PyPI publish on tag
+├── docs/
+│   ├── index.md
+│   ├── quickstart.md
+│   ├── writing-evals.md
+│   ├── scorers.md
+│   ├── traces.md
+│   ├── ci.md
+│   └── adr/
+├── src/evalstand/
+│   ├── __init__.py                # public API, kept under 10 exported names
+│   ├── api.py                     # evaluate(), Case, Score, @scorer
+│   ├── models.py                  # Case, Score, Trace, Result, Run
+│   ├── plugin.py                  # pytest collection + reporting hooks
+│   ├── runner.py                  # async execution, concurrency, repeats
+│   ├── tracing.py                 # trace context manager + LiteLLM callback
+│   ├── llm.py                     # LiteLLM wrapper: call, tokens, cost, retry
+│   ├── cache.py                   # SQLite response cache, record/replay
+│   ├── storage.py                 # SQLite run store + migrations
+│   ├── config.py                  # settings resolution
+│   ├── scorers/
+│   │   ├── base.py                # Scorer protocol, @scorer decorator
+│   │   ├── string.py              # exact, normalised, contains, regex
+│   │   ├── fuzzy.py               # levenshtein, ratio (rapidfuzz)
+│   │   ├── numeric.py             # tolerance-based
+│   │   ├── json_field.py          # per-field structured comparison
+│   │   └── llm.py                 # judge-style scorers
+│   ├── reporting/
+│   │   ├── console.py             # rich summary tables
+│   │   └── markdown.py            # CI summary body
+│   ├── tui/
+│   │   ├── app.py
+│   │   └── widgets/
+│   └── cli.py                     # typer entrypoint
+├── examples/
+│   ├── toy/                       # 3 cases, built in Phase 2, used throughout
+│   └── pdf_extraction/            # showcase example, Phase 5
+└── tests/
+    ├── cassettes/                 # recorded responses, committed
+    ├── unit/
+    └── integration/
+```
+
+### Core abstractions
+
+Four concepts. Resist adding a fifth.
+
+- **Case** — one test input: `id`, `input`, `expected` (optional), `metadata`.
+- **Task** — the user's function under test. Receives `Case.input`, returns any output. The LLM call happens here.
+- **Scorer** — a callable `(output, expected, case) -> Score`. `Score` carries `value: float` in `[0, 1]`, optional `passed: bool`, and `metadata: dict`.
+- **Run** — one execution of one eval. Persisted, immutable, comparable to previous runs.
+
+**Trace** is a supporting concept, not a fifth abstraction: a record of one LLM call made inside a task, forming a tree when calls nest.
+
+### Target API shape
+
+```python
+from evalstand import evaluate, Case, scorer
+from evalstand.scorers import levenshtein, exact
+
+
+def load_cases() -> list[Case]:
+    return [
+        Case(id="q1", input="What is the capital of France?", expected="Paris"),
+        Case(id="q2", input="What is 2 + 2?", expected="4"),
+    ]
+
+
+async def answer(question: str) -> str:
+    resp = await llm.acall("gpt-4o-mini", [{"role": "user", "content": question}])
+    return resp.text
+
+
+evaluate(
+    name="basic-qa",
+    cases=load_cases,
+    task=answer,
+    scorers=[exact, levenshtein],
+)
+```
+
+Saved as `qa_eval.py`, this runs under both `evalstand run` and bare `pytest`.
+
+---
+
+## 4. Approved dependencies
+
+| Purpose | Package |
+| --- | --- |
+| Packaging | `uv` |
+| Test runner base | `pytest` |
+| LLM provider layer | `litellm` |
+| CLI | `typer` |
+| Console rendering | `rich` |
+| TUI | `textual` |
+| File watching | `watchfiles` |
+| Data models | `pydantic` v2 |
+| Fuzzy matching | `rapidfuzz` |
+| Retry | `tenacity` |
+| Docs | `mkdocs-material` |
+| Dev | `ruff`, `mypy`, `pytest-cov`, `pre-commit` |
+| Example only | `reportlab`, `faker`, `pypdfium2` (behind an optional extra) |
+
+Storage is stdlib `sqlite3`. **Do not add an ORM.** Do not add a dependency outside this table without an ADR.
+
+---
+
+## 5. Hard guardrails
+
+The agent must not violate these.
+
+1. **CI runs with zero API keys and zero API spend.** All tests use recorded cassettes or mocks. Live tests are marked `@pytest.mark.live` and excluded by default.
+2. **Never commit secrets.** Keys come from environment variables only. A secret-scanning pre-commit hook is installed in Phase 0.
+3. **Expected outputs are written by humans or produced programmatically — never generated by an LLM.** A golden answer produced by the system under test is not ground truth.
+4. **No dependency outside Section 4 without an ADR.**
+5. **A phase is not done until `ruff check`, `ruff format --check`, `mypy`, and `pytest` all pass locally and in CI.**
+6. **Every number in the README must be reproducible by a command in the repo.**
+7. **Write an ADR for every non-obvious decision** in `docs/adr/NNNN-title.md`: context, decision, consequences. Keep them short.
+
+---
+
+## Phase 0 — Foundations
+
+**Goal:** a named, licensed, linted, CI-green empty package.
+**Estimate:** 6 hours.
+
+- [x] **0.1 Public name: `evalstand`.** Verified free on PyPI (`https://pypi.org/pypi/evalstand/json` -> 404) and GitHub (0 repositories matching the name) on 2026-09-01. Rejected because already taken on PyPI: `evalrig` (an AI-eval package), `proofmark` (an LLM-output testing package, uploaded 2026-08-17), `assay`, `evalbench`, `plumbline`, `rigor`, `tessera`, `proofbench`, `evalforge`, `evalloop`, `rubricon`.
+  - *Acceptance:* recorded in `docs/adr/0001-name.md`, applied consistently across `pyproject.toml`, `src/`, and docs.
+  - *Naming rule (binding):* the string `evalite` must not appear in any package name, module name, directory name, filename, class name, function name, or CLI command. A single attribution footnote at the bottom of the README is the only permitted occurrence in the shipped repo.
+- [ ] **0.2 Initialise the repo:** `uv init --lib`, Python 3.11+, `src/` layout, MIT `LICENSE`, `.gitignore`.
+  - *Acceptance:* `uv sync` succeeds; `uv run python -c "import evalstand"` succeeds.
+- [ ] **0.3 Configure tooling:** `ruff` (lint + format), `mypy` strict on `src/`, `pytest` with `--cov`, `pre-commit` with ruff plus a secret-scanning hook.
+  - *Acceptance:* `pre-commit run --all-files` passes on a clean tree.
+- [ ] **0.4 Write `.github/workflows/ci.yml`.** Matrix over Python 3.11, 3.12, 3.13. Steps: checkout, install uv, sync, ruff check, ruff format --check, mypy, pytest.
+  - *Acceptance:* CI green on first push; the workflow file contains no `secrets.` reference.
+- [ ] **0.5 Create `docs/adr/0000-adr-process.md`** and a template.
+- [ ] **0.6 Study the reference implementation** before writing any code — `evalite` (TypeScript, MIT), specifically its entry function, CLI, SQLite layer, and web UI. Write `docs/adr/0002-design-scope.md` recording which behaviours `evalstand` adopts, which it does differently, and why (TUI instead of React, pytest instead of Vitest).
+  - *Acceptance:* the ADR names specific behaviours and specific decisions, not generalities. Read it for design understanding only — do not copy code. This is an independent implementation.
+- [ ] **0.7 Placeholder README** with the problem statement, a "status: in development" banner, and the one-line attribution footnote. No marketing claims yet.
+
+**Exit criteria:** CI green, pre-commit passes, name applied everywhere, ADRs 0001 and 0002 written.
+
+---
+
+## Phase 1 — LLM layer, caching, and cost tracking
+
+**Goal:** a single model call works, is cached, and reports tokens, latency, and cost.
+**Estimate:** 12 hours.
+
+- [ ] **1.1 Define `models.py`.** `Case`, `Score`, `Trace`, `Result`, `Run` as Pydantic models or frozen dataclasses. `Score.value` is a float in `[0, 1]`. `Score.passed` is optional and derived from a threshold when absent. Every model is JSON-serialisable.
+  - *Acceptance:* round-trip serialisation tests pass for each model.
+- [ ] **1.2 Build `llm.py` over LiteLLM.** `call(model, messages, **params) -> LLMResponse` and async `acall`, returning text, input and output token counts, latency in ms, `cost_usd` from `litellm.completion_cost()`, and the raw provider response.
+  - *Acceptance:* unit tests with LiteLLM mocked verify token and cost extraction. One `@pytest.mark.live` test hits a real cheap model and is excluded from CI.
+- [ ] **1.3 Add streaming support.** `acall_stream` yields chunks and accumulates the final response with correct token and cost totals.
+  - *Acceptance:* a streamed call and a non-streamed call to the same prompt produce identical accumulated text and equivalent cost.
+- [ ] **1.4 Add retry policy** with `tenacity`: retry on 429 and 5xx with exponential backoff and jitter, max 3 attempts. Never retry 4xx auth or content-policy errors. Log every retry.
+  - *Acceptance:* tests cover 429-then-success and 401-immediate-fail.
+- [ ] **1.5 Build `cache.py`.** SQLite response cache keyed on the SHA-256 of canonical JSON of `(model, messages, temperature, top_p, max_tokens, seed, tools, response_format)`. Store the response, `created_at`, and `hit_count`. Support `--no-cache` and `--refresh-cache`.
+  - *Acceptance:* the same call twice yields one provider call and one cache hit; changing temperature yields a miss.
+- [ ] **1.6 Add record/replay for tests.** A cassette mode writing responses to JSON on record and reading on replay.
+  - *Acceptance:* the full test suite passes with every provider API key unset.
+- [ ] **1.7 ADR 0003:** why LiteLLM rather than provider SDKs.
+
+**Exit criteria:** a throwaway script makes a cached model call and prints text, tokens, latency, and cost. CI green without keys.
+
+---
+
+## Phase 2 — The `evaluate()` API and pytest collection
+
+**Goal:** an eval file is discovered and executed with a first-class authoring experience.
+**Estimate:** 14 hours.
+
+- [ ] **2.1 Design the public API** in `api.py` to the shape in Section 3. Keep the exported surface under 10 names. `cases` accepts a list, a callable returning a list, or an async callable.
+  - *Acceptance:* `docs/writing-evals.md` contains a complete working example under 25 lines.
+- [ ] **2.2 Implement the pytest plugin** in `plugin.py`. Use `pytest_collect_file` to collect `*_eval.py`, generate one pytest item per case, and use `pytest_runtest_makereport` to capture outcomes. Register through the `pytest11` entry point.
+  - *Acceptance:* `pytest examples/toy` discovers and runs the eval; `pytest -k q1` selects a single case.
+- [ ] **2.3 Support both sync and async tasks.** Detect with `inspect.iscoroutinefunction` and dispatch accordingly. The user should never have to think about it.
+  - *Acceptance:* two identical evals, one sync and one async, produce identical results.
+- [ ] **2.4 Ensure bare `pytest` works.** Running `pytest` with no custom CLI must collect and execute evals and report pass/fail sensibly.
+  - *Acceptance:* documented in `docs/ci.md` with a working example.
+- [ ] **2.5 Build `examples/toy/`** — three cases, one scorer, no external files. Every subsequent phase develops against this.
+- [ ] **2.6 Console reporting** in `reporting/console.py` using rich: a summary table (per-scorer mean, pass count, total cost, wall time) and a failures table.
+  - *Acceptance:* the toy example prints a readable summary within one screen.
+
+**Exit criteria:** `evalstand run examples/toy` and `pytest examples/toy` both work end to end.
+
+---
+
+## Phase 3 — Runner, traces, repeats, and streaming
+
+**Goal:** concurrent execution with the nested-call visibility that makes the UI useful.
+**Estimate:** 14 hours.
+
+- [ ] **3.1 Build `runner.py`.** Async execution with an `asyncio.Semaphore` (default concurrency 8, `--concurrency` flag). Deterministic ordered result collection. Per-case timeout. Errors are captured on the result rather than aborting the run.
+  - *Acceptance:* a suite where one case raises still completes and records the error; changing concurrency measurably changes wall time.
+- [ ] **3.2 Implement tracing** in `tracing.py`. This is the feature most worth porting carefully. Two mechanisms:
+  - A `trace(name)` context manager the user can wrap around any operation.
+  - **Automatic capture** of every call made through `llm.py` during a task, using a `contextvars.ContextVar` to associate calls with the currently executing case.
+  - Traces nest into a tree. Each node records name, start, duration, input, output, model, tokens, and cost.
+  - *Acceptance:* a task making three nested LLM calls produces a three-node trace tree with correct parent-child relationships and per-node cost, and the sum of node costs equals the case total.
+- [ ] **3.3 Implement `--repeat N`** (the reference implementation calls this `trialCount`). Each case runs N times with `repeat_index` recorded on every result. **Bypass the cache across repeats when temperature > 0**, or repeats are meaningless — make this explicit and test it.
+  - *Acceptance:* `--repeat 5` on a temperature-0.7 task produces at least one case with 5 distinct outputs.
+- [ ] **3.4 Wire streaming through the runner** so partial output is available to the reporting layer as it arrives.
+  - *Acceptance:* a streaming task shows incremental output in console reporting.
+- [ ] **3.5 Aggregate per-run totals:** total cost, total tokens, cache hit rate, wall time, pass count.
+
+**Exit criteria:** the toy example runs concurrently with repeats, and a trace tree is captured and printable.
+
+---
+
+## Phase 4 — Scorers
+
+**Goal:** enough built-in scorers that a user never writes one on day one, plus a clean path when they do.
+**Estimate:** 10 hours.
+
+- [ ] **4.1 Define the `Scorer` protocol** in `scorers/base.py` and a `@scorer` decorator that adapts a plain function. Support sync and async scorers.
+  - *Acceptance:* a user-defined 3-line scorer works without importing any base class.
+- [ ] **4.2 String scorers** in `string.py`: `exact`, `normalised_exact` (case, whitespace, and punctuation folding), `contains`, `regex_match`.
+- [ ] **4.3 Fuzzy scorers** in `fuzzy.py` using `rapidfuzz`: `levenshtein` (normalised to `[0, 1]`; this is `evalstand`'s default scorer) and `ratio`.
+  - *Acceptance:* `levenshtein("kitten", "sitting")` returns the documented normalised value.
+- [ ] **4.4 Numeric scorer** in `numeric.py`: absolute and relative tolerance, with sensible handling of `None` and unparseable output.
+- [ ] **4.5 Structured scorer** in `json_field.py`: compare two dicts field by field, returning both a macro-average and a per-field breakdown in `Score.metadata`.
+  - *Acceptance:* a partial match on 3 of 5 fields returns 0.6 with the failing field names in metadata.
+- [ ] **4.6 LLM scorers** in `llm.py`: a `judge` factory taking a rubric and returning a scorer, plus a `factuality`-style scorer comparing output against expected. Judge calls must go through `llm.py` so they are cached, traced, and costed like any other call.
+  - *Acceptance:* a judge scorer's LLM call appears in the case's trace tree.
+  - *Note:* these scorers are unvalidated by design in this version. `docs/scorers.md` must say so plainly.
+- [ ] **4.7 Every scorer gets unit tests** covering the happy path, empty output, and `None` expected.
+- [ ] **4.8 Write `docs/scorers.md`** documenting each built-in scorer and how to write a custom one.
+
+**Exit criteria:** eight or more built-in scorers, all tested and documented.
+
+---
+
+## Phase 5 — Storage, history, and the showcase example
+
+**Goal:** runs persist and can be compared; there is a realistic example to demo.
+**Estimate:** 14 hours.
+
+- [ ] **5.1 Design the SQLite schema** in `storage.py` with a `schema_version` table and sequential migrations under `src/evalstand/migrations/`:
+
+  ```sql
+  runs(id, name, git_sha, git_dirty, started_at, finished_at,
+       model_config_json, repeat_n, total_cases, total_cost_usd, status)
+  results(id, run_id, case_id, repeat_index, output_text, output_json,
+          latency_ms, input_tokens, output_tokens, cost_usd, error)
+  scores(id, result_id, scorer_name, value_float, passed, metadata_json)
+  traces(id, result_id, parent_id, name, started_at, duration_ms,
+         input_json, output_json, model, tokens_json, cost_usd)
+  case_snapshots(run_id, case_id, input_json, expected_json, metadata_json)
+  cache(key, model, response_json, created_at, hit_count)
+  ```
+  Snapshot cases per run so history stays valid when the dataset changes later.
+  - *Acceptance:* the migration applies to an empty database and is idempotent; a second run does not corrupt the first.
+- [ ] **5.2 Record provenance** on every run: git SHA, dirty-tree flag, model config, and a hash of the task source. Refuse to persist without a SHA unless `--allow-dirty` is passed.
+- [ ] **5.3 Build `evalstand history [name]`** listing runs with name, SHA, date, mean score, pass count, and cost.
+- [ ] **5.4 Build `evalstand show <run_id>`** rendering a full run: summary, per-case scores, and trace trees.
+- [ ] **5.5 Build `evalstand compare <run_a> <run_b>`.** Report per-scorer means for both runs, the delta, and the list of cases whose pass state flipped, with old and new output side by side.
+  - **Important:** report the delta as a plain difference. Do **not** label it a regression or an improvement — this version has no significance testing, and asserting a verdict without one would be a false claim. Say "changed" and show the flipped cases. `docs/ci.md` must state this limitation explicitly.
+  - *Acceptance:* comparing two runs shows the delta and flipped cases with no verdict language.
+- [ ] **5.6 Build the showcase example** at `examples/pdf_extraction/`: extract `invoice_number`, `vendor_name`, `invoice_date`, `total`, and `line_items[]` from documents.
+  - Generate 30 synthetic invoices with `reportlab` + `faker` at a fixed seed. Ground truth is written at generation time and is therefore true by construction — programmatic, not LLM-generated.
+  - Vary deliberately: multi-page documents, two currencies, a missing due date, an ambiguous date format.
+  - Use `json_field` and `numeric_tolerance` scorers plus one judge scorer for line-item completeness.
+  - *Acceptance:* `python generate.py --seed 42` reproduces byte-identical PDFs and golden JSON; the eval runs end to end from a clean clone with one API key set.
+- [ ] **5.7 Commit baseline results** as `examples/pdf_extraction/BASELINE.md` with per-field accuracy, cost per document, and observed failure modes.
+
+**Exit criteria:** runs persist, history and comparison work, the showcase example runs from a clean clone.
+
+---
+
+## Phase 6 — TUI and watch mode
+
+**Goal:** the live feedback loop that is the whole point of the tool.
+**Estimate:** 16 hours.
+
+- [ ] **6.1 Run view** in `tui/app.py`: header with eval name and model, progress bar, a streaming table of cases (id, status, score, latency, cost) updating as results land, and a footer with running totals.
+  - *Acceptance:* rows appear incrementally, not in one batch at the end.
+- [ ] **6.2 Summary panel:** per-scorer mean, pass count, total cost, wall time, cache hit rate.
+- [ ] **6.3 Case detail view:** press `enter` on a row for input, full output, expected, per-scorer breakdown with metadata, and the **trace tree** with per-node model, duration, tokens, and cost.
+  - *Acceptance:* a task with nested LLM calls renders an expandable, navigable trace tree.
+- [ ] **6.4 History view:** browse past runs, select one to open, select two to render the Phase 5 comparison.
+- [ ] **6.5 Custom columns.** Support a `columns=` argument on `evaluate()` letting the user add derived columns to the results table.
+  - *Acceptance:* the showcase example adds a "fields correct" column.
+- [ ] **6.6 Watch mode** with `watchfiles`: re-run affected evals when an eval file, task file, or prompt file changes. Debounce 300ms. Preserve scroll position and show a "changed: <file>" indicator.
+  - *Acceptance:* editing a prompt triggers a re-run within one second without restarting the process.
+- [ ] **6.7 Keybindings:** `q` quit, `r` re-run, `f` filter to failures, `c` compare with previous run, `/` search, `y` copy case id.
+- [ ] **6.8 Record a demo GIF** with `vhs` or `asciinema` + `agg`, embedded at the top of the README.
+  - *Acceptance:* under 5 MB, showing a full run, a trace tree, and watch-mode re-run in under 30 seconds.
+
+**Exit criteria:** the TUI satisfies every UI row in the Section 2 table. The README opens with a GIF that makes the value obvious in five seconds.
+
+---
+
+## Phase 7 — CI integration, docs, and release
+
+**Goal:** installable, usable in a pipeline, and understandable.
+**Estimate:** 12 hours.
+
+- [ ] **7.1 CI flags:** `--threshold <float>` (fail when the mean score falls below it) and `--fail-on-error`. Documented exit codes: `0` pass, `1` below threshold, `2` execution error.
+  - *Acceptance:* an exit-code table in the docs, each code reproducible in a test.
+- [ ] **7.2 Markdown summary output** in `reporting/markdown.py` — `--output markdown` produces a body suitable for a PR comment: summary table, failed cases, cost.
+- [ ] **7.3 Document the GitHub Actions recipe** in `docs/ci.md`: a workflow that runs evals on pull requests, posts the markdown summary as a comment, and gates on the threshold. Ship it as a copyable YAML block rather than a published Action in this version.
+- [ ] **7.4 Docs site** with mkdocs-material: quickstart, writing evals, scorers, traces, CI, architecture, ADR index. Deploy to GitHub Pages.
+- [ ] **7.5 Rewrite the README:** one-sentence problem statement, demo GIF, 60-second quickstart, feature list mapped to the Section 2 parity table, reproducible numbers from the showcase example, an honest **Limitations** section, the attribution footnote, licence.
+  - The Limitations section is required. State plainly that score deltas are reported without significance testing and that LLM judge scorers are unvalidated in this version.
+- [ ] **7.6 Publish to PyPI** via a tagged `release.yml` using trusted publishing. Tag `v1.0.0`.
+- [ ] **7.7 Record a 3-minute demo video:** write an eval, run it in watch mode, edit the prompt, watch the re-run, open a trace tree, break something and see the threshold gate fail in CI.
+
+**Exit criteria:** installable from PyPI, docs live, video recorded, parity table fully satisfied.
+
+---
+
+## Phase 8 — Optional stretch: web UI
+
+Start only when Phases 0–7 are complete and polished. A shipped TUI beats a half-finished web app.
+
+- [ ] **8.1** FastAPI backend exposing runs, results, scores, and traces as JSON.
+- [ ] **8.2** HTMX plus server-sent events front end for the live run table, keeping the stack entirely Python.
+- [ ] **8.3** Static HTML report export for CI artifacts. Worth doing even without the full web UI.
+- [ ] **8.4** `evalstand serve` command.
+
+---
+
+## 8. Parked for a future version
+
+Recorded here so the ideas are not lost, and so the agent does not build them now. **Do not start any of these until Phase 7 is complete and released.**
+
+- **Statistical significance testing.** Bootstrap confidence intervals on aggregate scores and McNemar's exact test for paired run-over-run comparisons, so `compare` can distinguish a real change from sampling noise instead of reporting a bare delta. This is the single most valuable extension and the natural v2.
+- **Judge calibration.** Measuring an LLM judge against human labels — Cohen's kappa, true and false positive rates reported separately, position bias, verbosity bias, self-preference bias — and refusing to run an uncalibrated judge.
+- **Dataset hygiene.** Near-duplicate detection within an eval set and overlap detection against few-shot examples.
+- **A published GitHub Action** rather than a copyable workflow.
+
+---
+
+## 9. Definition of done
+
+A phase is complete only when all of the following hold:
+
+1. `ruff check`, `ruff format --check`, `mypy`, and `pytest` pass locally and in CI.
+2. Test coverage for new modules is at or above 80%.
+3. CI runs with no API keys set.
+4. Every new design decision has an ADR.
+5. Public API changes are reflected in the docs.
+6. `CHANGELOG.md` is updated.
+7. The Section 2 parity table is updated to reflect what now works.
+
+---
+
+## 10. Risk register
+
+| Risk | Mitigation |
+| --- | --- |
+| API costs during development | Cache aggressively from Phase 1; develop against the cheapest available model; set a monthly budget alert. Expect $20–60 total for this project. |
+| Tracing turns out to be the hard part | It is. Budget the full Phase 3 allocation for it and build it against the toy example before the showcase example exists. `contextvars` behaviour under `asyncio.gather` is the specific thing to test early. |
+| pytest plugin fights the runner's async model | Prototype collection and async dispatch together in Phase 2 rather than sequentially. If the plugin proves intractable, fall back to a standalone runner and keep pytest compatibility as a stretch — but record it as an ADR, since parity item 19 would be waived. |
+| Scope creep into the parked v2 features | Section 8 exists for exactly this. Anything in it goes back in the parking lot. |
+| TUI consumes the whole budget | Timebox Phase 6 to 16 hours. If it overruns, ship console reporting only and move the TUI to a stretch phase. |
+| The tool ends up a shallow imitation | Parity item 7 (traces) and item 10 (repeats) are the two features that are genuinely hard. If those work well, the tool is real. If they are skipped, it is a wrapper. |

evalstand-0.0.0.dev0/README.md

@@ -0,0 +1,77 @@
+# evalstand
+
+> **Status: in development.** Phase 0 of 7. This release is a **name
+> reservation placeholder** and contains no working code. The example below
+> shows the intended API, which is not implemented yet.
+
+Evaluating an LLM application should feel like running a test suite.
+
+`evalstand` is a local-first LLM evaluation tool for Python. You write an eval
+file, run a watch command, and results stream into a live terminal UI — scores,
+nested call traces, token counts, latency, and cost. Everything runs on your
+machine and persists to a local SQLite database, so you can compare a run
+against the one before it.
+
+```python
+from evalstand import Case, evaluate
+from evalstand.scorers import exact, levenshtein
+
+
+def load_cases() -> list[Case]:
+    return [
+        Case(id="q1", input="What is the capital of France?", expected="Paris"),
+        Case(id="q2", input="What is 2 + 2?", expected="4"),
+    ]
+
+
+async def answer(question: str) -> str:
+    resp = await llm.acall("gpt-4o-mini", [{"role": "user", "content": question}])
+    return resp.text
+
+
+evaluate(
+    name="basic-qa",
+    cases=load_cases,
+    task=answer,
+    scorers=[exact, levenshtein],
+)
+```
+
+Save that as `qa_eval.py` and run it either way:
+
+```bash
+evalstand run qa_eval.py     # live TUI, watch mode, traces
+pytest qa_eval.py            # plain test runner, CI-friendly
+```
+
+## Why
+
+Existing Python options are either heavyweight platforms that push you toward a
+hosted service, or bare metric libraries with no runner, no persistence, and no
+live feedback loop. `evalstand` is the middle: a real runner with a real UI that
+stays on your machine.
+
+## Planned capabilities
+
+See [PLAN.md](PLAN.md) for the full build plan and the capability checklist that
+defines v1.
+
+## Limitations
+
+Stated up front, and kept accurate as the project grows:
+
+- Score differences between runs are reported as plain deltas. There is **no
+  statistical significance testing** in v1, so a delta is not evidence of a real
+  regression or improvement.
+- LLM-as-judge scorers are **unvalidated** — they have not been calibrated
+  against human labels.
+
+## Licence
+
+MIT. See [LICENSE](LICENSE).
+
+---
+
+<sub>Inspired by [evalite](https://github.com/mattpocock/evalite) (MIT), which
+showed that local LLM evals could feel like running tests. `evalstand` is an
+independent Python implementation.</sub>

evalstand-0.0.0.dev0/docs/adr/0000-adr-process.md

@@ -0,0 +1,46 @@
+# 0000 — ADR process
+
+- **Status:** accepted
+- **Date:** 2026-09-01
+
+## Context
+
+This project makes design decisions that are not obvious from the code, and it
+is built over several weeks. Without a record, the reasoning behind a decision
+is lost and gets relitigated.
+
+## Decision
+
+Every non-obvious decision gets an ADR in `docs/adr/NNNN-title.md`, numbered
+sequentially. Each records **context**, **decision**, and **consequences**.
+Keep them short — a screen or less. An ADR is never edited to change its
+decision; it is superseded by a later ADR that references it.
+
+An ADR is required for:
+
+- Adding a dependency outside the approved list in `PLAN.md` Section 4.
+- Waiving or changing a row in the Section 2 capability table.
+- Any architectural choice a future contributor would reasonably question.
+
+## Consequences
+
+Slight overhead per decision. In exchange, the reasoning survives, and the
+"why is it like this?" question has an answer that is not a guess.
+
+## Template
+
+```markdown
+# NNNN — Title
+
+- **Status:** proposed | accepted | superseded by [NNNN](NNNN-title.md)
+- **Date:** YYYY-MM-DD
+
+## Context
+What forced the decision.
+
+## Decision
+What we chose.
+
+## Consequences
+What this makes easy, and what it costs.
+```

evalstand-0.0.0.dev0/docs/adr/0001-name.md

@@ -0,0 +1,57 @@
+# 0001 — Project name: `evalstand`
+
+- **Status:** accepted
+- **Date:** 2026-09-01
+
+## Context
+
+The project needed a public name before scaffolding, because the name
+propagates into `pyproject.toml`, the `src/` layout, the CLI command, the
+import path, and every document.
+
+Two constraints applied. First, the name had to be genuinely free — on PyPI and
+on GitHub — so the package could be published without a rename later. Second,
+it had to be the project's own name. The working title `pyevalite` prefixed
+another project's name, which reads as a fork and positions the work as
+derivative rather than independent.
+
+Availability was checked against the PyPI JSON API (`/pypi/<name>/json`,
+where HTTP 404 means free) and the GitHub repository search API.
+
+Candidates rejected because the name was already taken on PyPI:
+
+| Name | Status |
+| --- | --- |
+| `evalrig` | taken — 0.0.1, "A test rig for AI evaluations" (2026-02-09) |
+| `proofmark` | taken — 0.1.1, "Structured LLM output that has been tested" (2026-08-17) |
+| `proofbench` | taken — config-driven eval harness |
+| `evalforge` | taken — LLM agent evaluation harness |
+| `evalloop` | taken — closed-loop eval monitoring |
+| `rubricon` | taken — specification-first generation for LLMs |
+| `evalbench`, `assay`, `plumbline`, `rigor`, `tessera` | taken — various |
+
+The first six are in this project's own problem space, which makes them
+unusable regardless of squatting status: users would confuse the packages.
+
+## Decision
+
+The project is named **`evalstand`**.
+
+Verified on 2026-09-01: PyPI returns 404, and GitHub repository search returns
+0 name matches.
+
+The name refers to a *test stand* — the rig an engineer mounts an engine on to
+measure it under controlled conditions. That is what this tool is for.
+
+**Binding naming rule:** the string `evalite` must not appear in any package
+name, module name, directory name, filename, class name, function name, or CLI
+command. A single attribution footnote at the bottom of the README is the only
+permitted occurrence in the shipped repository.
+
+## Consequences
+
+- The package, CLI command, and import path are all `evalstand`.
+- The name is unclaimed but unregistered — reserve it on PyPI early to avoid
+  losing it, since names in this space are being claimed quickly.
+- The name does not signal "LLM" on its own; the description and README carry
+  that.

evalstand-0.0.0.dev0/docs/adr/0002-design-scope.md

@@ -0,0 +1,36 @@
+# 0002 — Design scope
+
+- **Status:** proposed (blocked on task 0.6)
+- **Date:** 2026-09-01
+
+## Context
+
+`evalstand` is an independent Python implementation of an idea proven in
+TypeScript by `evalite` (MIT): that local LLM evaluation should feel like
+running a test suite.
+
+This ADR records which behaviours `evalstand` adopts, which it does
+differently, and why. **It cannot be completed until task 0.6 is done** —
+studying the reference implementation's entry function, CLI, SQLite layer, and
+web UI. Filling it in before that would be guesswork.
+
+Two decisions are already settled and recorded here:
+
+## Decision (partial)
+
+1. **Textual TUI instead of a React web UI.** The reference ships a React web
+   app. `evalstand` ships a terminal UI first, with a web UI as an optional
+   stretch phase. Rationale: the TUI reaches functional parity faster, keeps
+   the entire stack in Python, and demos well in a terminal recording.
+
+2. **pytest instead of Vitest** as the collection and execution substrate,
+   since that is the Python equivalent and gives bare-`pytest` compatibility
+   for free.
+
+3. **Study for design understanding, not code.** The reference is read to
+   understand behaviour and interface decisions. No code is copied. Where
+   TypeScript idiom does not carry into Python, the Python equivalent wins.
+
+## Consequences
+
+To be completed alongside the rest of this ADR after task 0.6.

evalstand-0.0.0.dev0/docs/ci.md

@@ -0,0 +1,3 @@
+# ci
+
+To be written.

evalstand-0.0.0.dev0/docs/index.md

@@ -0,0 +1,3 @@
+# index
+
+To be written.

evalstand-0.0.0.dev0/docs/quickstart.md

@@ -0,0 +1,3 @@
+# quickstart
+
+To be written.

evalstand-0.0.0.dev0/docs/scorers.md

@@ -0,0 +1,3 @@
+# scorers
+
+To be written.

evalstand-0.0.0.dev0/docs/traces.md

@@ -0,0 +1,3 @@
+# traces
+
+To be written.

evalstand-0.0.0.dev0/docs/writing-evals.md

@@ -0,0 +1,3 @@
+# writing-evals
+
+To be written.

evalstand-0.0.0.dev0/pyproject.toml

@@ -0,0 +1,90 @@
+[project]
+name = "evalstand"
+version = "0.0.0.dev0"
+description = "A local-first LLM evaluation tool for Python: write evals, run them like tests, watch results stream in."
+readme = "README.md"
+license = { file = "LICENSE" }
+requires-python = ">=3.11"
+keywords = ["llm", "evaluation", "eval", "testing", "pytest", "ai"]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "litellm>=1.55",
+    "typer>=0.15",
+    "rich>=13.9",
+    "textual>=1.0",
+    "watchfiles>=1.0",
+    "pydantic>=2.10",
+    "rapidfuzz>=3.11",
+    "tenacity>=9.0",
+    "pytest>=8.3",
+]
+
+[project.optional-dependencies]
+examples = ["reportlab>=4.2", "faker>=33.1", "pypdfium2>=4.30"]
+
+[project.scripts]
+evalstand = "evalstand.cli:app"
+
+[project.entry-points.pytest11]
+evalstand = "evalstand.plugin"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.9",
+    "mypy>=1.14",
+    "pytest-cov>=6.0",
+    "pre-commit>=4.0",
+    "mkdocs-material>=9.5",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evalstand"]
+
+[tool.hatch.build.targets.sdist]
+# Ship source and docs; leave development scaffolding out of the distribution.
+include = [
+    "/src",
+    "/tests",
+    "/docs",
+    "/examples",
+    "/README.md",
+    "/LICENSE",
+    "/CHANGELOG.md",
+    "/PLAN.md",
+    "/pyproject.toml",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "UP", "B", "A", "C4", "SIM", "RUF"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+files = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=evalstand --cov-report=term-missing"
+markers = [
+    "live: hits a real provider API; excluded from CI (deselect with -m 'not live')",
+]
+
+[tool.coverage.report]
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]

evalstand-0.0.0.dev0/src/evalstand/__init__.py

@@ -0,0 +1,12 @@
+"""evalstand — a local-first LLM evaluation tool for Python.
+
+Evaluating an LLM application should feel like running a test suite: write an
+eval file, run a watch command, and watch results stream in with scores,
+traces, token counts, and cost.
+
+The public API is deliberately small. Keep it under 10 exported names.
+"""
+
+__version__ = "0.0.0.dev0"
+
+__all__ = ["__version__"]

evalstand-0.0.0.dev0/src/evalstand/api.py

@@ -0,0 +1 @@
+"""evalstand.api — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/cache.py

@@ -0,0 +1 @@
+"""evalstand.cache — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/cli.py

@@ -0,0 +1 @@
+"""evalstand.cli — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/config.py

@@ -0,0 +1 @@
+"""evalstand.config — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/llm.py

@@ -0,0 +1 @@
+"""evalstand.llm — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/migrations/__init__.py

@@ -0,0 +1 @@
+"""Migrations for the evalstand SQLite store (Phase 5)."""

evalstand-0.0.0.dev0/src/evalstand/models.py

@@ -0,0 +1 @@
+"""evalstand.models — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/plugin.py

@@ -0,0 +1 @@
+"""evalstand.plugin — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/reporting/console.py

@@ -0,0 +1 @@
+"""evalstand.reporting.console — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/reporting/markdown.py

@@ -0,0 +1 @@
+"""evalstand.reporting.markdown — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/runner.py

@@ -0,0 +1 @@
+"""evalstand.runner — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/scorers/base.py

@@ -0,0 +1 @@
+"""evalstand.scorers.base — implemented in Phase 4."""

evalstand-0.0.0.dev0/src/evalstand/scorers/fuzzy.py

@@ -0,0 +1 @@
+"""evalstand.scorers.fuzzy — implemented in Phase 4."""

evalstand-0.0.0.dev0/src/evalstand/scorers/json_field.py

@@ -0,0 +1 @@
+"""evalstand.scorers.json_field — implemented in Phase 4."""

evalstand-0.0.0.dev0/src/evalstand/scorers/llm.py

@@ -0,0 +1 @@
+"""evalstand.scorers.llm — implemented in Phase 4."""

evalstand-0.0.0.dev0/src/evalstand/scorers/numeric.py

@@ -0,0 +1 @@
+"""evalstand.scorers.numeric — implemented in Phase 4."""

evalstand-0.0.0.dev0/src/evalstand/scorers/string.py

@@ -0,0 +1 @@
+"""evalstand.scorers.string — implemented in Phase 4."""

evalstand-0.0.0.dev0/src/evalstand/storage.py

@@ -0,0 +1 @@
+"""evalstand.storage — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/tracing.py

@@ -0,0 +1 @@
+"""evalstand.tracing — implemented in a later phase."""

evalstand-0.0.0.dev0/src/evalstand/tui/app.py

@@ -0,0 +1 @@
+"""evalstand.tui.app — implemented in Phase 6."""

evalstand-0.0.0.dev0/tests/unit/test_package.py

@@ -0,0 +1,7 @@
+"""Phase 0 smoke test: the package imports and reports a version."""
+
+import evalstand
+
+
+def test_package_imports() -> None:
+    assert evalstand.__version__