evalstand 0.0.0.dev0__py3-none-any.whl

evalstand/__init__.py

@@ -0,0 +1,12 @@
+"""evalstand — a local-first LLM evaluation tool for Python.
+
+Evaluating an LLM application should feel like running a test suite: write an
+eval file, run a watch command, and watch results stream in with scores,
+traces, token counts, and cost.
+
+The public API is deliberately small. Keep it under 10 exported names.
+"""
+
+__version__ = "0.0.0.dev0"
+
+__all__ = ["__version__"]

evalstand/api.py

@@ -0,0 +1 @@
+"""evalstand.api — implemented in a later phase."""

evalstand/cache.py

@@ -0,0 +1 @@
+"""evalstand.cache — implemented in a later phase."""

evalstand/cli.py

@@ -0,0 +1 @@
+"""evalstand.cli — implemented in a later phase."""

evalstand/config.py

@@ -0,0 +1 @@
+"""evalstand.config — implemented in a later phase."""

evalstand/llm.py

@@ -0,0 +1 @@
+"""evalstand.llm — implemented in a later phase."""

evalstand/migrations/__init__.py

@@ -0,0 +1 @@
+"""Migrations for the evalstand SQLite store (Phase 5)."""

evalstand/models.py

@@ -0,0 +1 @@
+"""evalstand.models — implemented in a later phase."""

evalstand/plugin.py

@@ -0,0 +1 @@
+"""evalstand.plugin — implemented in a later phase."""

evalstand/reporting/console.py

@@ -0,0 +1 @@
+"""evalstand.reporting.console — implemented in a later phase."""

evalstand/reporting/markdown.py

@@ -0,0 +1 @@
+"""evalstand.reporting.markdown — implemented in a later phase."""

evalstand/runner.py

@@ -0,0 +1 @@
+"""evalstand.runner — implemented in a later phase."""

evalstand/scorers/base.py

@@ -0,0 +1 @@
+"""evalstand.scorers.base — implemented in Phase 4."""

evalstand/scorers/fuzzy.py

@@ -0,0 +1 @@
+"""evalstand.scorers.fuzzy — implemented in Phase 4."""

evalstand/scorers/json_field.py

@@ -0,0 +1 @@
+"""evalstand.scorers.json_field — implemented in Phase 4."""

evalstand/scorers/llm.py

@@ -0,0 +1 @@
+"""evalstand.scorers.llm — implemented in Phase 4."""

evalstand/scorers/numeric.py

@@ -0,0 +1 @@
+"""evalstand.scorers.numeric — implemented in Phase 4."""

evalstand/scorers/string.py

@@ -0,0 +1 @@
+"""evalstand.scorers.string — implemented in Phase 4."""

evalstand/storage.py

@@ -0,0 +1 @@
+"""evalstand.storage — implemented in a later phase."""

evalstand/tracing.py

@@ -0,0 +1 @@
+"""evalstand.tracing — implemented in a later phase."""

evalstand/tui/app.py

@@ -0,0 +1 @@
+"""evalstand.tui.app — implemented in Phase 6."""

evalstand-0.0.0.dev0.dist-info/METADATA

@@ -0,0 +1,127 @@
+Metadata-Version: 2.5
+Name: evalstand
+Version: 0.0.0.dev0
+Summary: A local-first LLM evaluation tool for Python: write evals, run them like tests, watch results stream in.
+License: MIT License
+        
+        Copyright (c) 2026 evalstand contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: ai,eval,evaluation,llm,pytest,testing
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: litellm>=1.55
+Requires-Dist: pydantic>=2.10
+Requires-Dist: pytest>=8.3
+Requires-Dist: rapidfuzz>=3.11
+Requires-Dist: rich>=13.9
+Requires-Dist: tenacity>=9.0
+Requires-Dist: textual>=1.0
+Requires-Dist: typer>=0.15
+Requires-Dist: watchfiles>=1.0
+Provides-Extra: examples
+Requires-Dist: faker>=33.1; extra == 'examples'
+Requires-Dist: pypdfium2>=4.30; extra == 'examples'
+Requires-Dist: reportlab>=4.2; extra == 'examples'
+Description-Content-Type: text/markdown
+
+# evalstand
+
+> **Status: in development.** Phase 0 of 7. This release is a **name
+> reservation placeholder** and contains no working code. The example below
+> shows the intended API, which is not implemented yet.
+
+Evaluating an LLM application should feel like running a test suite.
+
+`evalstand` is a local-first LLM evaluation tool for Python. You write an eval
+file, run a watch command, and results stream into a live terminal UI — scores,
+nested call traces, token counts, latency, and cost. Everything runs on your
+machine and persists to a local SQLite database, so you can compare a run
+against the one before it.
+
+```python
+from evalstand import Case, evaluate
+from evalstand.scorers import exact, levenshtein
+
+
+def load_cases() -> list[Case]:
+    return [
+        Case(id="q1", input="What is the capital of France?", expected="Paris"),
+        Case(id="q2", input="What is 2 + 2?", expected="4"),
+    ]
+
+
+async def answer(question: str) -> str:
+    resp = await llm.acall("gpt-4o-mini", [{"role": "user", "content": question}])
+    return resp.text
+
+
+evaluate(
+    name="basic-qa",
+    cases=load_cases,
+    task=answer,
+    scorers=[exact, levenshtein],
+)
+```
+
+Save that as `qa_eval.py` and run it either way:
+
+```bash
+evalstand run qa_eval.py     # live TUI, watch mode, traces
+pytest qa_eval.py            # plain test runner, CI-friendly
+```
+
+## Why
+
+Existing Python options are either heavyweight platforms that push you toward a
+hosted service, or bare metric libraries with no runner, no persistence, and no
+live feedback loop. `evalstand` is the middle: a real runner with a real UI that
+stays on your machine.
+
+## Planned capabilities
+
+See [PLAN.md](PLAN.md) for the full build plan and the capability checklist that
+defines v1.
+
+## Limitations
+
+Stated up front, and kept accurate as the project grows:
+
+- Score differences between runs are reported as plain deltas. There is **no
+  statistical significance testing** in v1, so a delta is not evidence of a real
+  regression or improvement.
+- LLM-as-judge scorers are **unvalidated** — they have not been calibrated
+  against human labels.
+
+## Licence
+
+MIT. See [LICENSE](LICENSE).
+
+---
+
+<sub>Inspired by [evalite](https://github.com/mattpocock/evalite) (MIT), which
+showed that local LLM evals could feel like running tests. `evalstand` is an
+independent Python implementation.</sub>

evalstand-0.0.0.dev0.dist-info/RECORD

@@ -0,0 +1,30 @@
+evalstand/__init__.py,sha256=bkTRSFI2lelAlq5XqN9RnM38tEsPM0iK5JRDytSt7q0,378
+evalstand/api.py,sha256=bfvKl_Yv1ET9XFQ7Zlu6UB7CIF7elm0tlEt0WaiYRS0,54
+evalstand/cache.py,sha256=7E7d8XKpeDNldtAkt59jAaJqwotZEgLp3WHU52nne-c,56
+evalstand/cli.py,sha256=CQpWMVDlJ8BbAN4xarhyvIEnKYG2Ux9sLEqOsSH65xE,54
+evalstand/config.py,sha256=w9vGoRmh4lBEN06mJ8G_TtAtc7lw-cwRuWHbl3PXnBY,57
+evalstand/llm.py,sha256=94gbGTn9ryRY1dVM30vbVESvTls-fmlgQiMgl5h3A1M,54
+evalstand/models.py,sha256=4d3w324Wk_SzMK06qUBMsZbN5m6cXDOdZZ1Nmp4Df60,57
+evalstand/plugin.py,sha256=Q72BYBOzIgHMDx12ydfcpgPsV3ol_IAMQsvXmYkSUkI,57
+evalstand/runner.py,sha256=-3Yq0JPtl9TvGuqWLfsrO7AnE7a1VS5JpadMhz518U8,57
+evalstand/storage.py,sha256=Tfi2n_RqVHl06SjO7fwEULu6zNilEavvngIc9QViRak,58
+evalstand/tracing.py,sha256=GzF9ZtsQgIlx1zGek7arRmphcBDAAKZsOc-v3i2bIvY,58
+evalstand/migrations/__init__.py,sha256=I4F4NdLke4ao7fIAMAGZtOEhEmhrTG6QfMRKLmlorn4,59
+evalstand/reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalstand/reporting/console.py,sha256=ShXMSLUo28EwiK1tKkGbF-ZVDr2E1KpdG0fmcJbJKVg,68
+evalstand/reporting/markdown.py,sha256=1sSxI6H-gNByDaRGIGl3os3klT6lxIbpFjBACqlUk4U,69
+evalstand/scorers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalstand/scorers/base.py,sha256=vJCzRx74ZPoNJnb4HI3KskhJUF2oDdF20RXrvi6PF-g,57
+evalstand/scorers/fuzzy.py,sha256=WQfE0L6aa_DJJFCGbyio4PDSgSGwAq3RI7eCcPATP9Q,58
+evalstand/scorers/json_field.py,sha256=tlt8mAbtsG_4VWUDw7vyruiJCaToahtTWz7PDK8d5uM,63
+evalstand/scorers/llm.py,sha256=tRAmy-QwpkuljZ6NAp4kPy2SUOlysS0NTcJFfjdXdM8,56
+evalstand/scorers/numeric.py,sha256=OUfFaxCDWs53nUy-ouSQCCJvlAqZTfZ3vS339NhP8Mw,60
+evalstand/scorers/string.py,sha256=9O3gtvUCoODWiLTVGVtxkWA03dvUaxAo_iGUiKmqBes,59
+evalstand/tui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalstand/tui/app.py,sha256=VD7dBE72hsO6r5YwWS9sXGz1iiMbqQgaINujf-Jqg3Q,52
+evalstand/tui/widgets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalstand-0.0.0.dev0.dist-info/METADATA,sha256=l12I9HKSiO5JkkubEbXie2MjOlHBnq8HcOq89HnqInk,4697
+evalstand-0.0.0.dev0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+evalstand-0.0.0.dev0.dist-info/entry_points.txt,sha256=03shVFLd5RaIlU19bp3j9Kb8mo0HV4HRKybKD4UcNS4,89
+evalstand-0.0.0.dev0.dist-info/licenses/LICENSE,sha256=a076uNC8pnFdDmJIjHiL2eB5bCiJG9-PsOWMicW_Ytw,1079
+evalstand-0.0.0.dev0.dist-info/RECORD,,

evalstand-0.0.0.dev0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

evalstand-0.0.0.dev0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+evalstand = evalstand.cli:app
+
+[pytest11]
+evalstand = evalstand.plugin

evalstand-0.0.0.dev0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 evalstand contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.