evalstone 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalstone-0.1.0/.gitignore +44 -0
- evalstone-0.1.0/CHANGELOG.md +47 -0
- evalstone-0.1.0/LICENSE +21 -0
- evalstone-0.1.0/PKG-INFO +125 -0
- evalstone-0.1.0/README.md +88 -0
- evalstone-0.1.0/justfile +37 -0
- evalstone-0.1.0/pyproject.toml +73 -0
- evalstone-0.1.0/src/evalstone/__init__.py +131 -0
- evalstone-0.1.0/src/evalstone/__main__.py +3 -0
- evalstone-0.1.0/src/evalstone/_git.py +22 -0
- evalstone-0.1.0/src/evalstone/_json.py +59 -0
- evalstone-0.1.0/src/evalstone/checks/__init__.py +41 -0
- evalstone-0.1.0/src/evalstone/checks/_base.py +153 -0
- evalstone-0.1.0/src/evalstone/checks/comparison.py +154 -0
- evalstone-0.1.0/src/evalstone/checks/deterministic.py +279 -0
- evalstone-0.1.0/src/evalstone/checks/human.py +124 -0
- evalstone-0.1.0/src/evalstone/checks/judge.py +219 -0
- evalstone-0.1.0/src/evalstone/checks/rag.py +70 -0
- evalstone-0.1.0/src/evalstone/checks/safety.py +131 -0
- evalstone-0.1.0/src/evalstone/checks/trajectory.py +144 -0
- evalstone-0.1.0/src/evalstone/cli.py +411 -0
- evalstone-0.1.0/src/evalstone/config.py +267 -0
- evalstone-0.1.0/src/evalstone/dataset.py +184 -0
- evalstone-0.1.0/src/evalstone/gate.py +84 -0
- evalstone-0.1.0/src/evalstone/mcp_server.py +329 -0
- evalstone-0.1.0/src/evalstone/output.py +36 -0
- evalstone-0.1.0/src/evalstone/providers/__init__.py +185 -0
- evalstone-0.1.0/src/evalstone/providers/_http.py +58 -0
- evalstone-0.1.0/src/evalstone/providers/anthropic_.py +81 -0
- evalstone-0.1.0/src/evalstone/providers/gemini.py +55 -0
- evalstone-0.1.0/src/evalstone/providers/openai_compat.py +67 -0
- evalstone-0.1.0/src/evalstone/pytest_plugin.py +203 -0
- evalstone-0.1.0/src/evalstone/run.py +336 -0
- evalstone-0.1.0/src/evalstone/score.py +124 -0
- evalstone-0.1.0/src/evalstone/suite.py +135 -0
- evalstone-0.1.0/src/evalstone/target.py +289 -0
- evalstone-0.1.0/tests/conftest.py +13 -0
- evalstone-0.1.0/tests/test_action.py +108 -0
- evalstone-0.1.0/tests/test_cli.py +145 -0
- evalstone-0.1.0/tests/test_config.py +211 -0
- evalstone-0.1.0/tests/test_dataset.py +168 -0
- evalstone-0.1.0/tests/test_deterministic.py +204 -0
- evalstone-0.1.0/tests/test_docs.py +33 -0
- evalstone-0.1.0/tests/test_human.py +159 -0
- evalstone-0.1.0/tests/test_judge.py +251 -0
- evalstone-0.1.0/tests/test_mcp.py +280 -0
- evalstone-0.1.0/tests/test_providers.py +245 -0
- evalstone-0.1.0/tests/test_pytest_plugin.py +245 -0
- evalstone-0.1.0/tests/test_rag_and_comparison.py +243 -0
- evalstone-0.1.0/tests/test_safety.py +96 -0
- evalstone-0.1.0/tests/test_score_and_checks.py +150 -0
- evalstone-0.1.0/tests/test_suite_run_gate.py +233 -0
- evalstone-0.1.0/tests/test_target.py +134 -0
- evalstone-0.1.0/tests/test_trajectory.py +84 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# ── Secrets / local env ────────────────────────────────────────────────
|
|
2
|
+
**/.env
|
|
3
|
+
**/.env.local
|
|
4
|
+
**/.env.*.local
|
|
5
|
+
**/.env.production
|
|
6
|
+
!**/.env.example
|
|
7
|
+
|
|
8
|
+
# ── OS / editor ────────────────────────────────────────────────────────
|
|
9
|
+
.DS_Store
|
|
10
|
+
*.pem
|
|
11
|
+
*.swp
|
|
12
|
+
*~
|
|
13
|
+
**/certs/
|
|
14
|
+
|
|
15
|
+
# ── Python (python/) ───────────────────────────────────────────────────
|
|
16
|
+
**/venv/
|
|
17
|
+
**/.venv/
|
|
18
|
+
**/__pycache__/
|
|
19
|
+
*.pyc
|
|
20
|
+
**/.pytest_cache/
|
|
21
|
+
**/.ruff_cache/
|
|
22
|
+
**/.mypy_cache/
|
|
23
|
+
*.egg-info/
|
|
24
|
+
**/dist/
|
|
25
|
+
.coverage
|
|
26
|
+
htmlcov/
|
|
27
|
+
|
|
28
|
+
# ── Node / Next.js (javascript/) ───────────────────────────────────────
|
|
29
|
+
**/node_modules/
|
|
30
|
+
**/.next/
|
|
31
|
+
**/.source/
|
|
32
|
+
**/out/
|
|
33
|
+
**/build/
|
|
34
|
+
**/coverage/
|
|
35
|
+
*.tsbuildinfo
|
|
36
|
+
next-env.d.ts
|
|
37
|
+
**/test-results/
|
|
38
|
+
**/playwright-report/
|
|
39
|
+
**/playwright/.cache/
|
|
40
|
+
**/logs/
|
|
41
|
+
npm-debug.log*
|
|
42
|
+
|
|
43
|
+
# ── Evalstone run reports (written by `evalstone run`) ──────────────────
|
|
44
|
+
**/evalstone-report.json
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Changelog — evalstone (the library)
|
|
2
|
+
|
|
3
|
+
All notable changes to `python/evalstone`. Semantic versioning; tags `lib-vX.Y.Z`.
|
|
4
|
+
|
|
5
|
+
## 0.1.0 — 2026-10-13
|
|
6
|
+
|
|
7
|
+
The first usable release: the hero path from `evalstone.yaml` to a gated run.
|
|
8
|
+
|
|
9
|
+
- **Core**: `Sample` / `Dataset` (JSONL, content-hashed versions, deterministic
|
|
10
|
+
filter / sample / split), `Output`, `Score` (an error is never a pass), the
|
|
11
|
+
`Check` protocol, `@check`, the registry.
|
|
12
|
+
- **Deterministic checks**: `exact`, `contains`, `regex`, `json_schema`
|
|
13
|
+
(JSON Schema or pydantic), `numeric_tolerance`, `length`, `python`.
|
|
14
|
+
- **Targets**: `callable`, `http`, `recorded`, `prompt`.
|
|
15
|
+
- **Suite / Run / Gate**: per-check and per-tag summaries, dataset hash + git SHA
|
|
16
|
+
provenance, baseline regression gate, error-rate gate (default 0), JSON
|
|
17
|
+
report, `compare`.
|
|
18
|
+
- **CLI**: `init`, `run`, `gate`, `compare`, `show`, `list`.
|
|
19
|
+
- **Providers**: OpenAI-compatible (OpenAI, OpenRouter, Ollama, vLLM),
|
|
20
|
+
Anthropic (official SDK, `evalstone[anthropic]`), Gemini; pricing table with
|
|
21
|
+
overrides; unknown models reported as unpriced.
|
|
22
|
+
- **LLM-as-judge**: rubric → small scale → JSON verdict, reference excluded
|
|
23
|
+
unless asked, judge version hash, content-keyed cache.
|
|
24
|
+
- **RAG** (faithfulness, context relevance, answer relevance), **safety**
|
|
25
|
+
(refusal, no_pii, forbidden), **pairwise** comparison (both orders),
|
|
26
|
+
**trajectory** (tool_called, tool_order, max_steps, no_repeated_calls, finished).
|
|
27
|
+
- **Sample**: `samples/rag-bot` — a no-LLM retrieval bot with a gated suite.
|
|
28
|
+
- **MCP server** (`evalstone mcp`, extra `evalstone[mcp]`): `list_suites`,
|
|
29
|
+
`run_suite`, `judge_output`, `record_sample`, `compare_runs`, `get_run` over
|
|
30
|
+
stdio; calls run one at a time, stray stdout goes to stderr, one provider
|
|
31
|
+
registry and judge cache per session, report paths relative to the config.
|
|
32
|
+
- **pytest plugin** (entry point `pytest11`): `@suite("name")`, fixtures
|
|
33
|
+
`evalstone_run` / `evalstone_gate` / `evalstone_config`, options
|
|
34
|
+
`--evalstone-config` and `--evalstone-baseline` (a report or a directory of
|
|
35
|
+
them); one report per suite (`evalstone-report.<suite>.json`); a suite without
|
|
36
|
+
a gate fails the gated forms; the package imports lazily so the plugin costs
|
|
37
|
+
nothing in unrelated test sessions.
|
|
38
|
+
- **GitHub Action** (`action/`): run a suite on a pull request, job summary,
|
|
39
|
+
in-place PR comment (`evalstone show --markdown`), fail on the gate; inputs
|
|
40
|
+
bound through the environment; `report` output is an absolute path.
|
|
41
|
+
- **Human labels**: `evalstone label <report>` grades outputs in the terminal
|
|
42
|
+
(pass/fail or a 1..N scale, resumable, keyed by sample id + output hash); the
|
|
43
|
+
`human` check scores from the labels file and errors on unlabelled outputs.
|
|
44
|
+
- **Reports**: `Run.markdown()`, `Run.failure_lines()`, results carry the
|
|
45
|
+
sample's input and expected value.
|
|
46
|
+
- **Loading**: project modules are reused across runs in one process unless a
|
|
47
|
+
file changed or the registry was cleared, so test patches reach suites.
|
evalstone-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Marcelo Ribeiro
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
evalstone-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: evalstone
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates.
|
|
5
|
+
Project-URL: Homepage, https://evalstone.com
|
|
6
|
+
Project-URL: Documentation, https://evalstone.com/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/marceloribeiro/evalstone
|
|
8
|
+
Author: Marcelo Ribeiro
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,evals,evaluation,llm,rag,testing
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Software Development :: Testing
|
|
19
|
+
Requires-Python: >=3.12
|
|
20
|
+
Requires-Dist: httpx<1,>=0.28
|
|
21
|
+
Requires-Dist: jsonschema<5,>=4.26
|
|
22
|
+
Requires-Dist: pydantic<3,>=2.14
|
|
23
|
+
Requires-Dist: pyyaml<7,>=6.0
|
|
24
|
+
Requires-Dist: rich<16,>=15.0
|
|
25
|
+
Requires-Dist: typer<1,>=0.27
|
|
26
|
+
Provides-Extra: anthropic
|
|
27
|
+
Requires-Dist: anthropic<2,>=1.13; extra == 'anthropic'
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: anthropic<2,>=1.13; extra == 'dev'
|
|
30
|
+
Requires-Dist: mcp<3,>=2.3; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=9.1; extra == 'dev'
|
|
32
|
+
Requires-Dist: respx>=0.23; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff>=0.17; extra == 'dev'
|
|
34
|
+
Provides-Extra: mcp
|
|
35
|
+
Requires-Dist: mcp<3,>=2.3; extra == 'mcp'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# evalstone
|
|
39
|
+
|
|
40
|
+
The plug-and-play evals layer for LLM-based projects, as a Python library.
|
|
41
|
+
|
|
42
|
+
> **Status:** 0.1.0. Docs: [evalstone.com/docs/evalstone/library](https://evalstone.com/docs/evalstone/library).
|
|
43
|
+
> Source and samples: [github.com/marceloribeiro/evalstone](https://github.com/marceloribeiro/evalstone).
|
|
44
|
+
|
|
45
|
+
The library **computes**; it owns no database. A run lives in memory, prints a
|
|
46
|
+
table and writes a JSON report. Persistence happens through the Evalstone
|
|
47
|
+
center (PostgreSQL) when one is configured.
|
|
48
|
+
|
|
49
|
+
## Develop
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
just install # venv + editable install with dev extras
|
|
53
|
+
just check # ruff + pytest
|
|
54
|
+
just console # REPL with `es` loaded
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Quick start
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install evalstone
|
|
61
|
+
evalstone init my-evals # evalstone.yaml + a golden set + recorded outputs + a check
|
|
62
|
+
cd my-evals
|
|
63
|
+
evalstone run nightly # table + evalstone-report.json
|
|
64
|
+
evalstone gate nightly # exit 1 when the gate fails (CI)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
A fuller example — a small retrieval bot with a gated suite — lives in the
|
|
68
|
+
repository under `samples/rag-bot` (`git clone` it, `pip install -e
|
|
69
|
+
python/evalstone`, then `evalstone run nightly` inside the sample).
|
|
70
|
+
|
|
71
|
+
Judges and prompt targets need a provider key in the environment
|
|
72
|
+
(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or any
|
|
73
|
+
OpenAI-compatible base URL such as OpenRouter or Ollama via `providers:`).
|
|
74
|
+
Without one, judge results are **errors, never passes**, and the gate fails.
|
|
75
|
+
|
|
76
|
+
## Human labels
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
evalstone run nightly --report r.json
|
|
80
|
+
evalstone label r.json --out evals/labels.jsonl # p/f per output, or --scale 5
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Labels are keyed by sample id **and** a hash of the output, so a changed answer
|
|
84
|
+
never inherits an old grade; outputs whose target errored are not offered for
|
|
85
|
+
grading. `--scale N` passes at N-1 (a 2-point scale at 2), as judges do. A
|
|
86
|
+
`human` check (`{type: human, labels: evals/labels.jsonl}`, path relative to
|
|
87
|
+
`evalstone.yaml`) scores from them; an unlabelled output is an error, never a
|
|
88
|
+
pass. Labels are also the ground truth judges get calibrated against (center).
|
|
89
|
+
|
|
90
|
+
## pytest
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from evalstone.pytest_plugin import suite
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@suite("nightly") # runs the suite; a failing gate fails the test
|
|
97
|
+
def test_support_bot(run): # optional body sees the Run
|
|
98
|
+
assert run.summary.cost_usd < 1.0
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Fixtures `evalstone_run` / `evalstone_gate` / `evalstone_config`; options
|
|
102
|
+
`--evalstone-config`, `--evalstone-baseline` (a report, or a directory of them).
|
|
103
|
+
Each suite writes `evalstone-report.<suite>.json` beside the config; paths are
|
|
104
|
+
relative to the config's directory. A suite without a `gate:` fails the gated
|
|
105
|
+
forms. Works on test methods too. The plugin registers itself when evalstone is
|
|
106
|
+
installed, imports nothing of evalstone until a fixture runs, and pytest is not
|
|
107
|
+
a runtime dependency of the library.
|
|
108
|
+
|
|
109
|
+
## MCP server
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
pip install "evalstone[mcp]"
|
|
113
|
+
claude mcp add evalstone -- evalstone mcp # run from the project directory
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Tools: `list_suites`, `run_suite`, `judge_output`, `record_sample`, `compare_runs`,
|
|
117
|
+
`get_run`. The agent edits your prompt, runs the suite, reads the failures, and
|
|
118
|
+
compares runs — the gate stays yours. Report paths are relative to the directory
|
|
119
|
+
holding `evalstone.yaml` (pass `--config` to serve a project from elsewhere).
|
|
120
|
+
|
|
121
|
+
## Concepts
|
|
122
|
+
|
|
123
|
+
`Sample` → `Dataset` (content-hashed, so every run pins an exact version) ·
|
|
124
|
+
`Check` → `Score` (0..1 + pass/fail + reason; an error is never a pass) ·
|
|
125
|
+
`Suite` → `Run` → `Gate`. See the docs for the full vocabulary.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# evalstone
|
|
2
|
+
|
|
3
|
+
The plug-and-play evals layer for LLM-based projects, as a Python library.
|
|
4
|
+
|
|
5
|
+
> **Status:** 0.1.0. Docs: [evalstone.com/docs/evalstone/library](https://evalstone.com/docs/evalstone/library).
|
|
6
|
+
> Source and samples: [github.com/marceloribeiro/evalstone](https://github.com/marceloribeiro/evalstone).
|
|
7
|
+
|
|
8
|
+
The library **computes**; it owns no database. A run lives in memory, prints a
|
|
9
|
+
table and writes a JSON report. Persistence happens through the Evalstone
|
|
10
|
+
center (PostgreSQL) when one is configured.
|
|
11
|
+
|
|
12
|
+
## Develop
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
just install # venv + editable install with dev extras
|
|
16
|
+
just check # ruff + pytest
|
|
17
|
+
just console # REPL with `es` loaded
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick start
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install evalstone
|
|
24
|
+
evalstone init my-evals # evalstone.yaml + a golden set + recorded outputs + a check
|
|
25
|
+
cd my-evals
|
|
26
|
+
evalstone run nightly # table + evalstone-report.json
|
|
27
|
+
evalstone gate nightly # exit 1 when the gate fails (CI)
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
A fuller example — a small retrieval bot with a gated suite — lives in the
|
|
31
|
+
repository under `samples/rag-bot` (`git clone` it, `pip install -e
|
|
32
|
+
python/evalstone`, then `evalstone run nightly` inside the sample).
|
|
33
|
+
|
|
34
|
+
Judges and prompt targets need a provider key in the environment
|
|
35
|
+
(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or any
|
|
36
|
+
OpenAI-compatible base URL such as OpenRouter or Ollama via `providers:`).
|
|
37
|
+
Without one, judge results are **errors, never passes**, and the gate fails.
|
|
38
|
+
|
|
39
|
+
## Human labels
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
evalstone run nightly --report r.json
|
|
43
|
+
evalstone label r.json --out evals/labels.jsonl # p/f per output, or --scale 5
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Labels are keyed by sample id **and** a hash of the output, so a changed answer
|
|
47
|
+
never inherits an old grade; outputs whose target errored are not offered for
|
|
48
|
+
grading. `--scale N` passes at N-1 (a 2-point scale at 2), as judges do. A
|
|
49
|
+
`human` check (`{type: human, labels: evals/labels.jsonl}`, path relative to
|
|
50
|
+
`evalstone.yaml`) scores from them; an unlabelled output is an error, never a
|
|
51
|
+
pass. Labels are also the ground truth judges get calibrated against (center).
|
|
52
|
+
|
|
53
|
+
## pytest
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from evalstone.pytest_plugin import suite
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@suite("nightly") # runs the suite; a failing gate fails the test
|
|
60
|
+
def test_support_bot(run): # optional body sees the Run
|
|
61
|
+
assert run.summary.cost_usd < 1.0
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Fixtures `evalstone_run` / `evalstone_gate` / `evalstone_config`; options
|
|
65
|
+
`--evalstone-config`, `--evalstone-baseline` (a report, or a directory of them).
|
|
66
|
+
Each suite writes `evalstone-report.<suite>.json` beside the config; paths are
|
|
67
|
+
relative to the config's directory. A suite without a `gate:` fails the gated
|
|
68
|
+
forms. Works on test methods too. The plugin registers itself when evalstone is
|
|
69
|
+
installed, imports nothing of evalstone until a fixture runs, and pytest is not
|
|
70
|
+
a runtime dependency of the library.
|
|
71
|
+
|
|
72
|
+
## MCP server
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install "evalstone[mcp]"
|
|
76
|
+
claude mcp add evalstone -- evalstone mcp # run from the project directory
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Tools: `list_suites`, `run_suite`, `judge_output`, `record_sample`, `compare_runs`,
|
|
80
|
+
`get_run`. The agent edits your prompt, runs the suite, reads the failures, and
|
|
81
|
+
compares runs — the gate stays yours. Report paths are relative to the directory
|
|
82
|
+
holding `evalstone.yaml` (pass `--config` to serve a project from elsewhere).
|
|
83
|
+
|
|
84
|
+
## Concepts
|
|
85
|
+
|
|
86
|
+
`Sample` → `Dataset` (content-hashed, so every run pins an exact version) ·
|
|
87
|
+
`Check` → `Score` (0..1 + pass/fail + reason; an error is never a pass) ·
|
|
88
|
+
`Suite` → `Run` → `Gate`. See the docs for the full vocabulary.
|
evalstone-0.1.0/justfile
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# evalstone — the library. Builder-convention justfile (no database: the library owns none).
|
|
2
|
+
set dotenv-load := true
|
|
3
|
+
|
|
4
|
+
venv := "venv/bin"
|
|
5
|
+
python := env_var_or_default("PYTHON", "python3.13")
|
|
6
|
+
|
|
7
|
+
# Install the package in editable mode with dev extras into ./venv (PYTHON=python3.12 to override).
|
|
8
|
+
install:
|
|
9
|
+
test -d venv || {{python}} -m venv venv
|
|
10
|
+
{{venv}}/pip install -q --upgrade pip
|
|
11
|
+
{{venv}}/pip install -q -e ".[dev]"
|
|
12
|
+
|
|
13
|
+
# Run the test suite.
|
|
14
|
+
test:
|
|
15
|
+
{{venv}}/pytest
|
|
16
|
+
|
|
17
|
+
# Lint (ruff check) without changing files.
|
|
18
|
+
lint:
|
|
19
|
+
{{venv}}/ruff check .
|
|
20
|
+
{{venv}}/ruff format --check .
|
|
21
|
+
|
|
22
|
+
# Tests + linters — run before pushing (same as CI).
|
|
23
|
+
check: lint test
|
|
24
|
+
|
|
25
|
+
# Auto-fix lint problems and format the code.
|
|
26
|
+
formatters-fix:
|
|
27
|
+
{{venv}}/ruff check --fix .
|
|
28
|
+
{{venv}}/ruff format .
|
|
29
|
+
|
|
30
|
+
# Python REPL with evalstone imported.
|
|
31
|
+
console:
|
|
32
|
+
{{venv}}/python -i -c "import evalstone as es; print('evalstone', es.__version__, '— `es` is loaded')"
|
|
33
|
+
|
|
34
|
+
# Build sdist + wheel into dist/.
|
|
35
|
+
build:
|
|
36
|
+
{{venv}}/pip install -q build
|
|
37
|
+
{{venv}}/python -m build
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "evalstone"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.12"
|
|
12
|
+
authors = [{ name = "Marcelo Ribeiro" }]
|
|
13
|
+
keywords = ["llm", "evals", "evaluation", "testing", "rag", "agents"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Topic :: Software Development :: Testing",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"httpx>=0.28,<1",
|
|
25
|
+
"jsonschema>=4.26,<5",
|
|
26
|
+
"pydantic>=2.14,<3",
|
|
27
|
+
"pyyaml>=6.0,<7",
|
|
28
|
+
"rich>=15.0,<16",
|
|
29
|
+
"typer>=0.27,<1",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
anthropic = ["anthropic>=1.13,<2"]
|
|
34
|
+
mcp = ["mcp>=2.3,<3"]
|
|
35
|
+
dev = [
|
|
36
|
+
"anthropic>=1.13,<2",
|
|
37
|
+
"mcp>=2.3,<3",
|
|
38
|
+
"pytest>=9.1",
|
|
39
|
+
"respx>=0.23",
|
|
40
|
+
"ruff>=0.17",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.scripts]
|
|
44
|
+
evalstone = "evalstone.cli:main"
|
|
45
|
+
|
|
46
|
+
[project.entry-points.pytest11]
|
|
47
|
+
evalstone = "evalstone.pytest_plugin"
|
|
48
|
+
|
|
49
|
+
[project.urls]
|
|
50
|
+
Homepage = "https://evalstone.com"
|
|
51
|
+
Documentation = "https://evalstone.com/docs"
|
|
52
|
+
Repository = "https://github.com/marceloribeiro/evalstone"
|
|
53
|
+
|
|
54
|
+
[tool.hatch.version]
|
|
55
|
+
path = "src/evalstone/__init__.py"
|
|
56
|
+
|
|
57
|
+
[tool.hatch.build.targets.wheel]
|
|
58
|
+
packages = ["src/evalstone"]
|
|
59
|
+
|
|
60
|
+
[tool.pytest.ini_options]
|
|
61
|
+
testpaths = ["tests"]
|
|
62
|
+
addopts = "-q"
|
|
63
|
+
|
|
64
|
+
[tool.ruff]
|
|
65
|
+
line-length = 100
|
|
66
|
+
target-version = "py312"
|
|
67
|
+
src = ["src", "tests"]
|
|
68
|
+
|
|
69
|
+
[tool.ruff.lint]
|
|
70
|
+
select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
|
|
71
|
+
|
|
72
|
+
[tool.ruff.format]
|
|
73
|
+
quote-style = "double"
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Evalstone — the plug-and-play evals layer for LLM-based projects.
|
|
2
|
+
|
|
3
|
+
The library computes; it owns no database. See https://evalstone.com/docs.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import sys
|
|
7
|
+
import types
|
|
8
|
+
from importlib import import_module
|
|
9
|
+
from typing import TYPE_CHECKING, Any
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
|
|
13
|
+
# Exports resolve on first access (PEP 562) so that importing the package — which
|
|
14
|
+
# the pytest plugin does in every pytest session on the interpreter — costs
|
|
15
|
+
# nothing until a name is used.
|
|
16
|
+
_EXPORTS: dict[str, str] = {
|
|
17
|
+
"check": "evalstone.checks",
|
|
18
|
+
"registry": "evalstone.checks",
|
|
19
|
+
"Dataset": "evalstone.dataset",
|
|
20
|
+
"Sample": "evalstone.dataset",
|
|
21
|
+
"Source": "evalstone.dataset",
|
|
22
|
+
"Gate": "evalstone.gate",
|
|
23
|
+
"GateFailed": "evalstone.gate",
|
|
24
|
+
"GateReport": "evalstone.gate",
|
|
25
|
+
"Output": "evalstone.output",
|
|
26
|
+
"Providers": "evalstone.providers",
|
|
27
|
+
"Comparison": "evalstone.run",
|
|
28
|
+
"Result": "evalstone.run",
|
|
29
|
+
"Run": "evalstone.run",
|
|
30
|
+
"RunSummary": "evalstone.run",
|
|
31
|
+
"compare": "evalstone.run",
|
|
32
|
+
"write_report": "evalstone.run",
|
|
33
|
+
"Check": "evalstone.score",
|
|
34
|
+
"CheckKind": "evalstone.score",
|
|
35
|
+
"Context": "evalstone.score",
|
|
36
|
+
"MemoryCache": "evalstone.score",
|
|
37
|
+
"Score": "evalstone.score",
|
|
38
|
+
"Suite": "evalstone.suite",
|
|
39
|
+
"CallableTarget": "evalstone.target",
|
|
40
|
+
"HttpTarget": "evalstone.target",
|
|
41
|
+
"PromptTarget": "evalstone.target",
|
|
42
|
+
"RecordedTarget": "evalstone.target",
|
|
43
|
+
"Target": "evalstone.target",
|
|
44
|
+
"target": "evalstone.target",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if TYPE_CHECKING:
|
|
48
|
+
from evalstone import checks
|
|
49
|
+
from evalstone.checks import check, registry
|
|
50
|
+
from evalstone.dataset import Dataset, Sample, Source
|
|
51
|
+
from evalstone.gate import Gate, GateFailed, GateReport
|
|
52
|
+
from evalstone.output import Output
|
|
53
|
+
from evalstone.providers import Providers
|
|
54
|
+
from evalstone.run import Comparison, Result, Run, RunSummary, compare, write_report
|
|
55
|
+
from evalstone.score import Check, CheckKind, Context, MemoryCache, Score
|
|
56
|
+
from evalstone.suite import Suite
|
|
57
|
+
from evalstone.target import (
|
|
58
|
+
CallableTarget,
|
|
59
|
+
HttpTarget,
|
|
60
|
+
PromptTarget,
|
|
61
|
+
RecordedTarget,
|
|
62
|
+
Target,
|
|
63
|
+
target,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class _Package(types.ModuleType):
|
|
68
|
+
"""``evalstone.target`` is the factory function, as it always was, even though a
|
|
69
|
+
submodule of the same name exists: importing that submodule makes the import
|
|
70
|
+
system bind it onto the package, which the setter below ignores."""
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def target(self) -> Any:
|
|
74
|
+
return import_module("evalstone.target").target
|
|
75
|
+
|
|
76
|
+
@target.setter
|
|
77
|
+
def target(self, value: Any) -> None:
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
sys.modules[__name__].__class__ = _Package
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def __getattr__(name: str) -> Any:
|
|
85
|
+
if name == "checks":
|
|
86
|
+
return import_module("evalstone.checks")
|
|
87
|
+
try:
|
|
88
|
+
module = _EXPORTS[name]
|
|
89
|
+
except KeyError:
|
|
90
|
+
raise AttributeError(f"module 'evalstone' has no attribute {name!r}") from None
|
|
91
|
+
value = getattr(import_module(module), name)
|
|
92
|
+
globals()[name] = value
|
|
93
|
+
return value
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def __dir__() -> list[str]:
|
|
97
|
+
return sorted(set(__all__) | {n for n in globals() if n.startswith("__")})
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
__all__ = [
|
|
101
|
+
"CallableTarget",
|
|
102
|
+
"Check",
|
|
103
|
+
"CheckKind",
|
|
104
|
+
"Comparison",
|
|
105
|
+
"Context",
|
|
106
|
+
"Dataset",
|
|
107
|
+
"Gate",
|
|
108
|
+
"GateFailed",
|
|
109
|
+
"GateReport",
|
|
110
|
+
"HttpTarget",
|
|
111
|
+
"MemoryCache",
|
|
112
|
+
"Output",
|
|
113
|
+
"PromptTarget",
|
|
114
|
+
"Providers",
|
|
115
|
+
"RecordedTarget",
|
|
116
|
+
"Result",
|
|
117
|
+
"Run",
|
|
118
|
+
"RunSummary",
|
|
119
|
+
"Sample",
|
|
120
|
+
"Score",
|
|
121
|
+
"Source",
|
|
122
|
+
"Suite",
|
|
123
|
+
"Target",
|
|
124
|
+
"__version__",
|
|
125
|
+
"check",
|
|
126
|
+
"checks",
|
|
127
|
+
"compare",
|
|
128
|
+
"registry",
|
|
129
|
+
"target",
|
|
130
|
+
"write_report",
|
|
131
|
+
]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Best-effort git facts for run provenance. Never raises."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import subprocess
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def head_sha(cwd: str | Path | None = None) -> str | None:
|
|
10
|
+
try:
|
|
11
|
+
out = subprocess.run(
|
|
12
|
+
["git", "rev-parse", "HEAD"],
|
|
13
|
+
cwd=cwd,
|
|
14
|
+
capture_output=True,
|
|
15
|
+
text=True,
|
|
16
|
+
timeout=5,
|
|
17
|
+
check=False,
|
|
18
|
+
)
|
|
19
|
+
except (OSError, subprocess.SubprocessError):
|
|
20
|
+
return None
|
|
21
|
+
sha = out.stdout.strip()
|
|
22
|
+
return sha if out.returncode == 0 and sha else None
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Canonical JSON helpers: one stable serialisation so content hashes are reproducible.
|
|
2
|
+
|
|
3
|
+
Only JSON-native values plus a few explicitly handled types (sets, tuples,
|
|
4
|
+
bytes, dates, enums, pydantic models, dataclasses) are accepted. Anything else
|
|
5
|
+
raises ``TypeError`` instead of being stringified — ``str(obj)`` would bake a
|
|
6
|
+
memory address or a hash-randomised set order into a content hash.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import base64
|
|
12
|
+
import dataclasses
|
|
13
|
+
import datetime as dt
|
|
14
|
+
import enum
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _encode(value: Any) -> Any:
|
|
21
|
+
"""Convert the supported non-JSON types to a deterministic JSON-native form."""
|
|
22
|
+
if value is None or isinstance(value, str | int | float | bool):
|
|
23
|
+
return value
|
|
24
|
+
if isinstance(value, list):
|
|
25
|
+
return [_encode(v) for v in value]
|
|
26
|
+
if isinstance(value, dict):
|
|
27
|
+
if any(not isinstance(k, str) for k in value):
|
|
28
|
+
raise TypeError("dict keys must be strings to hash")
|
|
29
|
+
return {k: _encode(v) for k, v in value.items()}
|
|
30
|
+
if isinstance(value, enum.Enum):
|
|
31
|
+
return value.value
|
|
32
|
+
if isinstance(value, set | frozenset):
|
|
33
|
+
return sorted((_encode(v) for v in value), key=lambda v: json.dumps(v, sort_keys=True))
|
|
34
|
+
if isinstance(value, tuple):
|
|
35
|
+
return [_encode(v) for v in value]
|
|
36
|
+
if isinstance(value, bytes | bytearray):
|
|
37
|
+
return {"__bytes__": base64.b64encode(bytes(value)).decode("ascii")}
|
|
38
|
+
if isinstance(value, dt.datetime | dt.date | dt.time):
|
|
39
|
+
return value.isoformat()
|
|
40
|
+
if hasattr(value, "model_dump") and callable(value.model_dump): # pydantic
|
|
41
|
+
return value.model_dump(mode="json")
|
|
42
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
43
|
+
return dataclasses.asdict(value)
|
|
44
|
+
raise TypeError(f"not JSON-serialisable for hashing/text: {type(value).__name__}")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def dumps(value: Any, **kwargs: Any) -> str:
|
|
48
|
+
"""``json.dumps`` with the explicit encoder above (never ``default=str``)."""
|
|
49
|
+
return json.dumps(_encode(value), ensure_ascii=False, **kwargs)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def canonical(value: Any) -> str:
|
|
53
|
+
"""Serialise ``value`` deterministically (sorted keys, no whitespace, UTF-8 kept)."""
|
|
54
|
+
return dumps(_encode(value), sort_keys=True, separators=(",", ":"))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def sha256(value: Any) -> str:
|
|
58
|
+
"""sha256 hex digest of the canonical JSON form of ``value``."""
|
|
59
|
+
return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()
|