evalstone 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. evalstone-0.1.0/.gitignore +44 -0
  2. evalstone-0.1.0/CHANGELOG.md +47 -0
  3. evalstone-0.1.0/LICENSE +21 -0
  4. evalstone-0.1.0/PKG-INFO +125 -0
  5. evalstone-0.1.0/README.md +88 -0
  6. evalstone-0.1.0/justfile +37 -0
  7. evalstone-0.1.0/pyproject.toml +73 -0
  8. evalstone-0.1.0/src/evalstone/__init__.py +131 -0
  9. evalstone-0.1.0/src/evalstone/__main__.py +3 -0
  10. evalstone-0.1.0/src/evalstone/_git.py +22 -0
  11. evalstone-0.1.0/src/evalstone/_json.py +59 -0
  12. evalstone-0.1.0/src/evalstone/checks/__init__.py +41 -0
  13. evalstone-0.1.0/src/evalstone/checks/_base.py +153 -0
  14. evalstone-0.1.0/src/evalstone/checks/comparison.py +154 -0
  15. evalstone-0.1.0/src/evalstone/checks/deterministic.py +279 -0
  16. evalstone-0.1.0/src/evalstone/checks/human.py +124 -0
  17. evalstone-0.1.0/src/evalstone/checks/judge.py +219 -0
  18. evalstone-0.1.0/src/evalstone/checks/rag.py +70 -0
  19. evalstone-0.1.0/src/evalstone/checks/safety.py +131 -0
  20. evalstone-0.1.0/src/evalstone/checks/trajectory.py +144 -0
  21. evalstone-0.1.0/src/evalstone/cli.py +411 -0
  22. evalstone-0.1.0/src/evalstone/config.py +267 -0
  23. evalstone-0.1.0/src/evalstone/dataset.py +184 -0
  24. evalstone-0.1.0/src/evalstone/gate.py +84 -0
  25. evalstone-0.1.0/src/evalstone/mcp_server.py +329 -0
  26. evalstone-0.1.0/src/evalstone/output.py +36 -0
  27. evalstone-0.1.0/src/evalstone/providers/__init__.py +185 -0
  28. evalstone-0.1.0/src/evalstone/providers/_http.py +58 -0
  29. evalstone-0.1.0/src/evalstone/providers/anthropic_.py +81 -0
  30. evalstone-0.1.0/src/evalstone/providers/gemini.py +55 -0
  31. evalstone-0.1.0/src/evalstone/providers/openai_compat.py +67 -0
  32. evalstone-0.1.0/src/evalstone/pytest_plugin.py +203 -0
  33. evalstone-0.1.0/src/evalstone/run.py +336 -0
  34. evalstone-0.1.0/src/evalstone/score.py +124 -0
  35. evalstone-0.1.0/src/evalstone/suite.py +135 -0
  36. evalstone-0.1.0/src/evalstone/target.py +289 -0
  37. evalstone-0.1.0/tests/conftest.py +13 -0
  38. evalstone-0.1.0/tests/test_action.py +108 -0
  39. evalstone-0.1.0/tests/test_cli.py +145 -0
  40. evalstone-0.1.0/tests/test_config.py +211 -0
  41. evalstone-0.1.0/tests/test_dataset.py +168 -0
  42. evalstone-0.1.0/tests/test_deterministic.py +204 -0
  43. evalstone-0.1.0/tests/test_docs.py +33 -0
  44. evalstone-0.1.0/tests/test_human.py +159 -0
  45. evalstone-0.1.0/tests/test_judge.py +251 -0
  46. evalstone-0.1.0/tests/test_mcp.py +280 -0
  47. evalstone-0.1.0/tests/test_providers.py +245 -0
  48. evalstone-0.1.0/tests/test_pytest_plugin.py +245 -0
  49. evalstone-0.1.0/tests/test_rag_and_comparison.py +243 -0
  50. evalstone-0.1.0/tests/test_safety.py +96 -0
  51. evalstone-0.1.0/tests/test_score_and_checks.py +150 -0
  52. evalstone-0.1.0/tests/test_suite_run_gate.py +233 -0
  53. evalstone-0.1.0/tests/test_target.py +134 -0
  54. evalstone-0.1.0/tests/test_trajectory.py +84 -0
@@ -0,0 +1,44 @@
1
+ # ── Secrets / local env ────────────────────────────────────────────────
2
+ **/.env
3
+ **/.env.local
4
+ **/.env.*.local
5
+ **/.env.production
6
+ !**/.env.example
7
+
8
+ # ── OS / editor ────────────────────────────────────────────────────────
9
+ .DS_Store
10
+ *.pem
11
+ *.swp
12
+ *~
13
+ **/certs/
14
+
15
+ # ── Python (python/) ───────────────────────────────────────────────────
16
+ **/venv/
17
+ **/.venv/
18
+ **/__pycache__/
19
+ *.pyc
20
+ **/.pytest_cache/
21
+ **/.ruff_cache/
22
+ **/.mypy_cache/
23
+ *.egg-info/
24
+ **/dist/
25
+ .coverage
26
+ htmlcov/
27
+
28
+ # ── Node / Next.js (javascript/) ───────────────────────────────────────
29
+ **/node_modules/
30
+ **/.next/
31
+ **/.source/
32
+ **/out/
33
+ **/build/
34
+ **/coverage/
35
+ *.tsbuildinfo
36
+ next-env.d.ts
37
+ **/test-results/
38
+ **/playwright-report/
39
+ **/playwright/.cache/
40
+ **/logs/
41
+ npm-debug.log*
42
+
43
+ # ── Evalstone run reports (written by `evalstone run`) ──────────────────
44
+ **/evalstone-report.json
@@ -0,0 +1,47 @@
1
+ # Changelog — evalstone (the library)
2
+
3
+ All notable changes to `python/evalstone`. Semantic versioning; tags `lib-vX.Y.Z`.
4
+
5
+ ## 0.1.0 — 2026-10-13
6
+
7
+ The first usable release: the hero path from `evalstone.yaml` to a gated run.
8
+
9
+ - **Core**: `Sample` / `Dataset` (JSONL, content-hashed versions, deterministic
10
+ filter / sample / split), `Output`, `Score` (an error is never a pass), the
11
+ `Check` protocol, `@check`, the registry.
12
+ - **Deterministic checks**: `exact`, `contains`, `regex`, `json_schema`
13
+ (JSON Schema or pydantic), `numeric_tolerance`, `length`, `python`.
14
+ - **Targets**: `callable`, `http`, `recorded`, `prompt`.
15
+ - **Suite / Run / Gate**: per-check and per-tag summaries, dataset hash + git SHA
16
+ provenance, baseline regression gate, error-rate gate (default 0), JSON
17
+ report, `compare`.
18
+ - **CLI**: `init`, `run`, `gate`, `compare`, `show`, `list`.
19
+ - **Providers**: OpenAI-compatible (OpenAI, OpenRouter, Ollama, vLLM),
20
+ Anthropic (official SDK, `evalstone[anthropic]`), Gemini; pricing table with
21
+ overrides; unknown models reported as unpriced.
22
+ - **LLM-as-judge**: rubric → small scale → JSON verdict, reference excluded
23
+ unless asked, judge version hash, content-keyed cache.
24
+ - **RAG** (faithfulness, context relevance, answer relevance), **safety**
25
+ (refusal, no_pii, forbidden), **pairwise** comparison (both orders),
26
+ **trajectory** (tool_called, tool_order, max_steps, no_repeated_calls, finished).
27
+ - **Sample**: `samples/rag-bot` — a no-LLM retrieval bot with a gated suite.
28
+ - **MCP server** (`evalstone mcp`, extra `evalstone[mcp]`): `list_suites`,
29
+ `run_suite`, `judge_output`, `record_sample`, `compare_runs`, `get_run` over
30
+ stdio; calls run one at a time, stray stdout goes to stderr, one provider
31
+ registry and judge cache per session, report paths relative to the config.
32
+ - **pytest plugin** (entry point `pytest11`): `@suite("name")`, fixtures
33
+ `evalstone_run` / `evalstone_gate` / `evalstone_config`, options
34
+ `--evalstone-config` and `--evalstone-baseline` (a report or a directory of
35
+ them); one report per suite (`evalstone-report.<suite>.json`); a suite without
36
+ a gate fails the gated forms; the package imports lazily so the plugin costs
37
+ nothing in unrelated test sessions.
38
+ - **GitHub Action** (`action/`): run a suite on a pull request, job summary,
39
+ in-place PR comment (`evalstone show --markdown`), fail on the gate; inputs
40
+ bound through the environment; `report` output is an absolute path.
41
+ - **Human labels**: `evalstone label <report>` grades outputs in the terminal
42
+ (pass/fail or a 1..N scale, resumable, keyed by sample id + output hash); the
43
+ `human` check scores from the labels file and errors on unlabelled outputs.
44
+ - **Reports**: `Run.markdown()`, `Run.failure_lines()`, results carry the
45
+ sample's input and expected value.
46
+ - **Loading**: project modules are reused across runs in one process unless a
47
+ file changed or the registry was cleared, so test patches reach suites.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Marcelo Ribeiro
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,125 @@
1
+ Metadata-Version: 2.5
2
+ Name: evalstone
3
+ Version: 0.1.0
4
+ Summary: The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates.
5
+ Project-URL: Homepage, https://evalstone.com
6
+ Project-URL: Documentation, https://evalstone.com/docs
7
+ Project-URL: Repository, https://github.com/marceloribeiro/evalstone
8
+ Author: Marcelo Ribeiro
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: agents,evals,evaluation,llm,rag,testing
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Software Development :: Testing
19
+ Requires-Python: >=3.12
20
+ Requires-Dist: httpx<1,>=0.28
21
+ Requires-Dist: jsonschema<5,>=4.26
22
+ Requires-Dist: pydantic<3,>=2.14
23
+ Requires-Dist: pyyaml<7,>=6.0
24
+ Requires-Dist: rich<16,>=15.0
25
+ Requires-Dist: typer<1,>=0.27
26
+ Provides-Extra: anthropic
27
+ Requires-Dist: anthropic<2,>=1.13; extra == 'anthropic'
28
+ Provides-Extra: dev
29
+ Requires-Dist: anthropic<2,>=1.13; extra == 'dev'
30
+ Requires-Dist: mcp<3,>=2.3; extra == 'dev'
31
+ Requires-Dist: pytest>=9.1; extra == 'dev'
32
+ Requires-Dist: respx>=0.23; extra == 'dev'
33
+ Requires-Dist: ruff>=0.17; extra == 'dev'
34
+ Provides-Extra: mcp
35
+ Requires-Dist: mcp<3,>=2.3; extra == 'mcp'
36
+ Description-Content-Type: text/markdown
37
+
38
+ # evalstone
39
+
40
+ The plug-and-play evals layer for LLM-based projects, as a Python library.
41
+
42
+ > **Status:** 0.1.0. Docs: [evalstone.com/docs/evalstone/library](https://evalstone.com/docs/evalstone/library).
43
+ > Source and samples: [github.com/marceloribeiro/evalstone](https://github.com/marceloribeiro/evalstone).
44
+
45
+ The library **computes**; it owns no database. A run lives in memory, prints a
46
+ table and writes a JSON report. Persistence happens through the Evalstone
47
+ center (PostgreSQL) when one is configured.
48
+
49
+ ## Develop
50
+
51
+ ```bash
52
+ just install # venv + editable install with dev extras
53
+ just check # ruff + pytest
54
+ just console # REPL with `es` loaded
55
+ ```
56
+
57
+ ## Quick start
58
+
59
+ ```bash
60
+ pip install evalstone
61
+ evalstone init my-evals # evalstone.yaml + a golden set + recorded outputs + a check
62
+ cd my-evals
63
+ evalstone run nightly # table + evalstone-report.json
64
+ evalstone gate nightly # exit 1 when the gate fails (CI)
65
+ ```
66
+
67
+ A fuller example — a small retrieval bot with a gated suite — lives in the
68
+ repository under `samples/rag-bot` (`git clone` it, `pip install -e
69
+ python/evalstone`, then `evalstone run nightly` inside the sample).
70
+
71
+ Judges and prompt targets need a provider key in the environment
72
+ (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or any
73
+ OpenAI-compatible base URL such as OpenRouter or Ollama via `providers:`).
74
+ Without one, judge results are **errors, never passes**, and the gate fails.
75
+
76
+ ## Human labels
77
+
78
+ ```bash
79
+ evalstone run nightly --report r.json
80
+ evalstone label r.json --out evals/labels.jsonl # p/f per output, or --scale 5
81
+ ```
82
+
83
+ Labels are keyed by sample id **and** a hash of the output, so a changed answer
84
+ never inherits an old grade; outputs whose target errored are not offered for
85
+ grading. `--scale N` passes at N-1 (a 2-point scale at 2), as judges do. A
86
+ `human` check (`{type: human, labels: evals/labels.jsonl}`, path relative to
87
+ `evalstone.yaml`) scores from them; an unlabelled output is an error, never a
88
+ pass. Labels are also the ground truth judges get calibrated against (center).
89
+
90
+ ## pytest
91
+
92
+ ```python
93
+ from evalstone.pytest_plugin import suite
94
+
95
+
96
+ @suite("nightly") # runs the suite; a failing gate fails the test
97
+ def test_support_bot(run): # optional body sees the Run
98
+ assert run.summary.cost_usd < 1.0
99
+ ```
100
+
101
+ Fixtures `evalstone_run` / `evalstone_gate` / `evalstone_config`; options
102
+ `--evalstone-config`, `--evalstone-baseline` (a report, or a directory of them).
103
+ Each suite writes `evalstone-report.<suite>.json` beside the config; paths are
104
+ relative to the config's directory. A suite without a `gate:` fails the gated
105
+ forms. Works on test methods too. The plugin registers itself when evalstone is
106
+ installed, imports nothing of evalstone until a fixture runs, and pytest is not
107
+ a runtime dependency of the library.
108
+
109
+ ## MCP server
110
+
111
+ ```bash
112
+ pip install "evalstone[mcp]"
113
+ claude mcp add evalstone -- evalstone mcp # run from the project directory
114
+ ```
115
+
116
+ Tools: `list_suites`, `run_suite`, `judge_output`, `record_sample`, `compare_runs`,
117
+ `get_run`. The agent edits your prompt, runs the suite, reads the failures, and
118
+ compares runs — the gate stays yours. Report paths are relative to the directory
119
+ holding `evalstone.yaml` (pass `--config` to serve a project from elsewhere).
120
+
121
+ ## Concepts
122
+
123
+ `Sample` → `Dataset` (content-hashed, so every run pins an exact version) ·
124
+ `Check` → `Score` (0..1 + pass/fail + reason; an error is never a pass) ·
125
+ `Suite` → `Run` → `Gate`. See the docs for the full vocabulary.
@@ -0,0 +1,88 @@
1
+ # evalstone
2
+
3
+ The plug-and-play evals layer for LLM-based projects, as a Python library.
4
+
5
+ > **Status:** 0.1.0. Docs: [evalstone.com/docs/evalstone/library](https://evalstone.com/docs/evalstone/library).
6
+ > Source and samples: [github.com/marceloribeiro/evalstone](https://github.com/marceloribeiro/evalstone).
7
+
8
+ The library **computes**; it owns no database. A run lives in memory, prints a
9
+ table and writes a JSON report. Persistence happens through the Evalstone
10
+ center (PostgreSQL) when one is configured.
11
+
12
+ ## Develop
13
+
14
+ ```bash
15
+ just install # venv + editable install with dev extras
16
+ just check # ruff + pytest
17
+ just console # REPL with `es` loaded
18
+ ```
19
+
20
+ ## Quick start
21
+
22
+ ```bash
23
+ pip install evalstone
24
+ evalstone init my-evals # evalstone.yaml + a golden set + recorded outputs + a check
25
+ cd my-evals
26
+ evalstone run nightly # table + evalstone-report.json
27
+ evalstone gate nightly # exit 1 when the gate fails (CI)
28
+ ```
29
+
30
+ A fuller example — a small retrieval bot with a gated suite — lives in the
31
+ repository under `samples/rag-bot` (`git clone` it, `pip install -e
32
+ python/evalstone`, then `evalstone run nightly` inside the sample).
33
+
34
+ Judges and prompt targets need a provider key in the environment
35
+ (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, or any
36
+ OpenAI-compatible base URL such as OpenRouter or Ollama via `providers:`).
37
+ Without one, judge results are **errors, never passes**, and the gate fails.
38
+
39
+ ## Human labels
40
+
41
+ ```bash
42
+ evalstone run nightly --report r.json
43
+ evalstone label r.json --out evals/labels.jsonl # p/f per output, or --scale 5
44
+ ```
45
+
46
+ Labels are keyed by sample id **and** a hash of the output, so a changed answer
47
+ never inherits an old grade; outputs whose target errored are not offered for
48
+ grading. `--scale N` passes at N-1 (a 2-point scale at 2), as judges do. A
49
+ `human` check (`{type: human, labels: evals/labels.jsonl}`, path relative to
50
+ `evalstone.yaml`) scores from them; an unlabelled output is an error, never a
51
+ pass. Labels are also the ground truth judges get calibrated against (center).
52
+
53
+ ## pytest
54
+
55
+ ```python
56
+ from evalstone.pytest_plugin import suite
57
+
58
+
59
+ @suite("nightly") # runs the suite; a failing gate fails the test
60
+ def test_support_bot(run): # optional body sees the Run
61
+ assert run.summary.cost_usd < 1.0
62
+ ```
63
+
64
+ Fixtures `evalstone_run` / `evalstone_gate` / `evalstone_config`; options
65
+ `--evalstone-config`, `--evalstone-baseline` (a report, or a directory of them).
66
+ Each suite writes `evalstone-report.<suite>.json` beside the config; paths are
67
+ relative to the config's directory. A suite without a `gate:` fails the gated
68
+ forms. Works on test methods too. The plugin registers itself when evalstone is
69
+ installed, imports nothing of evalstone until a fixture runs, and pytest is not
70
+ a runtime dependency of the library.
71
+
72
+ ## MCP server
73
+
74
+ ```bash
75
+ pip install "evalstone[mcp]"
76
+ claude mcp add evalstone -- evalstone mcp # run from the project directory
77
+ ```
78
+
79
+ Tools: `list_suites`, `run_suite`, `judge_output`, `record_sample`, `compare_runs`,
80
+ `get_run`. The agent edits your prompt, runs the suite, reads the failures, and
81
+ compares runs — the gate stays yours. Report paths are relative to the directory
82
+ holding `evalstone.yaml` (pass `--config` to serve a project from elsewhere).
83
+
84
+ ## Concepts
85
+
86
+ `Sample` → `Dataset` (content-hashed, so every run pins an exact version) ·
87
+ `Check` → `Score` (0..1 + pass/fail + reason; an error is never a pass) ·
88
+ `Suite` → `Run` → `Gate`. See the docs for the full vocabulary.
@@ -0,0 +1,37 @@
1
+ # evalstone — the library. Builder-convention justfile (no database: the library owns none).
2
+ set dotenv-load := true
3
+
4
+ venv := "venv/bin"
5
+ python := env_var_or_default("PYTHON", "python3.13")
6
+
7
+ # Install the package in editable mode with dev extras into ./venv (PYTHON=python3.12 to override).
8
+ install:
9
+ test -d venv || {{python}} -m venv venv
10
+ {{venv}}/pip install -q --upgrade pip
11
+ {{venv}}/pip install -q -e ".[dev]"
12
+
13
+ # Run the test suite.
14
+ test:
15
+ {{venv}}/pytest
16
+
17
+ # Lint (ruff check) without changing files.
18
+ lint:
19
+ {{venv}}/ruff check .
20
+ {{venv}}/ruff format --check .
21
+
22
+ # Tests + linters — run before pushing (same as CI).
23
+ check: lint test
24
+
25
+ # Auto-fix lint problems and format the code.
26
+ formatters-fix:
27
+ {{venv}}/ruff check --fix .
28
+ {{venv}}/ruff format .
29
+
30
+ # Python REPL with evalstone imported.
31
+ console:
32
+ {{venv}}/python -i -c "import evalstone as es; print('evalstone', es.__version__, '— `es` is loaded')"
33
+
34
+ # Build sdist + wheel into dist/.
35
+ build:
36
+ {{venv}}/pip install -q build
37
+ {{venv}}/python -m build
@@ -0,0 +1,73 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.27"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "evalstone"
7
+ dynamic = ["version"]
8
+ description = "The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ requires-python = ">=3.12"
12
+ authors = [{ name = "Marcelo Ribeiro" }]
13
+ keywords = ["llm", "evals", "evaluation", "testing", "rag", "agents"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Developers",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3",
19
+ "Programming Language :: Python :: 3.12",
20
+ "Programming Language :: Python :: 3.13",
21
+ "Topic :: Software Development :: Testing",
22
+ ]
23
+ dependencies = [
24
+ "httpx>=0.28,<1",
25
+ "jsonschema>=4.26,<5",
26
+ "pydantic>=2.14,<3",
27
+ "pyyaml>=6.0,<7",
28
+ "rich>=15.0,<16",
29
+ "typer>=0.27,<1",
30
+ ]
31
+
32
+ [project.optional-dependencies]
33
+ anthropic = ["anthropic>=1.13,<2"]
34
+ mcp = ["mcp>=2.3,<3"]
35
+ dev = [
36
+ "anthropic>=1.13,<2",
37
+ "mcp>=2.3,<3",
38
+ "pytest>=9.1",
39
+ "respx>=0.23",
40
+ "ruff>=0.17",
41
+ ]
42
+
43
+ [project.scripts]
44
+ evalstone = "evalstone.cli:main"
45
+
46
+ [project.entry-points.pytest11]
47
+ evalstone = "evalstone.pytest_plugin"
48
+
49
+ [project.urls]
50
+ Homepage = "https://evalstone.com"
51
+ Documentation = "https://evalstone.com/docs"
52
+ Repository = "https://github.com/marceloribeiro/evalstone"
53
+
54
+ [tool.hatch.version]
55
+ path = "src/evalstone/__init__.py"
56
+
57
+ [tool.hatch.build.targets.wheel]
58
+ packages = ["src/evalstone"]
59
+
60
+ [tool.pytest.ini_options]
61
+ testpaths = ["tests"]
62
+ addopts = "-q"
63
+
64
+ [tool.ruff]
65
+ line-length = 100
66
+ target-version = "py312"
67
+ src = ["src", "tests"]
68
+
69
+ [tool.ruff.lint]
70
+ select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
71
+
72
+ [tool.ruff.format]
73
+ quote-style = "double"
@@ -0,0 +1,131 @@
1
+ """Evalstone — the plug-and-play evals layer for LLM-based projects.
2
+
3
+ The library computes; it owns no database. See https://evalstone.com/docs.
4
+ """
5
+
6
+ import sys
7
+ import types
8
+ from importlib import import_module
9
+ from typing import TYPE_CHECKING, Any
10
+
11
+ __version__ = "0.1.0"
12
+
13
+ # Exports resolve on first access (PEP 562) so that importing the package — which
14
+ # the pytest plugin does in every pytest session on the interpreter — costs
15
+ # nothing until a name is used.
16
+ _EXPORTS: dict[str, str] = {
17
+ "check": "evalstone.checks",
18
+ "registry": "evalstone.checks",
19
+ "Dataset": "evalstone.dataset",
20
+ "Sample": "evalstone.dataset",
21
+ "Source": "evalstone.dataset",
22
+ "Gate": "evalstone.gate",
23
+ "GateFailed": "evalstone.gate",
24
+ "GateReport": "evalstone.gate",
25
+ "Output": "evalstone.output",
26
+ "Providers": "evalstone.providers",
27
+ "Comparison": "evalstone.run",
28
+ "Result": "evalstone.run",
29
+ "Run": "evalstone.run",
30
+ "RunSummary": "evalstone.run",
31
+ "compare": "evalstone.run",
32
+ "write_report": "evalstone.run",
33
+ "Check": "evalstone.score",
34
+ "CheckKind": "evalstone.score",
35
+ "Context": "evalstone.score",
36
+ "MemoryCache": "evalstone.score",
37
+ "Score": "evalstone.score",
38
+ "Suite": "evalstone.suite",
39
+ "CallableTarget": "evalstone.target",
40
+ "HttpTarget": "evalstone.target",
41
+ "PromptTarget": "evalstone.target",
42
+ "RecordedTarget": "evalstone.target",
43
+ "Target": "evalstone.target",
44
+ "target": "evalstone.target",
45
+ }
46
+
47
+ if TYPE_CHECKING:
48
+ from evalstone import checks
49
+ from evalstone.checks import check, registry
50
+ from evalstone.dataset import Dataset, Sample, Source
51
+ from evalstone.gate import Gate, GateFailed, GateReport
52
+ from evalstone.output import Output
53
+ from evalstone.providers import Providers
54
+ from evalstone.run import Comparison, Result, Run, RunSummary, compare, write_report
55
+ from evalstone.score import Check, CheckKind, Context, MemoryCache, Score
56
+ from evalstone.suite import Suite
57
+ from evalstone.target import (
58
+ CallableTarget,
59
+ HttpTarget,
60
+ PromptTarget,
61
+ RecordedTarget,
62
+ Target,
63
+ target,
64
+ )
65
+
66
+
67
+ class _Package(types.ModuleType):
68
+ """``evalstone.target`` is the factory function, as it always was, even though a
69
+ submodule of the same name exists: importing that submodule makes the import
70
+ system bind it onto the package, which the setter below ignores."""
71
+
72
+ @property
73
+ def target(self) -> Any:
74
+ return import_module("evalstone.target").target
75
+
76
+ @target.setter
77
+ def target(self, value: Any) -> None:
78
+ pass
79
+
80
+
81
+ sys.modules[__name__].__class__ = _Package
82
+
83
+
84
+ def __getattr__(name: str) -> Any:
85
+ if name == "checks":
86
+ return import_module("evalstone.checks")
87
+ try:
88
+ module = _EXPORTS[name]
89
+ except KeyError:
90
+ raise AttributeError(f"module 'evalstone' has no attribute {name!r}") from None
91
+ value = getattr(import_module(module), name)
92
+ globals()[name] = value
93
+ return value
94
+
95
+
96
+ def __dir__() -> list[str]:
97
+ return sorted(set(__all__) | {n for n in globals() if n.startswith("__")})
98
+
99
+
100
+ __all__ = [
101
+ "CallableTarget",
102
+ "Check",
103
+ "CheckKind",
104
+ "Comparison",
105
+ "Context",
106
+ "Dataset",
107
+ "Gate",
108
+ "GateFailed",
109
+ "GateReport",
110
+ "HttpTarget",
111
+ "MemoryCache",
112
+ "Output",
113
+ "PromptTarget",
114
+ "Providers",
115
+ "RecordedTarget",
116
+ "Result",
117
+ "Run",
118
+ "RunSummary",
119
+ "Sample",
120
+ "Score",
121
+ "Source",
122
+ "Suite",
123
+ "Target",
124
+ "__version__",
125
+ "check",
126
+ "checks",
127
+ "compare",
128
+ "registry",
129
+ "target",
130
+ "write_report",
131
+ ]
@@ -0,0 +1,3 @@
1
+ from evalstone.cli import main
2
+
3
+ main()
@@ -0,0 +1,22 @@
1
+ """Best-effort git facts for run provenance. Never raises."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import subprocess
6
+ from pathlib import Path
7
+
8
+
9
+ def head_sha(cwd: str | Path | None = None) -> str | None:
10
+ try:
11
+ out = subprocess.run(
12
+ ["git", "rev-parse", "HEAD"],
13
+ cwd=cwd,
14
+ capture_output=True,
15
+ text=True,
16
+ timeout=5,
17
+ check=False,
18
+ )
19
+ except (OSError, subprocess.SubprocessError):
20
+ return None
21
+ sha = out.stdout.strip()
22
+ return sha if out.returncode == 0 and sha else None
@@ -0,0 +1,59 @@
1
+ """Canonical JSON helpers: one stable serialisation so content hashes are reproducible.
2
+
3
+ Only JSON-native values plus a few explicitly handled types (sets, tuples,
4
+ bytes, dates, enums, pydantic models, dataclasses) are accepted. Anything else
5
+ raises ``TypeError`` instead of being stringified — ``str(obj)`` would bake a
6
+ memory address or a hash-randomised set order into a content hash.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import base64
12
+ import dataclasses
13
+ import datetime as dt
14
+ import enum
15
+ import hashlib
16
+ import json
17
+ from typing import Any
18
+
19
+
20
+ def _encode(value: Any) -> Any:
21
+ """Convert the supported non-JSON types to a deterministic JSON-native form."""
22
+ if value is None or isinstance(value, str | int | float | bool):
23
+ return value
24
+ if isinstance(value, list):
25
+ return [_encode(v) for v in value]
26
+ if isinstance(value, dict):
27
+ if any(not isinstance(k, str) for k in value):
28
+ raise TypeError("dict keys must be strings to hash")
29
+ return {k: _encode(v) for k, v in value.items()}
30
+ if isinstance(value, enum.Enum):
31
+ return value.value
32
+ if isinstance(value, set | frozenset):
33
+ return sorted((_encode(v) for v in value), key=lambda v: json.dumps(v, sort_keys=True))
34
+ if isinstance(value, tuple):
35
+ return [_encode(v) for v in value]
36
+ if isinstance(value, bytes | bytearray):
37
+ return {"__bytes__": base64.b64encode(bytes(value)).decode("ascii")}
38
+ if isinstance(value, dt.datetime | dt.date | dt.time):
39
+ return value.isoformat()
40
+ if hasattr(value, "model_dump") and callable(value.model_dump): # pydantic
41
+ return value.model_dump(mode="json")
42
+ if dataclasses.is_dataclass(value) and not isinstance(value, type):
43
+ return dataclasses.asdict(value)
44
+ raise TypeError(f"not JSON-serialisable for hashing/text: {type(value).__name__}")
45
+
46
+
47
+ def dumps(value: Any, **kwargs: Any) -> str:
48
+ """``json.dumps`` with the explicit encoder above (never ``default=str``)."""
49
+ return json.dumps(_encode(value), ensure_ascii=False, **kwargs)
50
+
51
+
52
+ def canonical(value: Any) -> str:
53
+ """Serialise ``value`` deterministically (sorted keys, no whitespace, UTF-8 kept)."""
54
+ return dumps(_encode(value), sort_keys=True, separators=(",", ":"))
55
+
56
+
57
+ def sha256(value: Any) -> str:
58
+ """sha256 hex digest of the canonical JSON form of ``value``."""
59
+ return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()