eval-uate-me 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. eval_uate_me-1.0.0/LICENSE +21 -0
  2. eval_uate_me-1.0.0/PKG-INFO +138 -0
  3. eval_uate_me-1.0.0/README.md +94 -0
  4. eval_uate_me-1.0.0/pyproject.toml +58 -0
  5. eval_uate_me-1.0.0/setup.cfg +4 -0
  6. eval_uate_me-1.0.0/src/eval_uate_me/__init__.py +15 -0
  7. eval_uate_me-1.0.0/src/eval_uate_me/__main__.py +3 -0
  8. eval_uate_me-1.0.0/src/eval_uate_me/adapters.py +141 -0
  9. eval_uate_me-1.0.0/src/eval_uate_me/api.py +170 -0
  10. eval_uate_me-1.0.0/src/eval_uate_me/calibration.py +192 -0
  11. eval_uate_me-1.0.0/src/eval_uate_me/cli.py +496 -0
  12. eval_uate_me-1.0.0/src/eval_uate_me/config.py +312 -0
  13. eval_uate_me-1.0.0/src/eval_uate_me/dataset_meta.py +94 -0
  14. eval_uate_me-1.0.0/src/eval_uate_me/execution.py +177 -0
  15. eval_uate_me-1.0.0/src/eval_uate_me/formats.py +38 -0
  16. eval_uate_me-1.0.0/src/eval_uate_me/gate.py +195 -0
  17. eval_uate_me-1.0.0/src/eval_uate_me/graders.py +391 -0
  18. eval_uate_me-1.0.0/src/eval_uate_me/history.py +171 -0
  19. eval_uate_me-1.0.0/src/eval_uate_me/isolation.py +198 -0
  20. eval_uate_me-1.0.0/src/eval_uate_me/judgecheck.py +152 -0
  21. eval_uate_me-1.0.0/src/eval_uate_me/judges.py +244 -0
  22. eval_uate_me-1.0.0/src/eval_uate_me/labels.py +238 -0
  23. eval_uate_me-1.0.0/src/eval_uate_me/lint.py +181 -0
  24. eval_uate_me-1.0.0/src/eval_uate_me/manifest.py +73 -0
  25. eval_uate_me-1.0.0/src/eval_uate_me/metaeval.py +209 -0
  26. eval_uate_me-1.0.0/src/eval_uate_me/models.py +79 -0
  27. eval_uate_me-1.0.0/src/eval_uate_me/mutation.py +270 -0
  28. eval_uate_me-1.0.0/src/eval_uate_me/pairwise.py +200 -0
  29. eval_uate_me-1.0.0/src/eval_uate_me/privacy.py +58 -0
  30. eval_uate_me-1.0.0/src/eval_uate_me/py.typed +0 -0
  31. eval_uate_me-1.0.0/src/eval_uate_me/registry.py +129 -0
  32. eval_uate_me-1.0.0/src/eval_uate_me/report.py +203 -0
  33. eval_uate_me-1.0.0/src/eval_uate_me/report_html.py +177 -0
  34. eval_uate_me-1.0.0/src/eval_uate_me/runner.py +404 -0
  35. eval_uate_me-1.0.0/src/eval_uate_me/scaffold.py +166 -0
  36. eval_uate_me-1.0.0/src/eval_uate_me/selftest.py +167 -0
  37. eval_uate_me-1.0.0/src/eval_uate_me/splits.py +108 -0
  38. eval_uate_me-1.0.0/src/eval_uate_me/stats.py +133 -0
  39. eval_uate_me-1.0.0/src/eval_uate_me/targets.py +85 -0
  40. eval_uate_me-1.0.0/src/eval_uate_me/worker.py +38 -0
  41. eval_uate_me-1.0.0/src/eval_uate_me.egg-info/PKG-INFO +138 -0
  42. eval_uate_me-1.0.0/src/eval_uate_me.egg-info/SOURCES.txt +72 -0
  43. eval_uate_me-1.0.0/src/eval_uate_me.egg-info/dependency_links.txt +1 -0
  44. eval_uate_me-1.0.0/src/eval_uate_me.egg-info/entry_points.txt +2 -0
  45. eval_uate_me-1.0.0/src/eval_uate_me.egg-info/requires.txt +5 -0
  46. eval_uate_me-1.0.0/src/eval_uate_me.egg-info/top_level.txt +1 -0
  47. eval_uate_me-1.0.0/tests/test_adapters.py +175 -0
  48. eval_uate_me-1.0.0/tests/test_api.py +120 -0
  49. eval_uate_me-1.0.0/tests/test_async.py +134 -0
  50. eval_uate_me-1.0.0/tests/test_config.py +97 -0
  51. eval_uate_me-1.0.0/tests/test_core.py +255 -0
  52. eval_uate_me-1.0.0/tests/test_data_governance.py +272 -0
  53. eval_uate_me-1.0.0/tests/test_docker_isolation.py +100 -0
  54. eval_uate_me-1.0.0/tests/test_formats.py +123 -0
  55. eval_uate_me-1.0.0/tests/test_gate.py +220 -0
  56. eval_uate_me-1.0.0/tests/test_hygiene.py +33 -0
  57. eval_uate_me-1.0.0/tests/test_isolation.py +108 -0
  58. eval_uate_me-1.0.0/tests/test_judge_infra.py +235 -0
  59. eval_uate_me-1.0.0/tests/test_judges.py +297 -0
  60. eval_uate_me-1.0.0/tests/test_labelling.py +319 -0
  61. eval_uate_me-1.0.0/tests/test_lint.py +43 -0
  62. eval_uate_me-1.0.0/tests/test_metaeval.py +61 -0
  63. eval_uate_me-1.0.0/tests/test_mutation.py +264 -0
  64. eval_uate_me-1.0.0/tests/test_openai_compat.py +133 -0
  65. eval_uate_me-1.0.0/tests/test_plugins.py +98 -0
  66. eval_uate_me-1.0.0/tests/test_privacy.py +97 -0
  67. eval_uate_me-1.0.0/tests/test_public_api.py +69 -0
  68. eval_uate_me-1.0.0/tests/test_reporting.py +144 -0
  69. eval_uate_me-1.0.0/tests/test_resume_budget.py +112 -0
  70. eval_uate_me-1.0.0/tests/test_scaffold.py +50 -0
  71. eval_uate_me-1.0.0/tests/test_scoring_policy.py +65 -0
  72. eval_uate_me-1.0.0/tests/test_selftest.py +127 -0
  73. eval_uate_me-1.0.0/tests/test_statistics.py +69 -0
  74. eval_uate_me-1.0.0/tests/test_tutorial.py +45 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Chuanhui-Zheng
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,138 @@
1
+ Metadata-Version: 2.4
2
+ Name: eval_uate_me
3
+ Version: 1.0.0
4
+ Summary: A small, plugin-based eval harness
5
+ License: MIT License
6
+
7
+ Copyright (c) 2026 Chuanhui-Zheng
8
+
9
+ Permission is hereby granted, free of charge, to any person obtaining a copy
10
+ of this software and associated documentation files (the "Software"), to deal
11
+ in the Software without restriction, including without limitation the rights
12
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13
+ copies of the Software, and to permit persons to whom the Software is
14
+ furnished to do so, subject to the following conditions:
15
+
16
+ The above copyright notice and this permission notice shall be included in all
17
+ copies or substantial portions of the Software.
18
+
19
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
+ SOFTWARE.
26
+
27
+ Project-URL: Homepage, https://github.com/Chuanhui-Zheng/Eval-uate-Me
28
+ Keywords: evaluation,llm,agents,eval-harness
29
+ Classifier: Development Status :: 4 - Beta
30
+ Classifier: License :: OSI Approved :: MIT License
31
+ Classifier: Operating System :: OS Independent
32
+ Classifier: Typing :: Typed
33
+ Classifier: Programming Language :: Python :: 3.11
34
+ Classifier: Programming Language :: Python :: 3.12
35
+ Classifier: Topic :: Software Development :: Testing
36
+ Requires-Python: >=3.11
37
+ Description-Content-Type: text/markdown
38
+ License-File: LICENSE
39
+ Provides-Extra: dev
40
+ Requires-Dist: pytest>=8; extra == "dev"
41
+ Requires-Dist: pytest-cov>=5; extra == "dev"
42
+ Requires-Dist: mypy>=1.10; extra == "dev"
43
+ Dynamic: license-file
44
+
45
+ # eval_uate_me
46
+
47
+ **Run an evaluation, gate regressions, and test whether your evaluation catches a broken agent.**
48
+
49
+ A small, plugin-based harness for repeatable LLM-agent evaluations. Zero runtime dependencies (Python ≥ 3.11).
50
+
51
+ ```
52
+ cases.jsonl ──► Runner ──► Target ──► output ──► Graders ──► results + summary + gate
53
+ ```
54
+
55
+ **Status:** v1.0.7 is the current stable release. Includes a multi-turn tool-loop retrieval agent, session memory, process/Docker sandbox isolation, and an 11-grader CVE triage benchmark across four models. Zero runtime dependencies; deterministic example runs offline. Not published to PyPI; install from a clone or a release wheel.
56
+
57
+ ## The main path in three commands
58
+
59
+ ### 1. Run an evaluation
60
+
61
+ ```bash
62
+ python3.12 -m venv .venv && .venv/bin/pip install -e ".[dev]"
63
+ .venv/bin/eval_uate_me init my-eval && cd my-eval
64
+ ../.venv/bin/eval_uate_me check eval.toml
65
+ ../.venv/bin/eval_uate_me run eval.toml
66
+ ```
67
+
68
+ `init` creates a runnable starter project (config, cases, provenance file). Point the `[target]` at your agent: use the built-in `http` or `python` type for a service or a function, or write a plugin class for anything stateful. `check` validates config and data before anything runs.
69
+
70
+ ### 2. Gate regressions in CI
71
+
72
+ ```bash
73
+ eval_uate_me run eval.toml
74
+ eval_uate_me baseline promote --config eval.toml # writes eval.baseline.json — commit it
75
+ eval_uate_me gate eval.toml # exit 0 pass, 1 regression, 2 cannot decide
76
+ ```
77
+
78
+ The gate compares the latest run with the baseline. Add rules to the config once you have a baseline:
79
+
80
+ ```toml
81
+ [gate]
82
+ significant_only = true # fail only when the drop is statistically significant
83
+ [gate.min]
84
+ pass_rate = 0.90
85
+ [gate.max_drop]
86
+ pass_rate = 0.05
87
+ ```
88
+
89
+ A metric the rules name but the run lacks is a failure (a typo must not silently pass). A different dataset or split is refused rather than compared. GitHub Actions gets `::error::` annotations automatically.
90
+
91
+ ### 3. Test whether the evaluation catches a broken agent (`meta-eval`)
92
+
93
+ ```toml
94
+ [[mutants]]
95
+ name = "drops-the-answer"
96
+ [mutants.target]
97
+ type = "my_agent"
98
+ flaw = "omit"
99
+ ```
100
+
101
+ ```bash
102
+ eval_uate_me meta-eval eval.toml # killed / survived / no-op per mutant
103
+ eval_uate_me meta-eval eval.toml --auto # damage every field of the real outputs; name blind spots
104
+ ```
105
+
106
+ **killed** = the evaluation scored this mutant significantly lower. **survived** = the evaluation cannot tell it from the real agent — a blind spot to fix with a better grader or more cases. `--auto` finds the blind spots you did not think to write.
107
+
108
+ `examples/cve_triage/` is the primary worked example: a retrieval-loop agent over 100 public CISA KEV vulnerabilities,
109
+ 420 grouped cases, **11 graders** (schema, identity, CVSS, CWE, KEV flag, affected product, references, fabrication,
110
+ tool-call assertion, prompt-injection resistance, safety/disclaimer), and deliberate mutants for each failure mode.
111
+ Multi-model benchmark on the **dev split** (25 cases each): deepseek-v4.1-flash 96.7%, gpt-4o-mini 91.6%, claude-haiku-5.5 91.6% (v1.0.5 run). Held-out multi-model results not yet published.
112
+
113
+ `examples/ticket_triage/` is a second worked example: 50 customer-support tickets across 5 categories and 3 urgency
114
+ levels, **3 graders** (schema, category, urgency), and an LLM target (`eval.llm.toml`) for benchmarking real models
115
+ alongside the deterministic oracle. Demonstrates the harness on a pure-classification task with no tool use.
116
+
117
+ `examples/support_reply/` is a third worked example: 30 customer-support tickets where the agent must **draft a reply**
118
+ rather than classify. No single correct answer exists — quality is judged by an LLM judge on three dimensions
119
+ (addresses issue, actionable, appropriate tone). Demonstrates the harness on open-ended generation, the third evaluation
120
+ paradigm alongside factual lookup and classification.
121
+
122
+ ## Platforms
123
+
124
+ CI runs on **Linux** (Python 3.11 and 3.12), **Windows** (3.11 and 3.12) and **macOS** (3.12). Docker sandbox tests run on Linux. On Windows, `cpu_limit_s` and `memory_limit_mb` are not enforced in `process` isolation; use `docker` for hard limits.
125
+
126
+ ## Documentation
127
+
128
+ | File | Contents |
129
+ |---|---|
130
+ | [docs/TUTORIAL.md](docs/TUTORIAL.md) | step-by-step walkthrough; every command is run by the tests |
131
+ | [docs/REFERENCE.md](docs/REFERENCE.md) | every command, option and config key |
132
+ | [docs/API.md](docs/API.md) | writing targets, graders and judge backends (plugin API) |
133
+ | [docs/VERSIONING.md](docs/VERSIONING.md) | what is stable from 1.0 and what is not |
134
+ | [docs/DESIGN.md](docs/DESIGN.md) | trade-offs and known limits |
135
+ | [docs/ROADMAP.md](docs/ROADMAP.md) | what shipped, what is pending |
136
+ | [docs/RELEASING.md](docs/RELEASING.md) | how to cut a release |
137
+
138
+ Licensed under the [MIT licence](LICENSE).
@@ -0,0 +1,94 @@
1
+ # eval_uate_me
2
+
3
+ **Run an evaluation, gate regressions, and test whether your evaluation catches a broken agent.**
4
+
5
+ A small, plugin-based harness for repeatable LLM-agent evaluations. Zero runtime dependencies (Python ≥ 3.11).
6
+
7
+ ```
8
+ cases.jsonl ──► Runner ──► Target ──► output ──► Graders ──► results + summary + gate
9
+ ```
10
+
11
+ **Status:** v1.0.7 is the current stable release. Includes a multi-turn tool-loop retrieval agent, session memory, process/Docker sandbox isolation, and an 11-grader CVE triage benchmark across four models. Zero runtime dependencies; deterministic example runs offline. Not published to PyPI; install from a clone or a release wheel.
12
+
13
+ ## The main path in three commands
14
+
15
+ ### 1. Run an evaluation
16
+
17
+ ```bash
18
+ python3.12 -m venv .venv && .venv/bin/pip install -e ".[dev]"
19
+ .venv/bin/eval_uate_me init my-eval && cd my-eval
20
+ ../.venv/bin/eval_uate_me check eval.toml
21
+ ../.venv/bin/eval_uate_me run eval.toml
22
+ ```
23
+
24
+ `init` creates a runnable starter project (config, cases, provenance file). Point the `[target]` at your agent: use the built-in `http` or `python` type for a service or a function, or write a plugin class for anything stateful. `check` validates config and data before anything runs.
25
+
26
+ ### 2. Gate regressions in CI
27
+
28
+ ```bash
29
+ eval_uate_me run eval.toml
30
+ eval_uate_me baseline promote --config eval.toml # writes eval.baseline.json — commit it
31
+ eval_uate_me gate eval.toml # exit 0 pass, 1 regression, 2 cannot decide
32
+ ```
33
+
34
+ The gate compares the latest run with the baseline. Add rules to the config once you have a baseline:
35
+
36
+ ```toml
37
+ [gate]
38
+ significant_only = true # fail only when the drop is statistically significant
39
+ [gate.min]
40
+ pass_rate = 0.90
41
+ [gate.max_drop]
42
+ pass_rate = 0.05
43
+ ```
44
+
45
+ A metric the rules name but the run lacks is a failure (a typo must not silently pass). A different dataset or split is refused rather than compared. GitHub Actions gets `::error::` annotations automatically.
46
+
47
+ ### 3. Test whether the evaluation catches a broken agent (`meta-eval`)
48
+
49
+ ```toml
50
+ [[mutants]]
51
+ name = "drops-the-answer"
52
+ [mutants.target]
53
+ type = "my_agent"
54
+ flaw = "omit"
55
+ ```
56
+
57
+ ```bash
58
+ eval_uate_me meta-eval eval.toml # killed / survived / no-op per mutant
59
+ eval_uate_me meta-eval eval.toml --auto # damage every field of the real outputs; name blind spots
60
+ ```
61
+
62
+ **killed** = the evaluation scored this mutant significantly lower. **survived** = the evaluation cannot tell it from the real agent — a blind spot to fix with a better grader or more cases. `--auto` finds the blind spots you did not think to write.
63
+
64
+ `examples/cve_triage/` is the primary worked example: a retrieval-loop agent over 100 public CISA KEV vulnerabilities,
65
+ 420 grouped cases, **11 graders** (schema, identity, CVSS, CWE, KEV flag, affected product, references, fabrication,
66
+ tool-call assertion, prompt-injection resistance, safety/disclaimer), and deliberate mutants for each failure mode.
67
+ Multi-model benchmark on the **dev split** (25 cases each): deepseek-v4.1-flash 96.7%, gpt-4o-mini 91.6%, claude-haiku-5.5 91.6% (v1.0.5 run). Held-out multi-model results not yet published.
68
+
69
+ `examples/ticket_triage/` is a second worked example: 50 customer-support tickets across 5 categories and 3 urgency
70
+ levels, **3 graders** (schema, category, urgency), and an LLM target (`eval.llm.toml`) for benchmarking real models
71
+ alongside the deterministic oracle. Demonstrates the harness on a pure-classification task with no tool use.
72
+
73
+ `examples/support_reply/` is a third worked example: 30 customer-support tickets where the agent must **draft a reply**
74
+ rather than classify. No single correct answer exists — quality is judged by an LLM judge on three dimensions
75
+ (addresses issue, actionable, appropriate tone). Demonstrates the harness on open-ended generation, the third evaluation
76
+ paradigm alongside factual lookup and classification.
77
+
78
+ ## Platforms
79
+
80
+ CI runs on **Linux** (Python 3.11 and 3.12), **Windows** (3.11 and 3.12) and **macOS** (3.12). Docker sandbox tests run on Linux. On Windows, `cpu_limit_s` and `memory_limit_mb` are not enforced in `process` isolation; use `docker` for hard limits.
81
+
82
+ ## Documentation
83
+
84
+ | File | Contents |
85
+ |---|---|
86
+ | [docs/TUTORIAL.md](docs/TUTORIAL.md) | step-by-step walkthrough; every command is run by the tests |
87
+ | [docs/REFERENCE.md](docs/REFERENCE.md) | every command, option and config key |
88
+ | [docs/API.md](docs/API.md) | writing targets, graders and judge backends (plugin API) |
89
+ | [docs/VERSIONING.md](docs/VERSIONING.md) | what is stable from 1.0 and what is not |
90
+ | [docs/DESIGN.md](docs/DESIGN.md) | trade-offs and known limits |
91
+ | [docs/ROADMAP.md](docs/ROADMAP.md) | what shipped, what is pending |
92
+ | [docs/RELEASING.md](docs/RELEASING.md) | how to cut a release |
93
+
94
+ Licensed under the [MIT licence](LICENSE).
@@ -0,0 +1,58 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "eval_uate_me"
7
+ version = "1.0.0"
8
+ description = "A small, plugin-based eval harness"
9
+ readme = "README.md"
10
+ license = {file = "LICENSE"}
11
+ keywords = ["evaluation", "llm", "agents", "eval-harness"]
12
+ classifiers = [
13
+ "Development Status :: 4 - Beta",
14
+ "License :: OSI Approved :: MIT License",
15
+ "Operating System :: OS Independent",
16
+ "Typing :: Typed",
17
+ "Programming Language :: Python :: 3.11",
18
+ "Programming Language :: Python :: 3.12",
19
+ "Topic :: Software Development :: Testing",
20
+ ]
21
+ requires-python = ">=3.11"
22
+ dependencies = []
23
+
24
+ [project.urls]
25
+ Homepage = "https://github.com/Chuanhui-Zheng/Eval-uate-Me"
26
+
27
+ [project.optional-dependencies]
28
+ dev = ["pytest>=8", "pytest-cov>=5", "mypy>=1.10"]
29
+
30
+ [project.scripts]
31
+ eval_uate_me = "eval_uate_me.cli:main"
32
+
33
+ [tool.setuptools.packages.find]
34
+ where = ["src"]
35
+
36
+ [tool.setuptools.package-data]
37
+ eval_uate_me = ["py.typed"] # PEP 561: downstream plugins get this package's types
38
+
39
+ [tool.pytest.ini_options]
40
+ testpaths = ["tests", "examples/cve_triage/tests", "examples/ticket_triage/tests", "examples/support_reply/tests"]
41
+ pythonpath = ["examples/cve_triage", "examples/ticket_triage", "examples/support_reply"]
42
+
43
+ [tool.mypy]
44
+ python_version = "3.11"
45
+ files = ["src/eval_uate_me"]
46
+ warn_unused_ignores = true
47
+ warn_redundant_casts = true
48
+ check_untyped_defs = true
49
+
50
+ [tool.coverage.run]
51
+ source = ["eval_uate_me"]
52
+ omit = ["*/__main__.py"]
53
+
54
+ [tool.coverage.report]
55
+ # worker.py runs in a subprocess and is not measured; the Docker paths of isolation.py run only in the CI docker job
56
+ fail_under = 93
57
+ show_missing = true
58
+ skip_covered = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,15 @@
1
+ __version__ = "1.0.0"
2
+
3
+ from .models import Case, CaseResult, Score, TargetOutput, TraceStep
4
+ from .config import ConfigError, PluginError
5
+ from .registry import API_VERSION, register_grader, register_judge, register_target
6
+ from . import adapters # noqa: F401 (registers the http and python targets)
7
+ from .graders import Grader
8
+ from .judges import Judge
9
+ from .api import EvalResult, evaluate, run_eval
10
+ from .targets import Target
11
+
12
+ __all__ = [
13
+ "Case", "CaseResult", "Score", "TargetOutput",
14
+ "API_VERSION", "ConfigError", "EvalResult", "PluginError", "Grader", "Judge", "Target", "TraceStep", "evaluate", "register_grader", "register_judge", "register_target", "run_eval",
15
+ ]
@@ -0,0 +1,3 @@
1
+ import sys
2
+ from .cli import main
3
+ sys.exit(main())
@@ -0,0 +1,141 @@
1
+ """Built-in targets for systems you already have: an HTTP endpoint, or a Python function.
2
+
3
+ [target] [target]
4
+ type = "http" type = "python"
5
+ url = "http://localhost:8000/answer" callable = "my_app.agent:answer" # module:function
6
+ headers = {Authorization = "Bearer ${ENV:MY_TOKEN}"} # ${ENV:NAME} is read from the environment, never stored
7
+
8
+ `http` POSTs {"id", "input", "metadata"} as JSON and reads {"output": ..., optional "trace", "tool_calls", "usage",
9
+ "cost_usd", "error"}. `python` calls function(input) (or function(case) with with_case = true); it may return the output
10
+ itself or a TargetOutput, and may be `async def` (then the harness keeps many calls in flight). Neither needs a plugin class.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import importlib
15
+ import inspect
16
+ import json
17
+ import os
18
+ import re
19
+ import sys
20
+ import urllib.error
21
+ import urllib.request
22
+ from typing import Any
23
+ from urllib.parse import urlparse
24
+
25
+ from .config import ConfigError
26
+ from .models import Case, TargetOutput
27
+ from .registry import register_target
28
+ from .targets import Target
29
+
30
+ _ENV = re.compile(r"\$\{ENV:([A-Za-z_][A-Za-z0-9_]*)\}")
31
+
32
+
33
+ def expand_env(text: str, where: str) -> str:
34
+ """Replace ${ENV:NAME} with the environment variable, failing clearly if it is not set."""
35
+ def sub(m: re.Match[str]) -> str:
36
+ name = m.group(1)
37
+ if name not in os.environ:
38
+ raise ConfigError(f"{where}: environment variable {name} is not set")
39
+ return os.environ[name]
40
+ return _ENV.sub(sub, text)
41
+
42
+
43
+ def _dig(data: Any, path: str) -> Any:
44
+ for part in path.split("."):
45
+ if isinstance(data, dict) and part in data:
46
+ data = data[part]
47
+ else:
48
+ raise KeyError(path)
49
+ return data
50
+
51
+
52
+ @register_target("http")
53
+ class HttpTarget(Target):
54
+ """POST each case to a URL. Transport errors, non-2xx statuses and malformed bodies become error results for that case
55
+ (the run continues; use run.retries to retry them)."""
56
+ eval_uate_me_api = 1
57
+ network = True
58
+
59
+ def __init__(self, url: str, headers: dict[str, str] | None = None, timeout_s: float = 30.0,
60
+ output_field: str = "output", send_metadata: bool = True):
61
+ if urlparse(url).scheme not in ("http", "https"):
62
+ raise ConfigError(f"[target] url must start with http:// or https://, got {url!r}")
63
+ self.url, self.timeout_s, self.output_field, self.send_metadata = url, timeout_s, output_field, send_metadata
64
+ self.headers = {"Content-Type": "application/json", "Accept": "application/json",
65
+ **{k: expand_env(str(v), f"[target] headers.{k}") for k, v in (headers or {}).items()}}
66
+
67
+ def run(self, case: Case) -> TargetOutput:
68
+ payload = {"id": case.id, "input": case.input, **({"metadata": case.metadata} if self.send_metadata else {})}
69
+ req = urllib.request.Request(self.url, data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
70
+ headers=self.headers, method="POST")
71
+ try:
72
+ with urllib.request.urlopen(req, timeout=self.timeout_s) as resp:
73
+ raw = resp.read().decode("utf-8", errors="replace")
74
+ except urllib.error.HTTPError as e:
75
+ return TargetOutput(error=f"HTTP {e.code}: {e.read()[:200].decode('utf-8', errors='replace')}")
76
+ except (urllib.error.URLError, TimeoutError, OSError) as e:
77
+ return TargetOutput(error=f"http error: {getattr(e, 'reason', e)}")
78
+ try:
79
+ body = json.loads(raw)
80
+ except ValueError as e:
81
+ return TargetOutput(error=f"invalid JSON response: {e}")
82
+ if not isinstance(body, dict):
83
+ return TargetOutput(error="response JSON must be an object with an 'output' field")
84
+ if body.get("error"):
85
+ return TargetOutput(error=str(body["error"]))
86
+ try:
87
+ output = _dig(body, self.output_field)
88
+ except KeyError:
89
+ return TargetOutput(error=f"response JSON has no '{self.output_field}' field")
90
+ trace = list(body.get("trace", [])) + [{"tool": str(t)} for t in body.get("tool_calls", [])]
91
+ usage = {k: float(v) for k, v in (body.get("usage") or {}).items() if isinstance(v, (int, float)) and not isinstance(v, bool)}
92
+ if isinstance(body.get("cost_usd"), (int, float)):
93
+ usage["cost_usd"] = float(body["cost_usd"])
94
+ return TargetOutput(output=output, trace=trace, usage=usage)
95
+
96
+
97
+ def _to_output(value: Any) -> TargetOutput:
98
+ return value if isinstance(value, TargetOutput) else TargetOutput(output=value)
99
+
100
+
101
+ @register_target("python")
102
+ class PythonTarget(Target):
103
+ """Call a Python function. `callable = "package.module:function"`; add `paths` for folders to import from."""
104
+ eval_uate_me_api = 1
105
+
106
+ def __init__(self, callable: str, with_case: bool = False, paths: list[str] | None = None,
107
+ network: bool | None = None):
108
+ mod, _, attr = callable.partition(":")
109
+ if not mod or not attr:
110
+ raise ConfigError(f"[target] callable must look like 'package.module:function', got {callable!r}")
111
+ for p in paths or []:
112
+ if p not in sys.path:
113
+ sys.path.insert(0, p)
114
+ try:
115
+ obj: Any = importlib.import_module(mod)
116
+ for part in attr.split("."):
117
+ obj = getattr(obj, part)
118
+ except (ImportError, AttributeError) as e:
119
+ raise ConfigError(f"[target] cannot load '{callable}': {type(e).__name__}: {e}"
120
+ + ("" if paths else " (set `paths = [...]` if the module is not importable from here)")) from e
121
+ if not callable_(obj):
122
+ raise ConfigError(f"[target] '{callable}' is not callable")
123
+ self.fn, self.with_case, self.network = obj, with_case, network
124
+ if inspect.iscoroutinefunction(obj):
125
+ self.arun = self._arun # makes the harness use its async executor
126
+
127
+ def _arg(self, case: Case) -> Any:
128
+ return case if self.with_case else case.input
129
+
130
+ def run(self, case: Case) -> TargetOutput:
131
+ if inspect.iscoroutinefunction(self.fn):
132
+ import asyncio
133
+ return asyncio.run(self._arun(case))
134
+ return _to_output(self.fn(self._arg(case)))
135
+
136
+ async def _arun(self, case: Case) -> TargetOutput:
137
+ return _to_output(await self.fn(self._arg(case)))
138
+
139
+
140
+ def callable_(obj: Any) -> bool: # builtin name `callable` is shadowed by the option name above
141
+ return hasattr(obj, "__call__")
@@ -0,0 +1,170 @@
1
+ """Run evals from Python (a notebook, a script, a test) without a config file.
2
+
3
+ from eval_uate_me import evaluate
4
+
5
+ result = evaluate(my_function, cases=[{"input": "2+2", "expected": "4"}, ("3+3", "6")])
6
+ result.pass_rate # 1.0
7
+ result.failures() # the CaseResults that failed
8
+ result.to_rows() # flat dicts, ready for a table or a DataFrame
9
+
10
+ `evaluate` works in memory (nothing is written to disk). `run_eval("eval.toml")` runs a config exactly as the command line does
11
+ and returns the same kind of result, with the run directory. Both are part of the stable API.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import inspect
16
+ from dataclasses import dataclass, field
17
+ from pathlib import Path
18
+ from typing import Any, Callable, Iterable
19
+
20
+ from . import registry
21
+ from .adapters import PythonTarget
22
+ from .config import Config, ConfigError, load_config
23
+ from .execution import ExecPolicy, is_async, run_async, run_threaded
24
+ from .graders import ExactMatch, Grader
25
+ from .models import Case, CaseResult, FORMAT_VERSION, Score, TargetOutput
26
+ from .runner import _grade, _run_case, _run_case_async, run, summarize
27
+ from .targets import Target
28
+
29
+
30
+ @dataclass
31
+ class EvalResult:
32
+ """What an evaluation produced. `run_dir` is None for in-memory runs."""
33
+ results: list[CaseResult]
34
+ summary: dict[str, Any]
35
+ run_dir: Path | None = None
36
+ _rows: list[dict[str, Any]] = field(default_factory=list, repr=False)
37
+
38
+ @property
39
+ def pass_rate(self) -> float:
40
+ return float(self.summary["pass_rate"])
41
+
42
+ def failures(self) -> list[CaseResult]:
43
+ return [r for r in self.results if not r.passed]
44
+
45
+ def to_rows(self) -> list[dict[str, Any]]:
46
+ """One flat dict per attempt: case_id, attempt, passed, error, latency_s, input, expected, output and one
47
+ `score:<grader>` column per grader."""
48
+ rows = []
49
+ for r in self.results:
50
+ row: dict[str, Any] = {"case_id": r.case.id, "attempt": r.attempt, "passed": r.passed, "error": r.result.error,
51
+ "latency_s": r.result.latency_s, "input": r.case.input, "expected": r.case.expected,
52
+ "output": r.result.output}
53
+ row.update({f"score:{s.grader}": s.value for s in r.scores})
54
+ rows.append(row)
55
+ return rows
56
+
57
+
58
+ class _FunctionTarget(Target):
59
+ eval_uate_me_api = 1
60
+
61
+ def __init__(self, fn: Callable[..., Any]):
62
+ self.fn = fn
63
+ if inspect.iscoroutinefunction(fn):
64
+ self.arun = self._arun
65
+
66
+ def run(self, case: Case) -> TargetOutput:
67
+ if inspect.iscoroutinefunction(self.fn):
68
+ import asyncio
69
+ return asyncio.run(self._arun(case))
70
+ v = self.fn(case.input)
71
+ return v if isinstance(v, TargetOutput) else TargetOutput(output=v)
72
+
73
+ async def _arun(self, case: Case) -> TargetOutput:
74
+ v = await self.fn(case.input)
75
+ return v if isinstance(v, TargetOutput) else TargetOutput(output=v)
76
+
77
+
78
+ class _FunctionGrader(Grader):
79
+ eval_uate_me_api = 1
80
+
81
+ def __init__(self, fn: Callable[[Case, TargetOutput], Any]):
82
+ self.fn, self.name = fn, getattr(fn, "__name__", "function_grader")
83
+
84
+ def grade(self, case: Case, out: TargetOutput) -> Score:
85
+ v = self.fn(case, out)
86
+ if isinstance(v, Score):
87
+ return v
88
+ if isinstance(v, bool):
89
+ return Score(self.name, 1.0 if v else 0.0, v, "" if v else "returned False")
90
+ raise TypeError(f"grader function '{self.name}' must return a bool or a Score, got {type(v).__name__}")
91
+
92
+
93
+ def _target(t: Any) -> Target:
94
+ if isinstance(t, Target):
95
+ return t
96
+ if isinstance(t, str):
97
+ return PythonTarget(callable=t)
98
+ if callable(t):
99
+ return _FunctionTarget(t)
100
+ raise TypeError("target must be a Target, a function, or a 'module:function' string")
101
+
102
+
103
+ def _case(c: Any, n: int) -> Case:
104
+ if isinstance(c, Case):
105
+ return c
106
+ if isinstance(c, dict):
107
+ return Case(id=str(c.get("id", f"case-{n}")), input=c.get("input"), expected=c.get("expected"), metadata=dict(c.get("metadata", {})))
108
+ if isinstance(c, (tuple, list)) and len(c) in (1, 2):
109
+ return Case(id=f"case-{n}", input=c[0], expected=c[1] if len(c) == 2 else None)
110
+ raise TypeError("a case must be a Case, a dict, or an (input, expected) tuple")
111
+
112
+
113
+ def _grader(g: Any) -> Grader:
114
+ if isinstance(g, Grader):
115
+ return g
116
+ if isinstance(g, str):
117
+ return registry.get_grader(g)
118
+ if isinstance(g, tuple) and len(g) == 2 and isinstance(g[0], str) and isinstance(g[1], dict):
119
+ return registry.get_grader(g[0], **g[1])
120
+ if callable(g):
121
+ return _FunctionGrader(g)
122
+ raise TypeError("a grader must be a Grader, a registered name, a (name, options) tuple, or a function(case, output) -> bool")
123
+
124
+
125
+ def evaluate(target: Any, cases: Iterable[Any], graders: Iterable[Any] | None = None, *, repeats: int = 1, concurrency: int = 4,
126
+ retries: int = 0, timeout_s: float | None = None, name: str = "eval") -> EvalResult:
127
+ """Run `target` over `cases` and grade each answer, in memory.
128
+
129
+ target a Target, a function(input) -> output (sync or async), or "module:function"
130
+ cases Case objects, dicts {id?, input, expected?, metadata?}, or (input, expected) tuples
131
+ graders Grader objects, registered names ("exact_match"), (name, {options}) tuples, or function(case, output) -> bool;
132
+ default: exact_match
133
+ """
134
+ tgt = _target(target)
135
+ case_list = [_case(c, n) for n, c in enumerate(cases, 1)]
136
+ if not case_list:
137
+ raise ConfigError("evaluate() needs at least one case")
138
+ ids = [c.id for c in case_list]
139
+ if len(set(ids)) != len(ids):
140
+ raise ConfigError(f"duplicate case ids: {sorted({i for i in ids if ids.count(i) > 1})}")
141
+ grader_list = [_grader(g) for g in (graders if graders is not None else [ExactMatch()])]
142
+ policy = ExecPolicy(retries=retries, retry_delay_s=0.0 if retries else 1.0, timeout_s=timeout_s)
143
+ jobs = [(c, a) for c in case_list for a in range(max(1, repeats))]
144
+ done: dict[tuple[str, int], CaseResult] = {}
145
+
146
+ def work(job: tuple[Case, int]) -> None:
147
+ r = _run_case(tgt, grader_list, job[0], job[1], policy)
148
+ done[(r.case.id, r.attempt)] = r
149
+
150
+ async def awork(job: tuple[Case, int]) -> None:
151
+ r = await _run_case_async(tgt, grader_list, job[0], job[1], policy)
152
+ done[(r.case.id, r.attempt)] = r
153
+
154
+ if is_async(tgt):
155
+ run_async(jobs, awork, concurrency)
156
+ else:
157
+ run_threaded(jobs, work, concurrency)
158
+ results = [done[(c.id, a)] for c, a in jobs]
159
+ summary = summarize(results, name, planned=len(jobs))
160
+ summary["cost_usd_spent"] = sum(r.result.usage.get("cost_usd", 0.0) for r in results)
161
+ summary["format_version"] = FORMAT_VERSION
162
+ return EvalResult(results, summary, None)
163
+
164
+
165
+ def run_eval(config: str | Path | Config, *, split: str | None = None, resume: str | Path | None = None,
166
+ retry_errors: bool = False, force: bool = False) -> EvalResult:
167
+ """Run an eval from a config file exactly as `eval_uate_me run` does (the run directory is written) and return the result."""
168
+ cfg = config if isinstance(config, Config) else load_config(config)
169
+ results, summary, run_dir = run(cfg, resume=Path(resume) if resume else None, retry_errors=retry_errors, force=force, split=split)
170
+ return EvalResult(results, summary, run_dir)