eval-uate-me 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eval_uate_me-1.0.0/LICENSE +21 -0
- eval_uate_me-1.0.0/PKG-INFO +138 -0
- eval_uate_me-1.0.0/README.md +94 -0
- eval_uate_me-1.0.0/pyproject.toml +58 -0
- eval_uate_me-1.0.0/setup.cfg +4 -0
- eval_uate_me-1.0.0/src/eval_uate_me/__init__.py +15 -0
- eval_uate_me-1.0.0/src/eval_uate_me/__main__.py +3 -0
- eval_uate_me-1.0.0/src/eval_uate_me/adapters.py +141 -0
- eval_uate_me-1.0.0/src/eval_uate_me/api.py +170 -0
- eval_uate_me-1.0.0/src/eval_uate_me/calibration.py +192 -0
- eval_uate_me-1.0.0/src/eval_uate_me/cli.py +496 -0
- eval_uate_me-1.0.0/src/eval_uate_me/config.py +312 -0
- eval_uate_me-1.0.0/src/eval_uate_me/dataset_meta.py +94 -0
- eval_uate_me-1.0.0/src/eval_uate_me/execution.py +177 -0
- eval_uate_me-1.0.0/src/eval_uate_me/formats.py +38 -0
- eval_uate_me-1.0.0/src/eval_uate_me/gate.py +195 -0
- eval_uate_me-1.0.0/src/eval_uate_me/graders.py +391 -0
- eval_uate_me-1.0.0/src/eval_uate_me/history.py +171 -0
- eval_uate_me-1.0.0/src/eval_uate_me/isolation.py +198 -0
- eval_uate_me-1.0.0/src/eval_uate_me/judgecheck.py +152 -0
- eval_uate_me-1.0.0/src/eval_uate_me/judges.py +244 -0
- eval_uate_me-1.0.0/src/eval_uate_me/labels.py +238 -0
- eval_uate_me-1.0.0/src/eval_uate_me/lint.py +181 -0
- eval_uate_me-1.0.0/src/eval_uate_me/manifest.py +73 -0
- eval_uate_me-1.0.0/src/eval_uate_me/metaeval.py +209 -0
- eval_uate_me-1.0.0/src/eval_uate_me/models.py +79 -0
- eval_uate_me-1.0.0/src/eval_uate_me/mutation.py +270 -0
- eval_uate_me-1.0.0/src/eval_uate_me/pairwise.py +200 -0
- eval_uate_me-1.0.0/src/eval_uate_me/privacy.py +58 -0
- eval_uate_me-1.0.0/src/eval_uate_me/py.typed +0 -0
- eval_uate_me-1.0.0/src/eval_uate_me/registry.py +129 -0
- eval_uate_me-1.0.0/src/eval_uate_me/report.py +203 -0
- eval_uate_me-1.0.0/src/eval_uate_me/report_html.py +177 -0
- eval_uate_me-1.0.0/src/eval_uate_me/runner.py +404 -0
- eval_uate_me-1.0.0/src/eval_uate_me/scaffold.py +166 -0
- eval_uate_me-1.0.0/src/eval_uate_me/selftest.py +167 -0
- eval_uate_me-1.0.0/src/eval_uate_me/splits.py +108 -0
- eval_uate_me-1.0.0/src/eval_uate_me/stats.py +133 -0
- eval_uate_me-1.0.0/src/eval_uate_me/targets.py +85 -0
- eval_uate_me-1.0.0/src/eval_uate_me/worker.py +38 -0
- eval_uate_me-1.0.0/src/eval_uate_me.egg-info/PKG-INFO +138 -0
- eval_uate_me-1.0.0/src/eval_uate_me.egg-info/SOURCES.txt +72 -0
- eval_uate_me-1.0.0/src/eval_uate_me.egg-info/dependency_links.txt +1 -0
- eval_uate_me-1.0.0/src/eval_uate_me.egg-info/entry_points.txt +2 -0
- eval_uate_me-1.0.0/src/eval_uate_me.egg-info/requires.txt +5 -0
- eval_uate_me-1.0.0/src/eval_uate_me.egg-info/top_level.txt +1 -0
- eval_uate_me-1.0.0/tests/test_adapters.py +175 -0
- eval_uate_me-1.0.0/tests/test_api.py +120 -0
- eval_uate_me-1.0.0/tests/test_async.py +134 -0
- eval_uate_me-1.0.0/tests/test_config.py +97 -0
- eval_uate_me-1.0.0/tests/test_core.py +255 -0
- eval_uate_me-1.0.0/tests/test_data_governance.py +272 -0
- eval_uate_me-1.0.0/tests/test_docker_isolation.py +100 -0
- eval_uate_me-1.0.0/tests/test_formats.py +123 -0
- eval_uate_me-1.0.0/tests/test_gate.py +220 -0
- eval_uate_me-1.0.0/tests/test_hygiene.py +33 -0
- eval_uate_me-1.0.0/tests/test_isolation.py +108 -0
- eval_uate_me-1.0.0/tests/test_judge_infra.py +235 -0
- eval_uate_me-1.0.0/tests/test_judges.py +297 -0
- eval_uate_me-1.0.0/tests/test_labelling.py +319 -0
- eval_uate_me-1.0.0/tests/test_lint.py +43 -0
- eval_uate_me-1.0.0/tests/test_metaeval.py +61 -0
- eval_uate_me-1.0.0/tests/test_mutation.py +264 -0
- eval_uate_me-1.0.0/tests/test_openai_compat.py +133 -0
- eval_uate_me-1.0.0/tests/test_plugins.py +98 -0
- eval_uate_me-1.0.0/tests/test_privacy.py +97 -0
- eval_uate_me-1.0.0/tests/test_public_api.py +69 -0
- eval_uate_me-1.0.0/tests/test_reporting.py +144 -0
- eval_uate_me-1.0.0/tests/test_resume_budget.py +112 -0
- eval_uate_me-1.0.0/tests/test_scaffold.py +50 -0
- eval_uate_me-1.0.0/tests/test_scoring_policy.py +65 -0
- eval_uate_me-1.0.0/tests/test_selftest.py +127 -0
- eval_uate_me-1.0.0/tests/test_statistics.py +69 -0
- eval_uate_me-1.0.0/tests/test_tutorial.py +45 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chuanhui-Zheng
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eval_uate_me
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A small, plugin-based eval harness
|
|
5
|
+
License: MIT License
|
|
6
|
+
|
|
7
|
+
Copyright (c) 2026 Chuanhui-Zheng
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
|
26
|
+
|
|
27
|
+
Project-URL: Homepage, https://github.com/Chuanhui-Zheng/Eval-uate-Me
|
|
28
|
+
Keywords: evaluation,llm,agents,eval-harness
|
|
29
|
+
Classifier: Development Status :: 4 - Beta
|
|
30
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
+
Classifier: Operating System :: OS Independent
|
|
32
|
+
Classifier: Typing :: Typed
|
|
33
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
35
|
+
Classifier: Topic :: Software Development :: Testing
|
|
36
|
+
Requires-Python: >=3.11
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
License-File: LICENSE
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
41
|
+
Requires-Dist: pytest-cov>=5; extra == "dev"
|
|
42
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# eval_uate_me
|
|
46
|
+
|
|
47
|
+
**Run an evaluation, gate regressions, and test whether your evaluation catches a broken agent.**
|
|
48
|
+
|
|
49
|
+
A small, plugin-based harness for repeatable LLM-agent evaluations. Zero runtime dependencies (Python ≥ 3.11).
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
cases.jsonl ──► Runner ──► Target ──► output ──► Graders ──► results + summary + gate
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**Status:** v1.0.7 is the current stable release. Includes a multi-turn tool-loop retrieval agent, session memory, process/Docker sandbox isolation, and an 11-grader CVE triage benchmark across four models. Zero runtime dependencies; deterministic example runs offline. Not published to PyPI; install from a clone or a release wheel.
|
|
56
|
+
|
|
57
|
+
## The main path in three commands
|
|
58
|
+
|
|
59
|
+
### 1. Run an evaluation
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python3.12 -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
63
|
+
.venv/bin/eval_uate_me init my-eval && cd my-eval
|
|
64
|
+
../.venv/bin/eval_uate_me check eval.toml
|
|
65
|
+
../.venv/bin/eval_uate_me run eval.toml
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`init` creates a runnable starter project (config, cases, provenance file). Point the `[target]` at your agent: use the built-in `http` or `python` type for a service or a function, or write a plugin class for anything stateful. `check` validates config and data before anything runs.
|
|
69
|
+
|
|
70
|
+
### 2. Gate regressions in CI
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
eval_uate_me run eval.toml
|
|
74
|
+
eval_uate_me baseline promote --config eval.toml # writes eval.baseline.json — commit it
|
|
75
|
+
eval_uate_me gate eval.toml # exit 0 pass, 1 regression, 2 cannot decide
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The gate compares the latest run with the baseline. Add rules to the config once you have a baseline:
|
|
79
|
+
|
|
80
|
+
```toml
|
|
81
|
+
[gate]
|
|
82
|
+
significant_only = true # fail only when the drop is statistically significant
|
|
83
|
+
[gate.min]
|
|
84
|
+
pass_rate = 0.90
|
|
85
|
+
[gate.max_drop]
|
|
86
|
+
pass_rate = 0.05
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
A metric the rules name but the run lacks is a failure (a typo must not silently pass). A different dataset or split is refused rather than compared. GitHub Actions gets `::error::` annotations automatically.
|
|
90
|
+
|
|
91
|
+
### 3. Test whether the evaluation catches a broken agent (`meta-eval`)
|
|
92
|
+
|
|
93
|
+
```toml
|
|
94
|
+
[[mutants]]
|
|
95
|
+
name = "drops-the-answer"
|
|
96
|
+
[mutants.target]
|
|
97
|
+
type = "my_agent"
|
|
98
|
+
flaw = "omit"
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
eval_uate_me meta-eval eval.toml # killed / survived / no-op per mutant
|
|
103
|
+
eval_uate_me meta-eval eval.toml --auto # damage every field of the real outputs; name blind spots
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
**killed** = the evaluation scored this mutant significantly lower. **survived** = the evaluation cannot tell it from the real agent — a blind spot to fix with a better grader or more cases. `--auto` finds the blind spots you did not think to write.
|
|
107
|
+
|
|
108
|
+
`examples/cve_triage/` is the primary worked example: a retrieval-loop agent over 100 public CISA KEV vulnerabilities,
|
|
109
|
+
420 grouped cases, **11 graders** (schema, identity, CVSS, CWE, KEV flag, affected product, references, fabrication,
|
|
110
|
+
tool-call assertion, prompt-injection resistance, safety/disclaimer), and deliberate mutants for each failure mode.
|
|
111
|
+
Multi-model benchmark on the **dev split** (25 cases each): deepseek-v4.1-flash 96.7%, gpt-4o-mini 91.6%, claude-haiku-5.5 91.6% (v1.0.5 run). Held-out multi-model results not yet published.
|
|
112
|
+
|
|
113
|
+
`examples/ticket_triage/` is a second worked example: 50 customer-support tickets across 5 categories and 3 urgency
|
|
114
|
+
levels, **3 graders** (schema, category, urgency), and an LLM target (`eval.llm.toml`) for benchmarking real models
|
|
115
|
+
alongside the deterministic oracle. Demonstrates the harness on a pure-classification task with no tool use.
|
|
116
|
+
|
|
117
|
+
`examples/support_reply/` is a third worked example: 30 customer-support tickets where the agent must **draft a reply**
|
|
118
|
+
rather than classify. No single correct answer exists — quality is judged by an LLM judge on three dimensions
|
|
119
|
+
(addresses issue, actionable, appropriate tone). Demonstrates the harness on open-ended generation, the third evaluation
|
|
120
|
+
paradigm alongside factual lookup and classification.
|
|
121
|
+
|
|
122
|
+
## Platforms
|
|
123
|
+
|
|
124
|
+
CI runs on **Linux** (Python 3.11 and 3.12), **Windows** (3.11 and 3.12) and **macOS** (3.12). Docker sandbox tests run on Linux. On Windows, `cpu_limit_s` and `memory_limit_mb` are not enforced in `process` isolation; use `docker` for hard limits.
|
|
125
|
+
|
|
126
|
+
## Documentation
|
|
127
|
+
|
|
128
|
+
| File | Contents |
|
|
129
|
+
|---|---|
|
|
130
|
+
| [docs/TUTORIAL.md](docs/TUTORIAL.md) | step-by-step walkthrough; every command is run by the tests |
|
|
131
|
+
| [docs/REFERENCE.md](docs/REFERENCE.md) | every command, option and config key |
|
|
132
|
+
| [docs/API.md](docs/API.md) | writing targets, graders and judge backends (plugin API) |
|
|
133
|
+
| [docs/VERSIONING.md](docs/VERSIONING.md) | what is stable from 1.0 and what is not |
|
|
134
|
+
| [docs/DESIGN.md](docs/DESIGN.md) | trade-offs and known limits |
|
|
135
|
+
| [docs/ROADMAP.md](docs/ROADMAP.md) | what shipped, what is pending |
|
|
136
|
+
| [docs/RELEASING.md](docs/RELEASING.md) | how to cut a release |
|
|
137
|
+
|
|
138
|
+
Licensed under the [MIT licence](LICENSE).
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# eval_uate_me
|
|
2
|
+
|
|
3
|
+
**Run an evaluation, gate regressions, and test whether your evaluation catches a broken agent.**
|
|
4
|
+
|
|
5
|
+
A small, plugin-based harness for repeatable LLM-agent evaluations. Zero runtime dependencies (Python ≥ 3.11).
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
cases.jsonl ──► Runner ──► Target ──► output ──► Graders ──► results + summary + gate
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
**Status:** v1.0.7 is the current stable release. Includes a multi-turn tool-loop retrieval agent, session memory, process/Docker sandbox isolation, and an 11-grader CVE triage benchmark across four models. Zero runtime dependencies; deterministic example runs offline. Not published to PyPI; install from a clone or a release wheel.
|
|
12
|
+
|
|
13
|
+
## The main path in three commands
|
|
14
|
+
|
|
15
|
+
### 1. Run an evaluation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
python3.12 -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
19
|
+
.venv/bin/eval_uate_me init my-eval && cd my-eval
|
|
20
|
+
../.venv/bin/eval_uate_me check eval.toml
|
|
21
|
+
../.venv/bin/eval_uate_me run eval.toml
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
`init` creates a runnable starter project (config, cases, provenance file). Point the `[target]` at your agent: use the built-in `http` or `python` type for a service or a function, or write a plugin class for anything stateful. `check` validates config and data before anything runs.
|
|
25
|
+
|
|
26
|
+
### 2. Gate regressions in CI
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
eval_uate_me run eval.toml
|
|
30
|
+
eval_uate_me baseline promote --config eval.toml # writes eval.baseline.json — commit it
|
|
31
|
+
eval_uate_me gate eval.toml # exit 0 pass, 1 regression, 2 cannot decide
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The gate compares the latest run with the baseline. Add rules to the config once you have a baseline:
|
|
35
|
+
|
|
36
|
+
```toml
|
|
37
|
+
[gate]
|
|
38
|
+
significant_only = true # fail only when the drop is statistically significant
|
|
39
|
+
[gate.min]
|
|
40
|
+
pass_rate = 0.90
|
|
41
|
+
[gate.max_drop]
|
|
42
|
+
pass_rate = 0.05
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
A metric the rules name but the run lacks is a failure (a typo must not silently pass). A different dataset or split is refused rather than compared. GitHub Actions gets `::error::` annotations automatically.
|
|
46
|
+
|
|
47
|
+
### 3. Test whether the evaluation catches a broken agent (`meta-eval`)
|
|
48
|
+
|
|
49
|
+
```toml
|
|
50
|
+
[[mutants]]
|
|
51
|
+
name = "drops-the-answer"
|
|
52
|
+
[mutants.target]
|
|
53
|
+
type = "my_agent"
|
|
54
|
+
flaw = "omit"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
eval_uate_me meta-eval eval.toml # killed / survived / no-op per mutant
|
|
59
|
+
eval_uate_me meta-eval eval.toml --auto # damage every field of the real outputs; name blind spots
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
**killed** = the evaluation scored this mutant significantly lower. **survived** = the evaluation cannot tell it from the real agent — a blind spot to fix with a better grader or more cases. `--auto` finds the blind spots you did not think to write.
|
|
63
|
+
|
|
64
|
+
`examples/cve_triage/` is the primary worked example: a retrieval-loop agent over 100 public CISA KEV vulnerabilities,
|
|
65
|
+
420 grouped cases, **11 graders** (schema, identity, CVSS, CWE, KEV flag, affected product, references, fabrication,
|
|
66
|
+
tool-call assertion, prompt-injection resistance, safety/disclaimer), and deliberate mutants for each failure mode.
|
|
67
|
+
Multi-model benchmark on the **dev split** (25 cases each): deepseek-v4.1-flash 96.7%, gpt-4o-mini 91.6%, claude-haiku-5.5 91.6% (v1.0.5 run). Held-out multi-model results not yet published.
|
|
68
|
+
|
|
69
|
+
`examples/ticket_triage/` is a second worked example: 50 customer-support tickets across 5 categories and 3 urgency
|
|
70
|
+
levels, **3 graders** (schema, category, urgency), and an LLM target (`eval.llm.toml`) for benchmarking real models
|
|
71
|
+
alongside the deterministic oracle. Demonstrates the harness on a pure-classification task with no tool use.
|
|
72
|
+
|
|
73
|
+
`examples/support_reply/` is a third worked example: 30 customer-support tickets where the agent must **draft a reply**
|
|
74
|
+
rather than classify. No single correct answer exists — quality is judged by an LLM judge on three dimensions
|
|
75
|
+
(addresses issue, actionable, appropriate tone). Demonstrates the harness on open-ended generation, the third evaluation
|
|
76
|
+
paradigm alongside factual lookup and classification.
|
|
77
|
+
|
|
78
|
+
## Platforms
|
|
79
|
+
|
|
80
|
+
CI runs on **Linux** (Python 3.11 and 3.12), **Windows** (3.11 and 3.12) and **macOS** (3.12). Docker sandbox tests run on Linux. On Windows, `cpu_limit_s` and `memory_limit_mb` are not enforced in `process` isolation; use `docker` for hard limits.
|
|
81
|
+
|
|
82
|
+
## Documentation
|
|
83
|
+
|
|
84
|
+
| File | Contents |
|
|
85
|
+
|---|---|
|
|
86
|
+
| [docs/TUTORIAL.md](docs/TUTORIAL.md) | step-by-step walkthrough; every command is run by the tests |
|
|
87
|
+
| [docs/REFERENCE.md](docs/REFERENCE.md) | every command, option and config key |
|
|
88
|
+
| [docs/API.md](docs/API.md) | writing targets, graders and judge backends (plugin API) |
|
|
89
|
+
| [docs/VERSIONING.md](docs/VERSIONING.md) | what is stable from 1.0 and what is not |
|
|
90
|
+
| [docs/DESIGN.md](docs/DESIGN.md) | trade-offs and known limits |
|
|
91
|
+
| [docs/ROADMAP.md](docs/ROADMAP.md) | what shipped, what is pending |
|
|
92
|
+
| [docs/RELEASING.md](docs/RELEASING.md) | how to cut a release |
|
|
93
|
+
|
|
94
|
+
Licensed under the [MIT licence](LICENSE).
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "eval_uate_me"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "A small, plugin-based eval harness"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {file = "LICENSE"}
|
|
11
|
+
keywords = ["evaluation", "llm", "agents", "eval-harness"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"License :: OSI Approved :: MIT License",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Typing :: Typed",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Topic :: Software Development :: Testing",
|
|
20
|
+
]
|
|
21
|
+
requires-python = ">=3.11"
|
|
22
|
+
dependencies = []
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/Chuanhui-Zheng/Eval-uate-Me"
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
dev = ["pytest>=8", "pytest-cov>=5", "mypy>=1.10"]
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
eval_uate_me = "eval_uate_me.cli:main"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.package-data]
|
|
37
|
+
eval_uate_me = ["py.typed"] # PEP 561: downstream plugins get this package's types
|
|
38
|
+
|
|
39
|
+
[tool.pytest.ini_options]
|
|
40
|
+
testpaths = ["tests", "examples/cve_triage/tests", "examples/ticket_triage/tests", "examples/support_reply/tests"]
|
|
41
|
+
pythonpath = ["examples/cve_triage", "examples/ticket_triage", "examples/support_reply"]
|
|
42
|
+
|
|
43
|
+
[tool.mypy]
|
|
44
|
+
python_version = "3.11"
|
|
45
|
+
files = ["src/eval_uate_me"]
|
|
46
|
+
warn_unused_ignores = true
|
|
47
|
+
warn_redundant_casts = true
|
|
48
|
+
check_untyped_defs = true
|
|
49
|
+
|
|
50
|
+
[tool.coverage.run]
|
|
51
|
+
source = ["eval_uate_me"]
|
|
52
|
+
omit = ["*/__main__.py"]
|
|
53
|
+
|
|
54
|
+
[tool.coverage.report]
|
|
55
|
+
# worker.py runs in a subprocess and is not measured; the Docker paths of isolation.py run only in the CI docker job
|
|
56
|
+
fail_under = 93
|
|
57
|
+
show_missing = true
|
|
58
|
+
skip_covered = true
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
__version__ = "1.0.0"
|
|
2
|
+
|
|
3
|
+
from .models import Case, CaseResult, Score, TargetOutput, TraceStep
|
|
4
|
+
from .config import ConfigError, PluginError
|
|
5
|
+
from .registry import API_VERSION, register_grader, register_judge, register_target
|
|
6
|
+
from . import adapters # noqa: F401 (registers the http and python targets)
|
|
7
|
+
from .graders import Grader
|
|
8
|
+
from .judges import Judge
|
|
9
|
+
from .api import EvalResult, evaluate, run_eval
|
|
10
|
+
from .targets import Target
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"Case", "CaseResult", "Score", "TargetOutput",
|
|
14
|
+
"API_VERSION", "ConfigError", "EvalResult", "PluginError", "Grader", "Judge", "Target", "TraceStep", "evaluate", "register_grader", "register_judge", "register_target", "run_eval",
|
|
15
|
+
]
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Built-in targets for systems you already have: an HTTP endpoint, or a Python function.
|
|
2
|
+
|
|
3
|
+
[target] [target]
|
|
4
|
+
type = "http" type = "python"
|
|
5
|
+
url = "http://localhost:8000/answer" callable = "my_app.agent:answer" # module:function
|
|
6
|
+
headers = {Authorization = "Bearer ${ENV:MY_TOKEN}"} # ${ENV:NAME} is read from the environment, never stored
|
|
7
|
+
|
|
8
|
+
`http` POSTs {"id", "input", "metadata"} as JSON and reads {"output": ..., optional "trace", "tool_calls", "usage",
|
|
9
|
+
"cost_usd", "error"}. `python` calls function(input) (or function(case) with with_case = true); it may return the output
|
|
10
|
+
itself or a TargetOutput, and may be `async def` (then the harness keeps many calls in flight). Neither needs a plugin class.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import importlib
|
|
15
|
+
import inspect
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
import urllib.error
|
|
21
|
+
import urllib.request
|
|
22
|
+
from typing import Any
|
|
23
|
+
from urllib.parse import urlparse
|
|
24
|
+
|
|
25
|
+
from .config import ConfigError
|
|
26
|
+
from .models import Case, TargetOutput
|
|
27
|
+
from .registry import register_target
|
|
28
|
+
from .targets import Target
|
|
29
|
+
|
|
30
|
+
_ENV = re.compile(r"\$\{ENV:([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def expand_env(text: str, where: str) -> str:
|
|
34
|
+
"""Replace ${ENV:NAME} with the environment variable, failing clearly if it is not set."""
|
|
35
|
+
def sub(m: re.Match[str]) -> str:
|
|
36
|
+
name = m.group(1)
|
|
37
|
+
if name not in os.environ:
|
|
38
|
+
raise ConfigError(f"{where}: environment variable {name} is not set")
|
|
39
|
+
return os.environ[name]
|
|
40
|
+
return _ENV.sub(sub, text)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _dig(data: Any, path: str) -> Any:
|
|
44
|
+
for part in path.split("."):
|
|
45
|
+
if isinstance(data, dict) and part in data:
|
|
46
|
+
data = data[part]
|
|
47
|
+
else:
|
|
48
|
+
raise KeyError(path)
|
|
49
|
+
return data
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@register_target("http")
|
|
53
|
+
class HttpTarget(Target):
|
|
54
|
+
"""POST each case to a URL. Transport errors, non-2xx statuses and malformed bodies become error results for that case
|
|
55
|
+
(the run continues; use run.retries to retry them)."""
|
|
56
|
+
eval_uate_me_api = 1
|
|
57
|
+
network = True
|
|
58
|
+
|
|
59
|
+
def __init__(self, url: str, headers: dict[str, str] | None = None, timeout_s: float = 30.0,
|
|
60
|
+
output_field: str = "output", send_metadata: bool = True):
|
|
61
|
+
if urlparse(url).scheme not in ("http", "https"):
|
|
62
|
+
raise ConfigError(f"[target] url must start with http:// or https://, got {url!r}")
|
|
63
|
+
self.url, self.timeout_s, self.output_field, self.send_metadata = url, timeout_s, output_field, send_metadata
|
|
64
|
+
self.headers = {"Content-Type": "application/json", "Accept": "application/json",
|
|
65
|
+
**{k: expand_env(str(v), f"[target] headers.{k}") for k, v in (headers or {}).items()}}
|
|
66
|
+
|
|
67
|
+
def run(self, case: Case) -> TargetOutput:
|
|
68
|
+
payload = {"id": case.id, "input": case.input, **({"metadata": case.metadata} if self.send_metadata else {})}
|
|
69
|
+
req = urllib.request.Request(self.url, data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
|
70
|
+
headers=self.headers, method="POST")
|
|
71
|
+
try:
|
|
72
|
+
with urllib.request.urlopen(req, timeout=self.timeout_s) as resp:
|
|
73
|
+
raw = resp.read().decode("utf-8", errors="replace")
|
|
74
|
+
except urllib.error.HTTPError as e:
|
|
75
|
+
return TargetOutput(error=f"HTTP {e.code}: {e.read()[:200].decode('utf-8', errors='replace')}")
|
|
76
|
+
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
|
77
|
+
return TargetOutput(error=f"http error: {getattr(e, 'reason', e)}")
|
|
78
|
+
try:
|
|
79
|
+
body = json.loads(raw)
|
|
80
|
+
except ValueError as e:
|
|
81
|
+
return TargetOutput(error=f"invalid JSON response: {e}")
|
|
82
|
+
if not isinstance(body, dict):
|
|
83
|
+
return TargetOutput(error="response JSON must be an object with an 'output' field")
|
|
84
|
+
if body.get("error"):
|
|
85
|
+
return TargetOutput(error=str(body["error"]))
|
|
86
|
+
try:
|
|
87
|
+
output = _dig(body, self.output_field)
|
|
88
|
+
except KeyError:
|
|
89
|
+
return TargetOutput(error=f"response JSON has no '{self.output_field}' field")
|
|
90
|
+
trace = list(body.get("trace", [])) + [{"tool": str(t)} for t in body.get("tool_calls", [])]
|
|
91
|
+
usage = {k: float(v) for k, v in (body.get("usage") or {}).items() if isinstance(v, (int, float)) and not isinstance(v, bool)}
|
|
92
|
+
if isinstance(body.get("cost_usd"), (int, float)):
|
|
93
|
+
usage["cost_usd"] = float(body["cost_usd"])
|
|
94
|
+
return TargetOutput(output=output, trace=trace, usage=usage)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _to_output(value: Any) -> TargetOutput:
|
|
98
|
+
return value if isinstance(value, TargetOutput) else TargetOutput(output=value)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@register_target("python")
|
|
102
|
+
class PythonTarget(Target):
|
|
103
|
+
"""Call a Python function. `callable = "package.module:function"`; add `paths` for folders to import from."""
|
|
104
|
+
eval_uate_me_api = 1
|
|
105
|
+
|
|
106
|
+
def __init__(self, callable: str, with_case: bool = False, paths: list[str] | None = None,
|
|
107
|
+
network: bool | None = None):
|
|
108
|
+
mod, _, attr = callable.partition(":")
|
|
109
|
+
if not mod or not attr:
|
|
110
|
+
raise ConfigError(f"[target] callable must look like 'package.module:function', got {callable!r}")
|
|
111
|
+
for p in paths or []:
|
|
112
|
+
if p not in sys.path:
|
|
113
|
+
sys.path.insert(0, p)
|
|
114
|
+
try:
|
|
115
|
+
obj: Any = importlib.import_module(mod)
|
|
116
|
+
for part in attr.split("."):
|
|
117
|
+
obj = getattr(obj, part)
|
|
118
|
+
except (ImportError, AttributeError) as e:
|
|
119
|
+
raise ConfigError(f"[target] cannot load '{callable}': {type(e).__name__}: {e}"
|
|
120
|
+
+ ("" if paths else " (set `paths = [...]` if the module is not importable from here)")) from e
|
|
121
|
+
if not callable_(obj):
|
|
122
|
+
raise ConfigError(f"[target] '{callable}' is not callable")
|
|
123
|
+
self.fn, self.with_case, self.network = obj, with_case, network
|
|
124
|
+
if inspect.iscoroutinefunction(obj):
|
|
125
|
+
self.arun = self._arun # makes the harness use its async executor
|
|
126
|
+
|
|
127
|
+
def _arg(self, case: Case) -> Any:
|
|
128
|
+
return case if self.with_case else case.input
|
|
129
|
+
|
|
130
|
+
def run(self, case: Case) -> TargetOutput:
|
|
131
|
+
if inspect.iscoroutinefunction(self.fn):
|
|
132
|
+
import asyncio
|
|
133
|
+
return asyncio.run(self._arun(case))
|
|
134
|
+
return _to_output(self.fn(self._arg(case)))
|
|
135
|
+
|
|
136
|
+
async def _arun(self, case: Case) -> TargetOutput:
|
|
137
|
+
return _to_output(await self.fn(self._arg(case)))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def callable_(obj: Any) -> bool: # builtin name `callable` is shadowed by the option name above
|
|
141
|
+
return hasattr(obj, "__call__")
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""Run evals from Python (a notebook, a script, a test) without a config file.
|
|
2
|
+
|
|
3
|
+
from eval_uate_me import evaluate
|
|
4
|
+
|
|
5
|
+
result = evaluate(my_function, cases=[{"input": "2+2", "expected": "4"}, ("3+3", "6")])
|
|
6
|
+
result.pass_rate # 1.0
|
|
7
|
+
result.failures() # the CaseResults that failed
|
|
8
|
+
result.to_rows() # flat dicts, ready for a table or a DataFrame
|
|
9
|
+
|
|
10
|
+
`evaluate` works in memory (nothing is written to disk). `run_eval("eval.toml")` runs a config exactly as the command line does
|
|
11
|
+
and returns the same kind of result, with the run directory. Both are part of the stable API.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import inspect
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Callable, Iterable
|
|
19
|
+
|
|
20
|
+
from . import registry
|
|
21
|
+
from .adapters import PythonTarget
|
|
22
|
+
from .config import Config, ConfigError, load_config
|
|
23
|
+
from .execution import ExecPolicy, is_async, run_async, run_threaded
|
|
24
|
+
from .graders import ExactMatch, Grader
|
|
25
|
+
from .models import Case, CaseResult, FORMAT_VERSION, Score, TargetOutput
|
|
26
|
+
from .runner import _grade, _run_case, _run_case_async, run, summarize
|
|
27
|
+
from .targets import Target
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class EvalResult:
|
|
32
|
+
"""What an evaluation produced. `run_dir` is None for in-memory runs."""
|
|
33
|
+
results: list[CaseResult]
|
|
34
|
+
summary: dict[str, Any]
|
|
35
|
+
run_dir: Path | None = None
|
|
36
|
+
_rows: list[dict[str, Any]] = field(default_factory=list, repr=False)
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def pass_rate(self) -> float:
|
|
40
|
+
return float(self.summary["pass_rate"])
|
|
41
|
+
|
|
42
|
+
def failures(self) -> list[CaseResult]:
|
|
43
|
+
return [r for r in self.results if not r.passed]
|
|
44
|
+
|
|
45
|
+
def to_rows(self) -> list[dict[str, Any]]:
|
|
46
|
+
"""One flat dict per attempt: case_id, attempt, passed, error, latency_s, input, expected, output and one
|
|
47
|
+
`score:<grader>` column per grader."""
|
|
48
|
+
rows = []
|
|
49
|
+
for r in self.results:
|
|
50
|
+
row: dict[str, Any] = {"case_id": r.case.id, "attempt": r.attempt, "passed": r.passed, "error": r.result.error,
|
|
51
|
+
"latency_s": r.result.latency_s, "input": r.case.input, "expected": r.case.expected,
|
|
52
|
+
"output": r.result.output}
|
|
53
|
+
row.update({f"score:{s.grader}": s.value for s in r.scores})
|
|
54
|
+
rows.append(row)
|
|
55
|
+
return rows
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class _FunctionTarget(Target):
|
|
59
|
+
eval_uate_me_api = 1
|
|
60
|
+
|
|
61
|
+
def __init__(self, fn: Callable[..., Any]):
|
|
62
|
+
self.fn = fn
|
|
63
|
+
if inspect.iscoroutinefunction(fn):
|
|
64
|
+
self.arun = self._arun
|
|
65
|
+
|
|
66
|
+
def run(self, case: Case) -> TargetOutput:
|
|
67
|
+
if inspect.iscoroutinefunction(self.fn):
|
|
68
|
+
import asyncio
|
|
69
|
+
return asyncio.run(self._arun(case))
|
|
70
|
+
v = self.fn(case.input)
|
|
71
|
+
return v if isinstance(v, TargetOutput) else TargetOutput(output=v)
|
|
72
|
+
|
|
73
|
+
async def _arun(self, case: Case) -> TargetOutput:
|
|
74
|
+
v = await self.fn(case.input)
|
|
75
|
+
return v if isinstance(v, TargetOutput) else TargetOutput(output=v)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class _FunctionGrader(Grader):
|
|
79
|
+
eval_uate_me_api = 1
|
|
80
|
+
|
|
81
|
+
def __init__(self, fn: Callable[[Case, TargetOutput], Any]):
|
|
82
|
+
self.fn, self.name = fn, getattr(fn, "__name__", "function_grader")
|
|
83
|
+
|
|
84
|
+
def grade(self, case: Case, out: TargetOutput) -> Score:
|
|
85
|
+
v = self.fn(case, out)
|
|
86
|
+
if isinstance(v, Score):
|
|
87
|
+
return v
|
|
88
|
+
if isinstance(v, bool):
|
|
89
|
+
return Score(self.name, 1.0 if v else 0.0, v, "" if v else "returned False")
|
|
90
|
+
raise TypeError(f"grader function '{self.name}' must return a bool or a Score, got {type(v).__name__}")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _target(t: Any) -> Target:
|
|
94
|
+
if isinstance(t, Target):
|
|
95
|
+
return t
|
|
96
|
+
if isinstance(t, str):
|
|
97
|
+
return PythonTarget(callable=t)
|
|
98
|
+
if callable(t):
|
|
99
|
+
return _FunctionTarget(t)
|
|
100
|
+
raise TypeError("target must be a Target, a function, or a 'module:function' string")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _case(c: Any, n: int) -> Case:
|
|
104
|
+
if isinstance(c, Case):
|
|
105
|
+
return c
|
|
106
|
+
if isinstance(c, dict):
|
|
107
|
+
return Case(id=str(c.get("id", f"case-{n}")), input=c.get("input"), expected=c.get("expected"), metadata=dict(c.get("metadata", {})))
|
|
108
|
+
if isinstance(c, (tuple, list)) and len(c) in (1, 2):
|
|
109
|
+
return Case(id=f"case-{n}", input=c[0], expected=c[1] if len(c) == 2 else None)
|
|
110
|
+
raise TypeError("a case must be a Case, a dict, or an (input, expected) tuple")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _grader(g: Any) -> Grader:
|
|
114
|
+
if isinstance(g, Grader):
|
|
115
|
+
return g
|
|
116
|
+
if isinstance(g, str):
|
|
117
|
+
return registry.get_grader(g)
|
|
118
|
+
if isinstance(g, tuple) and len(g) == 2 and isinstance(g[0], str) and isinstance(g[1], dict):
|
|
119
|
+
return registry.get_grader(g[0], **g[1])
|
|
120
|
+
if callable(g):
|
|
121
|
+
return _FunctionGrader(g)
|
|
122
|
+
raise TypeError("a grader must be a Grader, a registered name, a (name, options) tuple, or a function(case, output) -> bool")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def evaluate(target: Any, cases: Iterable[Any], graders: Iterable[Any] | None = None, *, repeats: int = 1, concurrency: int = 4,
|
|
126
|
+
retries: int = 0, timeout_s: float | None = None, name: str = "eval") -> EvalResult:
|
|
127
|
+
"""Run `target` over `cases` and grade each answer, in memory.
|
|
128
|
+
|
|
129
|
+
target a Target, a function(input) -> output (sync or async), or "module:function"
|
|
130
|
+
cases Case objects, dicts {id?, input, expected?, metadata?}, or (input, expected) tuples
|
|
131
|
+
graders Grader objects, registered names ("exact_match"), (name, {options}) tuples, or function(case, output) -> bool;
|
|
132
|
+
default: exact_match
|
|
133
|
+
"""
|
|
134
|
+
tgt = _target(target)
|
|
135
|
+
case_list = [_case(c, n) for n, c in enumerate(cases, 1)]
|
|
136
|
+
if not case_list:
|
|
137
|
+
raise ConfigError("evaluate() needs at least one case")
|
|
138
|
+
ids = [c.id for c in case_list]
|
|
139
|
+
if len(set(ids)) != len(ids):
|
|
140
|
+
raise ConfigError(f"duplicate case ids: {sorted({i for i in ids if ids.count(i) > 1})}")
|
|
141
|
+
grader_list = [_grader(g) for g in (graders if graders is not None else [ExactMatch()])]
|
|
142
|
+
policy = ExecPolicy(retries=retries, retry_delay_s=0.0 if retries else 1.0, timeout_s=timeout_s)
|
|
143
|
+
jobs = [(c, a) for c in case_list for a in range(max(1, repeats))]
|
|
144
|
+
done: dict[tuple[str, int], CaseResult] = {}
|
|
145
|
+
|
|
146
|
+
def work(job: tuple[Case, int]) -> None:
|
|
147
|
+
r = _run_case(tgt, grader_list, job[0], job[1], policy)
|
|
148
|
+
done[(r.case.id, r.attempt)] = r
|
|
149
|
+
|
|
150
|
+
async def awork(job: tuple[Case, int]) -> None:
|
|
151
|
+
r = await _run_case_async(tgt, grader_list, job[0], job[1], policy)
|
|
152
|
+
done[(r.case.id, r.attempt)] = r
|
|
153
|
+
|
|
154
|
+
if is_async(tgt):
|
|
155
|
+
run_async(jobs, awork, concurrency)
|
|
156
|
+
else:
|
|
157
|
+
run_threaded(jobs, work, concurrency)
|
|
158
|
+
results = [done[(c.id, a)] for c, a in jobs]
|
|
159
|
+
summary = summarize(results, name, planned=len(jobs))
|
|
160
|
+
summary["cost_usd_spent"] = sum(r.result.usage.get("cost_usd", 0.0) for r in results)
|
|
161
|
+
summary["format_version"] = FORMAT_VERSION
|
|
162
|
+
return EvalResult(results, summary, None)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def run_eval(config: str | Path | Config, *, split: str | None = None, resume: str | Path | None = None,
|
|
166
|
+
retry_errors: bool = False, force: bool = False) -> EvalResult:
|
|
167
|
+
"""Run an eval from a config file exactly as `eval_uate_me run` does (the run directory is written) and return the result."""
|
|
168
|
+
cfg = config if isinstance(config, Config) else load_config(config)
|
|
169
|
+
results, summary, run_dir = run(cfg, resume=Path(resume) if resume else None, retry_errors=retry_errors, force=force, split=split)
|
|
170
|
+
return EvalResult(results, summary, run_dir)
|