runreckon 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runreckon-0.1.0/LICENSE +21 -0
- runreckon-0.1.0/PKG-INFO +219 -0
- runreckon-0.1.0/README.md +200 -0
- runreckon-0.1.0/pyproject.toml +34 -0
- runreckon-0.1.0/setup.cfg +4 -0
- runreckon-0.1.0/src/runreckon/__init__.py +3 -0
- runreckon-0.1.0/src/runreckon/cli.py +138 -0
- runreckon-0.1.0/src/runreckon/examples/clean-run/spec.json +1 -0
- runreckon-0.1.0/src/runreckon/examples/clean-run/trace.jsonl +8 -0
- runreckon-0.1.0/src/runreckon/examples/expected.json +1 -0
- runreckon-0.1.0/src/runreckon/examples/policy-breach/spec.json +1 -0
- runreckon-0.1.0/src/runreckon/examples/policy-breach/trace.jsonl +10 -0
- runreckon-0.1.0/src/runreckon/examples/sloppy-success/spec.json +1 -0
- runreckon-0.1.0/src/runreckon/examples/sloppy-success/trace.jsonl +20 -0
- runreckon-0.1.0/src/runreckon/inject.py +193 -0
- runreckon-0.1.0/src/runreckon/schema.py +109 -0
- runreckon-0.1.0/src/runreckon/scorecard.py +103 -0
- runreckon-0.1.0/src/runreckon/scorers.py +324 -0
- runreckon-0.1.0/src/runreckon/selftest.py +269 -0
- runreckon-0.1.0/src/runreckon/semantic.py +211 -0
- runreckon-0.1.0/src/runreckon/spec.py +94 -0
- runreckon-0.1.0/src/runreckon/trace.py +191 -0
- runreckon-0.1.0/src/runreckon.egg-info/PKG-INFO +219 -0
- runreckon-0.1.0/src/runreckon.egg-info/SOURCES.txt +31 -0
- runreckon-0.1.0/src/runreckon.egg-info/dependency_links.txt +1 -0
- runreckon-0.1.0/src/runreckon.egg-info/entry_points.txt +2 -0
- runreckon-0.1.0/src/runreckon.egg-info/top_level.txt +1 -0
- runreckon-0.1.0/tests/test_cli.py +90 -0
- runreckon-0.1.0/tests/test_inject.py +44 -0
- runreckon-0.1.0/tests/test_schema.py +60 -0
- runreckon-0.1.0/tests/test_scorers.py +195 -0
- runreckon-0.1.0/tests/test_semantic.py +88 -0
- runreckon-0.1.0/tests/test_trace.py +59 -0
runreckon-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ranadhir Ghosh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
runreckon-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: runreckon
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Score how an AI agent executed a run, not just whether it finished: deterministic scoring of recorded agent traces (tool selection, argument validity, step economy, error recovery, policy compliance, cost) with a fault-injection self-test that measures the scorer itself.
|
|
5
|
+
Author: Ranadhir Ghosh
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ranadhir19/runreckon
|
|
8
|
+
Keywords: agent-evaluation,llm-evals,ai-agents,agent-traces,evaluation,mcp,observability,ci
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# runreckon
|
|
21
|
+
|
|
22
|
+
**Score how an agent executed a run, not just whether it finished.**
|
|
23
|
+
|
|
24
|
+
An agent can complete a task and still do it badly: call the same search
|
|
25
|
+
four times, reach for tools it was never given, pass malformed arguments,
|
|
26
|
+
blow the token budget, quietly break a policy on the way — and deliver a
|
|
27
|
+
final answer that passes every outcome check. Outcome-only evaluation
|
|
28
|
+
calls that run a success. runreckon scores the *execution*: feed it a
|
|
29
|
+
recorded run (a JSONL trace) and the task spec it was supposed to
|
|
30
|
+
follow, and it produces a scorecard for the run itself.
|
|
31
|
+
|
|
32
|
+
## The showcase case
|
|
33
|
+
|
|
34
|
+
This run **succeeded**. Its brief was saved, its final checks pass, its
|
|
35
|
+
answer is fully grounded. Here is its scorecard (`runreckon demo`, the
|
|
36
|
+
`sloppy-success` example, real output):
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
outcome 100.0
|
|
40
|
+
tool_selection 88.9
|
|
41
|
+
- step 5: tool 'web_scrape' is not in the spec's permitted set
|
|
42
|
+
- step 6: tool 'summarize' is not in the spec's permitted set
|
|
43
|
+
argument_validity 85.7
|
|
44
|
+
- step 7 (fetch_page): $: missing required property 'url'
|
|
45
|
+
step_economy 66.7
|
|
46
|
+
- step 2: 'web_search' repeated with identical arguments after already succeeding — wasted step
|
|
47
|
+
- step 3: 'web_search' repeated with identical arguments after already succeeding — wasted step
|
|
48
|
+
- step 4: 'web_search' repeated with identical arguments after already succeeding — wasted step
|
|
49
|
+
- 'web_search' called 4 times with identical arguments — loop pattern
|
|
50
|
+
error_recovery 100.0
|
|
51
|
+
policy 100.0
|
|
52
|
+
cost_latency 43.8
|
|
53
|
+
- budget exceeded: max_steps 9 > 8
|
|
54
|
+
- budget exceeded: max_tokens 9370 > 6000
|
|
55
|
+
groundedness 100.0
|
|
56
|
+
------------------------------------------------------------
|
|
57
|
+
COMPOSITE 87.6 — serviceable
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The task succeeded; the run was still bad — four identical searches, two
|
|
61
|
+
undeclared tools, one malformed call, two budgets blown. With
|
|
62
|
+
`--min-score 0.90` this run fails CI. That gap between "it worked" and
|
|
63
|
+
"it ran well" is what runreckon measures.
|
|
64
|
+
|
|
65
|
+
## Quickstart
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install git+https://github.com/ranadhir19/runreckon.git
|
|
69
|
+
runreckon demo # the three bundled examples
|
|
70
|
+
runreckon score my-run.jsonl --spec my-task.json # your own run
|
|
71
|
+
runreckon score my-run.jsonl --spec my-task.json --format json
|
|
72
|
+
runreckon selftest # measure the scorer itself
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Pure Python standard library. No dependencies, no API keys, no network:
|
|
76
|
+
the deterministic core runs fully offline. Developed on Python 3.12;
|
|
77
|
+
any Python ≥ 3.10 works.
|
|
78
|
+
|
|
79
|
+
Exit codes are CI-shaped: `0` pass, `1` fail (composite below
|
|
80
|
+
`--min-score`, default 0.75, or any policy violation), `2` the trace or
|
|
81
|
+
spec could not be read.
|
|
82
|
+
|
|
83
|
+
## What it scores
|
|
84
|
+
|
|
85
|
+
| Dimension | What it checks | How |
|
|
86
|
+
|---|---|---|
|
|
87
|
+
| outcome | Did the run deliver? Final-state checks, required answer content | exact |
|
|
88
|
+
| tool_selection | The tools the spec expected were used; no undeclared tools were touched | exact |
|
|
89
|
+
| argument_validity | Every call's arguments against the spec's JSON Schemas | exact |
|
|
90
|
+
| step_economy | Repeated calls that already succeeded, loop patterns, calls after the final event | exact |
|
|
91
|
+
| error_recovery | Failures that were recovered vs failures silently stepped over | exact |
|
|
92
|
+
| policy | Forbidden tools, forbidden argument patterns, required ordering (approval before action), call limits, forbidden end-states | exact — and a hard gate |
|
|
93
|
+
| cost_latency | Steps, tokens, recorded latency, and priced cost against the spec's budgets | exact |
|
|
94
|
+
| groundedness | Every checkable claim in the final answer (numbers, URLs, emails, quoted strings) appears in the evidence the agent actually had | exact |
|
|
95
|
+
|
|
96
|
+
Two design rules:
|
|
97
|
+
|
|
98
|
+
- **Deterministic first.** Everything that can be computed exactly is
|
|
99
|
+
computed exactly. A repeated call after an *error* is a retry, not
|
|
100
|
+
waste; a repeat with *different arguments* is new work; a number the
|
|
101
|
+
*user* put in the task prompt is grounded, not invented. The scorers
|
|
102
|
+
encode those distinctions, and the self-test's negative controls
|
|
103
|
+
prove they hold.
|
|
104
|
+
- **Judges are optional and measured.** Groundedness can also be
|
|
105
|
+
assessed by a pluggable judge (`--judge module:function`, or the
|
|
106
|
+
bundled deterministic stand-in). A judge is only trusted as far as it
|
|
107
|
+
has been measured: whenever one is enabled, its agreement and Cohen's
|
|
108
|
+
kappa on a bundled labelled calibration set are printed on the
|
|
109
|
+
scorecard next to its verdict — reported alongside the deterministic
|
|
110
|
+
score, never blended into it. The stand-in's measured calibration:
|
|
111
|
+
agreement 0.833, kappa 0.667 on 12 items, with two instructive
|
|
112
|
+
failures (a heavy-paraphrase miss, a negation false-pass). That is
|
|
113
|
+
the point: an unmeasured judge is a hope, a measured one is an
|
|
114
|
+
instrument with known error.
|
|
115
|
+
|
|
116
|
+
This calibration is a disclosure about runreckon's own optional
|
|
117
|
+
judge, printed so its verdicts are never taken on faith. Systematic
|
|
118
|
+
grader validation — measuring any grader against labelled data and
|
|
119
|
+
auditing the task suites it scores — is evalcrucible's job, not
|
|
120
|
+
runreckon's.
|
|
121
|
+
|
|
122
|
+
## The evaluator evaluates itself
|
|
123
|
+
|
|
124
|
+
runreckon ships a fault-injection self-test. It takes clean traces that
|
|
125
|
+
score perfectly, plants one known defect per copy — a wrong tool, an
|
|
126
|
+
invalid argument, an ungrounded claim, a silent policy breach, loop
|
|
127
|
+
waste, a masked failure — and checks that the scorer responsible for
|
|
128
|
+
that defect class catches it. It also runs negative controls:
|
|
129
|
+
legitimate traces that resemble defects and must not be flagged.
|
|
130
|
+
|
|
131
|
+
Measured on this tree (3 clean bases × 6 defect classes, regenerated
|
|
132
|
+
and byte-verified by `validate.py`; full detail in
|
|
133
|
+
[SELFTEST_RESULTS.md](SELFTEST_RESULTS.md)):
|
|
134
|
+
|
|
135
|
+
| Defect class | Scorer responsible | Planted | Caught | Rate |
|
|
136
|
+
|---|---|---|---|---|
|
|
137
|
+
| wrong_tool | tool_selection | 3 | 3 | 1.000 |
|
|
138
|
+
| invalid_args | argument_validity | 3 | 3 | 1.000 |
|
|
139
|
+
| ungrounded_claim | groundedness | 3 | 3 | 1.000 |
|
|
140
|
+
| policy_breach | policy | 3 | 3 | 1.000 |
|
|
141
|
+
| loop_waste | step_economy | 3 | 3 | 1.000 |
|
|
142
|
+
| silent_failure | error_recovery | 3 | 3 | 1.000 |
|
|
143
|
+
|
|
144
|
+
**18/18 planted defects caught; 3/3 negative controls pass.** The base
|
|
145
|
+
set is small and author-built — treat the rates as a regression floor
|
|
146
|
+
for the scorers, not as a field measurement. Widen the bases before
|
|
147
|
+
quoting the rates as evidence about anyone else's agents.
|
|
148
|
+
|
|
149
|
+
The self-test also caught a real bug during development: an email
|
|
150
|
+
address at the end of a sentence ("…sent to ana@example.com.") was
|
|
151
|
+
extracted with its sentence-final period attached and falsely flagged
|
|
152
|
+
as ungrounded. The fix is in, and a regression test pins it.
|
|
153
|
+
|
|
154
|
+
## Inputs
|
|
155
|
+
|
|
156
|
+
A **trace** is JSONL — one event per line: `run`, `call`, `result`,
|
|
157
|
+
`note`, `final`. A **spec** is JSON: expected/allowed tools, argument
|
|
158
|
+
schemas, a reference path, policy rules, final-state checks, budgets.
|
|
159
|
+
Full formats: [docs/trace-format.md](docs/trace-format.md) and
|
|
160
|
+
[docs/spec-format.md](docs/spec-format.md). The three bundled examples
|
|
161
|
+
(a clean run, the showcase above, and a policy breach) live in the
|
|
162
|
+
package under `runreckon/examples/` and run with `runreckon demo`.
|
|
163
|
+
|
|
164
|
+
## Gate a pull request on run quality
|
|
165
|
+
|
|
166
|
+
This repo is also a GitHub Action. Record a trace in your agent's test
|
|
167
|
+
suite, commit the trace and its spec, and fail the PR when execution
|
|
168
|
+
quality regresses:
|
|
169
|
+
|
|
170
|
+
```yaml
|
|
171
|
+
- uses: ranadhir19/runreckon@v0.1.0
|
|
172
|
+
with:
|
|
173
|
+
trace: tests/traces/checkout-run.jsonl
|
|
174
|
+
spec: tests/traces/checkout-task.json
|
|
175
|
+
min-score: "0.90"
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
See [docs/action.md](docs/action.md) for inputs and a fuller workflow.
|
|
179
|
+
|
|
180
|
+
## How this differs from evalcrucible
|
|
181
|
+
|
|
182
|
+
[evalcrucible](https://github.com/ranadhir19/evalcrucible) is the
|
|
183
|
+
grader tool: it validates graders against labelled data and audits
|
|
184
|
+
static (prompt → gold) task suites, so the benchmark itself can be
|
|
185
|
+
trusted. runreckon is the eval tool: given a task spec you already
|
|
186
|
+
trust, it scores how an agent executed a recorded run.
|
|
187
|
+
|
|
188
|
+
## Limits
|
|
189
|
+
|
|
190
|
+
- Scoring is only as good as the spec. A spec with no schemas, no
|
|
191
|
+
policy, and no budgets scores only what it declares; dimensions
|
|
192
|
+
without inputs report `n/a` and are excluded from the composite,
|
|
193
|
+
never silently zeroed or gifted.
|
|
194
|
+
- Groundedness checks *checkable* claims — numbers, URLs, emails,
|
|
195
|
+
quoted strings. A fluent falsehood with no checkable token in it is
|
|
196
|
+
invisible to the deterministic layer; that is what the (measured)
|
|
197
|
+
judge layer is for.
|
|
198
|
+
- Latency is the sum of per-event recorded latencies, not wall-clock;
|
|
199
|
+
traces that do not record tokens or latency cannot be scored on them.
|
|
200
|
+
- The fault-injection bases are three small author-built runs. The
|
|
201
|
+
mechanism generalises; the specific rates are a floor, not a census.
|
|
202
|
+
|
|
203
|
+
## Development
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
git clone https://github.com/ranadhir19/runreckon.git
|
|
207
|
+
cd runreckon
|
|
208
|
+
python3 validate.py # must end with OVERALL PASS
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
The harness compiles the package, runs the unit tests (authored from
|
|
212
|
+
the failure-mode domain, including the near-miss negatives), reproduces
|
|
213
|
+
all three example scorecards against their expected bands, re-runs the
|
|
214
|
+
self-test and requires `SELFTEST_RESULTS.md` to come back
|
|
215
|
+
byte-identical, and runs the portfolio's pre-push integrity gates.
|
|
216
|
+
|
|
217
|
+
## License
|
|
218
|
+
|
|
219
|
+
MIT — see [LICENSE](LICENSE). Copyright (c) 2026 Ranadhir Ghosh.
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# runreckon
|
|
2
|
+
|
|
3
|
+
**Score how an agent executed a run, not just whether it finished.**
|
|
4
|
+
|
|
5
|
+
An agent can complete a task and still do it badly: call the same search
|
|
6
|
+
four times, reach for tools it was never given, pass malformed arguments,
|
|
7
|
+
blow the token budget, quietly break a policy on the way — and deliver a
|
|
8
|
+
final answer that passes every outcome check. Outcome-only evaluation
|
|
9
|
+
calls that run a success. runreckon scores the *execution*: feed it a
|
|
10
|
+
recorded run (a JSONL trace) and the task spec it was supposed to
|
|
11
|
+
follow, and it produces a scorecard for the run itself.
|
|
12
|
+
|
|
13
|
+
## The showcase case
|
|
14
|
+
|
|
15
|
+
This run **succeeded**. Its brief was saved, its final checks pass, its
|
|
16
|
+
answer is fully grounded. Here is its scorecard (`runreckon demo`, the
|
|
17
|
+
`sloppy-success` example, real output):
|
|
18
|
+
|
|
19
|
+
```text
|
|
20
|
+
outcome 100.0
|
|
21
|
+
tool_selection 88.9
|
|
22
|
+
- step 5: tool 'web_scrape' is not in the spec's permitted set
|
|
23
|
+
- step 6: tool 'summarize' is not in the spec's permitted set
|
|
24
|
+
argument_validity 85.7
|
|
25
|
+
- step 7 (fetch_page): $: missing required property 'url'
|
|
26
|
+
step_economy 66.7
|
|
27
|
+
- step 2: 'web_search' repeated with identical arguments after already succeeding — wasted step
|
|
28
|
+
- step 3: 'web_search' repeated with identical arguments after already succeeding — wasted step
|
|
29
|
+
- step 4: 'web_search' repeated with identical arguments after already succeeding — wasted step
|
|
30
|
+
- 'web_search' called 4 times with identical arguments — loop pattern
|
|
31
|
+
error_recovery 100.0
|
|
32
|
+
policy 100.0
|
|
33
|
+
cost_latency 43.8
|
|
34
|
+
- budget exceeded: max_steps 9 > 8
|
|
35
|
+
- budget exceeded: max_tokens 9370 > 6000
|
|
36
|
+
groundedness 100.0
|
|
37
|
+
------------------------------------------------------------
|
|
38
|
+
COMPOSITE 87.6 — serviceable
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The task succeeded; the run was still bad — four identical searches, two
|
|
42
|
+
undeclared tools, one malformed call, two budgets blown. With
|
|
43
|
+
`--min-score 0.90` this run fails CI. That gap between "it worked" and
|
|
44
|
+
"it ran well" is what runreckon measures.
|
|
45
|
+
|
|
46
|
+
## Quickstart
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install git+https://github.com/ranadhir19/runreckon.git
|
|
50
|
+
runreckon demo # the three bundled examples
|
|
51
|
+
runreckon score my-run.jsonl --spec my-task.json # your own run
|
|
52
|
+
runreckon score my-run.jsonl --spec my-task.json --format json
|
|
53
|
+
runreckon selftest # measure the scorer itself
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Pure Python standard library. No dependencies, no API keys, no network:
|
|
57
|
+
the deterministic core runs fully offline. Developed on Python 3.12;
|
|
58
|
+
any Python ≥ 3.10 works.
|
|
59
|
+
|
|
60
|
+
Exit codes are CI-shaped: `0` pass, `1` fail (composite below
|
|
61
|
+
`--min-score`, default 0.75, or any policy violation), `2` the trace or
|
|
62
|
+
spec could not be read.
|
|
63
|
+
|
|
64
|
+
## What it scores
|
|
65
|
+
|
|
66
|
+
| Dimension | What it checks | How |
|
|
67
|
+
|---|---|---|
|
|
68
|
+
| outcome | Did the run deliver? Final-state checks, required answer content | exact |
|
|
69
|
+
| tool_selection | The tools the spec expected were used; no undeclared tools were touched | exact |
|
|
70
|
+
| argument_validity | Every call's arguments against the spec's JSON Schemas | exact |
|
|
71
|
+
| step_economy | Repeated calls that already succeeded, loop patterns, calls after the final event | exact |
|
|
72
|
+
| error_recovery | Failures that were recovered vs failures silently stepped over | exact |
|
|
73
|
+
| policy | Forbidden tools, forbidden argument patterns, required ordering (approval before action), call limits, forbidden end-states | exact — and a hard gate |
|
|
74
|
+
| cost_latency | Steps, tokens, recorded latency, and priced cost against the spec's budgets | exact |
|
|
75
|
+
| groundedness | Every checkable claim in the final answer (numbers, URLs, emails, quoted strings) appears in the evidence the agent actually had | exact |
|
|
76
|
+
|
|
77
|
+
Two design rules:
|
|
78
|
+
|
|
79
|
+
- **Deterministic first.** Everything that can be computed exactly is
|
|
80
|
+
computed exactly. A repeated call after an *error* is a retry, not
|
|
81
|
+
waste; a repeat with *different arguments* is new work; a number the
|
|
82
|
+
*user* put in the task prompt is grounded, not invented. The scorers
|
|
83
|
+
encode those distinctions, and the self-test's negative controls
|
|
84
|
+
prove they hold.
|
|
85
|
+
- **Judges are optional and measured.** Groundedness can also be
|
|
86
|
+
assessed by a pluggable judge (`--judge module:function`, or the
|
|
87
|
+
bundled deterministic stand-in). A judge is only trusted as far as it
|
|
88
|
+
has been measured: whenever one is enabled, its agreement and Cohen's
|
|
89
|
+
kappa on a bundled labelled calibration set are printed on the
|
|
90
|
+
scorecard next to its verdict — reported alongside the deterministic
|
|
91
|
+
score, never blended into it. The stand-in's measured calibration:
|
|
92
|
+
agreement 0.833, kappa 0.667 on 12 items, with two instructive
|
|
93
|
+
failures (a heavy-paraphrase miss, a negation false-pass). That is
|
|
94
|
+
the point: an unmeasured judge is a hope, a measured one is an
|
|
95
|
+
instrument with known error.
|
|
96
|
+
|
|
97
|
+
This calibration is a disclosure about runreckon's own optional
|
|
98
|
+
judge, printed so its verdicts are never taken on faith. Systematic
|
|
99
|
+
grader validation — measuring any grader against labelled data and
|
|
100
|
+
auditing the task suites it scores — is evalcrucible's job, not
|
|
101
|
+
runreckon's.
|
|
102
|
+
|
|
103
|
+
## The evaluator evaluates itself
|
|
104
|
+
|
|
105
|
+
runreckon ships a fault-injection self-test. It takes clean traces that
|
|
106
|
+
score perfectly, plants one known defect per copy — a wrong tool, an
|
|
107
|
+
invalid argument, an ungrounded claim, a silent policy breach, loop
|
|
108
|
+
waste, a masked failure — and checks that the scorer responsible for
|
|
109
|
+
that defect class catches it. It also runs negative controls:
|
|
110
|
+
legitimate traces that resemble defects and must not be flagged.
|
|
111
|
+
|
|
112
|
+
Measured on this tree (3 clean bases × 6 defect classes, regenerated
|
|
113
|
+
and byte-verified by `validate.py`; full detail in
|
|
114
|
+
[SELFTEST_RESULTS.md](SELFTEST_RESULTS.md)):
|
|
115
|
+
|
|
116
|
+
| Defect class | Scorer responsible | Planted | Caught | Rate |
|
|
117
|
+
|---|---|---|---|---|
|
|
118
|
+
| wrong_tool | tool_selection | 3 | 3 | 1.000 |
|
|
119
|
+
| invalid_args | argument_validity | 3 | 3 | 1.000 |
|
|
120
|
+
| ungrounded_claim | groundedness | 3 | 3 | 1.000 |
|
|
121
|
+
| policy_breach | policy | 3 | 3 | 1.000 |
|
|
122
|
+
| loop_waste | step_economy | 3 | 3 | 1.000 |
|
|
123
|
+
| silent_failure | error_recovery | 3 | 3 | 1.000 |
|
|
124
|
+
|
|
125
|
+
**18/18 planted defects caught; 3/3 negative controls pass.** The base
|
|
126
|
+
set is small and author-built — treat the rates as a regression floor
|
|
127
|
+
for the scorers, not as a field measurement. Widen the bases before
|
|
128
|
+
quoting the rates as evidence about anyone else's agents.
|
|
129
|
+
|
|
130
|
+
The self-test also caught a real bug during development: an email
|
|
131
|
+
address at the end of a sentence ("…sent to ana@example.com.") was
|
|
132
|
+
extracted with its sentence-final period attached and falsely flagged
|
|
133
|
+
as ungrounded. The fix is in, and a regression test pins it.
|
|
134
|
+
|
|
135
|
+
## Inputs
|
|
136
|
+
|
|
137
|
+
A **trace** is JSONL — one event per line: `run`, `call`, `result`,
|
|
138
|
+
`note`, `final`. A **spec** is JSON: expected/allowed tools, argument
|
|
139
|
+
schemas, a reference path, policy rules, final-state checks, budgets.
|
|
140
|
+
Full formats: [docs/trace-format.md](docs/trace-format.md) and
|
|
141
|
+
[docs/spec-format.md](docs/spec-format.md). The three bundled examples
|
|
142
|
+
(a clean run, the showcase above, and a policy breach) live in the
|
|
143
|
+
package under `runreckon/examples/` and run with `runreckon demo`.
|
|
144
|
+
|
|
145
|
+
## Gate a pull request on run quality
|
|
146
|
+
|
|
147
|
+
This repo is also a GitHub Action. Record a trace in your agent's test
|
|
148
|
+
suite, commit the trace and its spec, and fail the PR when execution
|
|
149
|
+
quality regresses:
|
|
150
|
+
|
|
151
|
+
```yaml
|
|
152
|
+
- uses: ranadhir19/runreckon@v0.1.0
|
|
153
|
+
with:
|
|
154
|
+
trace: tests/traces/checkout-run.jsonl
|
|
155
|
+
spec: tests/traces/checkout-task.json
|
|
156
|
+
min-score: "0.90"
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
See [docs/action.md](docs/action.md) for inputs and a fuller workflow.
|
|
160
|
+
|
|
161
|
+
## How this differs from evalcrucible
|
|
162
|
+
|
|
163
|
+
[evalcrucible](https://github.com/ranadhir19/evalcrucible) is the
|
|
164
|
+
grader tool: it validates graders against labelled data and audits
|
|
165
|
+
static (prompt → gold) task suites, so the benchmark itself can be
|
|
166
|
+
trusted. runreckon is the eval tool: given a task spec you already
|
|
167
|
+
trust, it scores how an agent executed a recorded run.
|
|
168
|
+
|
|
169
|
+
## Limits
|
|
170
|
+
|
|
171
|
+
- Scoring is only as good as the spec. A spec with no schemas, no
|
|
172
|
+
policy, and no budgets scores only what it declares; dimensions
|
|
173
|
+
without inputs report `n/a` and are excluded from the composite,
|
|
174
|
+
never silently zeroed or gifted.
|
|
175
|
+
- Groundedness checks *checkable* claims — numbers, URLs, emails,
|
|
176
|
+
quoted strings. A fluent falsehood with no checkable token in it is
|
|
177
|
+
invisible to the deterministic layer; that is what the (measured)
|
|
178
|
+
judge layer is for.
|
|
179
|
+
- Latency is the sum of per-event recorded latencies, not wall-clock;
|
|
180
|
+
traces that do not record tokens or latency cannot be scored on them.
|
|
181
|
+
- The fault-injection bases are three small author-built runs. The
|
|
182
|
+
mechanism generalises; the specific rates are a floor, not a census.
|
|
183
|
+
|
|
184
|
+
## Development
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
git clone https://github.com/ranadhir19/runreckon.git
|
|
188
|
+
cd runreckon
|
|
189
|
+
python3 validate.py # must end with OVERALL PASS
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
The harness compiles the package, runs the unit tests (authored from
|
|
193
|
+
the failure-mode domain, including the near-miss negatives), reproduces
|
|
194
|
+
all three example scorecards against their expected bands, re-runs the
|
|
195
|
+
self-test and requires `SELFTEST_RESULTS.md` to come back
|
|
196
|
+
byte-identical, and runs the portfolio's pre-push integrity gates.
|
|
197
|
+
|
|
198
|
+
## License
|
|
199
|
+
|
|
200
|
+
MIT — see [LICENSE](LICENSE). Copyright (c) 2026 Ranadhir Ghosh.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "runreckon"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Score how an AI agent executed a run, not just whether it finished: deterministic scoring of recorded agent traces (tool selection, argument validity, step economy, error recovery, policy compliance, cost) with a fault-injection self-test that measures the scorer itself."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Ranadhir Ghosh" }]
|
|
13
|
+
keywords = ["agent-evaluation", "llm-evals", "ai-agents", "agent-traces", "evaluation", "mcp", "observability", "ci"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
20
|
+
"Topic :: Software Development :: Testing",
|
|
21
|
+
]
|
|
22
|
+
dependencies = []
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
runreckon = "runreckon.cli:main"
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/ranadhir19/runreckon"
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
where = ["src"]
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.package-data]
|
|
34
|
+
runreckon = ["examples/*/*.json", "examples/*/*.jsonl", "examples/*.json"]
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""runreckon command line.
|
|
2
|
+
|
|
3
|
+
runreckon score TRACE.jsonl --spec SPEC.json [--format text|json]
|
|
4
|
+
[--min-score 0.75] [--judge standin]
|
|
5
|
+
[--no-fail-on-policy]
|
|
6
|
+
runreckon demo [--format text|json]
|
|
7
|
+
runreckon selftest [--format text|json] [--write PATH]
|
|
8
|
+
runreckon version
|
|
9
|
+
|
|
10
|
+
Exit codes: 0 = the run passes (composite >= --min-score and, unless
|
|
11
|
+
--no-fail-on-policy, no policy violation); 1 = the run fails;
|
|
12
|
+
2 = the trace or spec could not be read (format error).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import importlib
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from . import __version__
|
|
24
|
+
from .scorecard import Scorecard
|
|
25
|
+
from .scorers import run_deterministic
|
|
26
|
+
from .semantic import judge_report, score_groundedness, standin_judge
|
|
27
|
+
from .spec import SpecError, load_spec
|
|
28
|
+
from .trace import TraceFormatError, load_trace
|
|
29
|
+
|
|
30
|
+
EXAMPLES_DIR = Path(__file__).parent / "examples"
|
|
31
|
+
EXAMPLE_CASES = ["clean-run", "sloppy-success", "policy-breach"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _load_judge(name: str):
|
|
35
|
+
if name == "standin":
|
|
36
|
+
return standin_judge, "standin (deterministic heuristic)"
|
|
37
|
+
module_name, _, func_name = name.partition(":")
|
|
38
|
+
if not func_name:
|
|
39
|
+
raise SpecError(f"judge {name!r} must be 'standin' or 'module:function'")
|
|
40
|
+
module = importlib.import_module(module_name)
|
|
41
|
+
return getattr(module, func_name), name
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _score(trace, spec, judge_name: str | None) -> Scorecard:
|
|
45
|
+
dimensions = run_deterministic(trace, spec)
|
|
46
|
+
dimensions.append(score_groundedness(trace, spec))
|
|
47
|
+
judge = None
|
|
48
|
+
if judge_name:
|
|
49
|
+
judge_fn, label = _load_judge(judge_name)
|
|
50
|
+
judge = judge_report(trace, spec, judge_fn, label)
|
|
51
|
+
return Scorecard(trace, spec, dimensions, judge=judge)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cmd_score(args) -> int:
|
|
55
|
+
try:
|
|
56
|
+
trace = load_trace(args.trace)
|
|
57
|
+
spec = load_spec(args.spec)
|
|
58
|
+
card = _score(trace, spec, args.judge)
|
|
59
|
+
except (TraceFormatError, SpecError, OSError) as exc:
|
|
60
|
+
print(f"runreckon: cannot score: {exc}", file=sys.stderr)
|
|
61
|
+
return 2
|
|
62
|
+
if args.format == "json":
|
|
63
|
+
print(json.dumps(card.to_dict(), indent=2))
|
|
64
|
+
else:
|
|
65
|
+
print(card.render_text())
|
|
66
|
+
return card.exit_code(min_score=args.min_score,
|
|
67
|
+
fail_on_policy=not args.no_fail_on_policy)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def cmd_demo(args) -> int:
|
|
71
|
+
worst = 0
|
|
72
|
+
for case in EXAMPLE_CASES:
|
|
73
|
+
folder = EXAMPLES_DIR / case
|
|
74
|
+
trace = load_trace(str(folder / "trace.jsonl"))
|
|
75
|
+
spec = load_spec(str(folder / "spec.json"))
|
|
76
|
+
card = _score(trace, spec, None)
|
|
77
|
+
if args.format == "json":
|
|
78
|
+
print(json.dumps({case: card.to_dict()}, indent=2))
|
|
79
|
+
else:
|
|
80
|
+
print(f"### example: {case}")
|
|
81
|
+
print(card.render_text())
|
|
82
|
+
print()
|
|
83
|
+
worst = max(worst, card.exit_code())
|
|
84
|
+
return worst
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cmd_selftest(args) -> int:
|
|
88
|
+
from .selftest import render_report, run_selftest
|
|
89
|
+
report = run_selftest()
|
|
90
|
+
if args.write:
|
|
91
|
+
Path(args.write).write_text(render_report(report), encoding="utf-8")
|
|
92
|
+
print(f"self-test report written to {args.write}")
|
|
93
|
+
elif args.format == "json":
|
|
94
|
+
print(json.dumps(report, indent=2))
|
|
95
|
+
else:
|
|
96
|
+
print(render_report(report))
|
|
97
|
+
return 0
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def main(argv: list[str] | None = None) -> int:
|
|
101
|
+
parser = argparse.ArgumentParser(
|
|
102
|
+
prog="runreckon",
|
|
103
|
+
description="Score how an agent executed a run, not just whether it finished.")
|
|
104
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
105
|
+
|
|
106
|
+
p_score = sub.add_parser("score", help="score one trace against a task spec")
|
|
107
|
+
p_score.add_argument("trace", help="path to the JSONL trace")
|
|
108
|
+
p_score.add_argument("--spec", required=True, help="path to the task spec JSON")
|
|
109
|
+
p_score.add_argument("--format", choices=["text", "json"], default="text")
|
|
110
|
+
p_score.add_argument("--min-score", type=float, default=0.75,
|
|
111
|
+
help="minimum composite (0-1) to pass (default 0.75)")
|
|
112
|
+
p_score.add_argument("--judge", default=None,
|
|
113
|
+
help="'standin' or 'module:function' — off by default; "
|
|
114
|
+
"when enabled, its calibration is printed")
|
|
115
|
+
p_score.add_argument("--no-fail-on-policy", action="store_true",
|
|
116
|
+
help="do not hard-fail on policy violations")
|
|
117
|
+
p_score.set_defaults(func=cmd_score)
|
|
118
|
+
|
|
119
|
+
p_demo = sub.add_parser("demo", help="score the three bundled example runs")
|
|
120
|
+
p_demo.add_argument("--format", choices=["text", "json"], default="text")
|
|
121
|
+
p_demo.set_defaults(func=cmd_demo)
|
|
122
|
+
|
|
123
|
+
p_self = sub.add_parser(
|
|
124
|
+
"selftest", help="fault-injection self-test: measure the scorer itself")
|
|
125
|
+
p_self.add_argument("--format", choices=["text", "json"], default="text")
|
|
126
|
+
p_self.add_argument("--write", default=None,
|
|
127
|
+
help="write the canonical markdown report to this path")
|
|
128
|
+
p_self.set_defaults(func=cmd_selftest)
|
|
129
|
+
|
|
130
|
+
p_ver = sub.add_parser("version", help="print the version")
|
|
131
|
+
p_ver.set_defaults(func=lambda _args: print(__version__) or 0)
|
|
132
|
+
|
|
133
|
+
args = parser.parse_args(argv)
|
|
134
|
+
return args.func(args)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
if __name__ == "__main__":
|
|
138
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"allowed_tools":["web_search","fetch_page","save_brief"],"budgets":{"max_cost_usd":0.1,"max_latency_ms":60000,"max_steps":8,"max_tokens":6000},"expected_tools":["web_search","fetch_page","save_brief"],"final_checks":[{"state_equals":{"path":"brief.saved","value":true}},{"answer_contains":"TEU"}],"goal":"Research the Port of Jacksonville's 2024 container volume and save a short brief","policy":[{"forbid_tool":"send_email"},{"forbid_arg":{"arg":"url","pattern":"internal\\.","tool":"fetch_page"}},{"max_calls":{"count":4,"tool":"web_search"}}],"pricing":{"input_per_1k_usd":0.003,"output_per_1k_usd":0.015},"reference_path":["web_search","fetch_page","save_brief"],"task_id":"research-brief","task_prompt":"Find the Port of Jacksonville's total container volume (TEU) for 2024 from a public source, then save a brief containing the figure. Do not email anyone.","tool_schemas":{"fetch_page":{"additionalProperties":false,"properties":{"url":{"pattern":"^https?://","type":"string"}},"required":["url"],"type":"object"},"save_brief":{"additionalProperties":false,"properties":{"body":{"minLength":1,"type":"string"},"title":{"minLength":1,"type":"string"}},"required":["title","body"],"type":"object"},"web_search":{"additionalProperties":false,"properties":{"max_results":{"maximum":10,"minimum":1,"type":"integer"},"query":{"minLength":3,"type":"string"}},"required":["query"],"type":"object"}}}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
{"type": "run", "run_id": "ex-clean-001", "task_id": "research-brief", "agent": "brief-writer 1.0"}
|
|
2
|
+
{"type": "call", "step": 1, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
|
|
3
|
+
{"type": "result", "step": 1, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 800, "tokens_in": 120, "tokens_out": 40}
|
|
4
|
+
{"type": "call", "step": 2, "tool": "fetch_page", "args": {"url": "https://www.jaxport.com/news/2024-volumes"}}
|
|
5
|
+
{"type": "result", "step": 2, "ok": true, "output": {"text": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}, "error": null, "latency_ms": 1200, "tokens_in": 900, "tokens_out": 60}
|
|
6
|
+
{"type": "call", "step": 3, "tool": "save_brief", "args": {"title": "JAXPORT 2024 container volume", "body": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}}
|
|
7
|
+
{"type": "result", "step": 3, "ok": true, "output": {"saved": true, "path": "briefs/jaxport-2024.md"}, "error": null, "latency_ms": 150, "tokens_in": 200, "tokens_out": 20}
|
|
8
|
+
{"type": "final", "answer": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023. The brief is saved.", "state": {"brief": {"saved": true}}, "latency_ms": 900, "tokens_in": 300, "tokens_out": 80}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"clean-run":{"composite":[0.99,1.0],"dimensions":{"argument_validity":[0.99,1.0],"cost_latency":[0.99,1.0],"error_recovery":[0.99,1.0],"groundedness":[0.99,1.0],"outcome":[0.99,1.0],"policy":[0.99,1.0],"step_economy":[0.99,1.0],"tool_selection":[0.99,1.0]},"exit_default":0,"policy_violations":0},"policy-breach":{"composite":[0.7,0.95],"dimensions":{"outcome":[0.99,1.0],"policy":[0.0,0.0]},"exit_default":1,"findings_contain":["forbidden tool 'send_email'"],"policy_violations":1},"sloppy-success":{"composite":[0.8,0.92],"dimensions":{"argument_validity":[0.8,0.9],"cost_latency":[0.35,0.55],"error_recovery":[0.99,1.0],"groundedness":[0.99,1.0],"outcome":[0.99,1.0],"policy":[0.99,1.0],"step_economy":[0.6,0.72],"tool_selection":[0.8,0.95]},"exit_default":0,"exit_min_090":1,"findings_contain":["wasted step","budget exceeded","not in the spec's permitted set"],"policy_violations":0}}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"allowed_tools":["web_search","fetch_page","save_brief"],"budgets":{"max_cost_usd":0.1,"max_latency_ms":60000,"max_steps":8,"max_tokens":6000},"expected_tools":["web_search","fetch_page","save_brief"],"final_checks":[{"state_equals":{"path":"brief.saved","value":true}},{"answer_contains":"TEU"}],"goal":"Research the Port of Jacksonville's 2024 container volume and save a short brief","policy":[{"forbid_tool":"send_email"},{"forbid_arg":{"arg":"url","pattern":"internal\\.","tool":"fetch_page"}},{"max_calls":{"count":4,"tool":"web_search"}}],"pricing":{"input_per_1k_usd":0.003,"output_per_1k_usd":0.015},"reference_path":["web_search","fetch_page","save_brief"],"task_id":"research-brief","task_prompt":"Find the Port of Jacksonville's total container volume (TEU) for 2024 from a public source, then save a brief containing the figure. Do not email anyone.","tool_schemas":{"fetch_page":{"additionalProperties":false,"properties":{"url":{"pattern":"^https?://","type":"string"}},"required":["url"],"type":"object"},"save_brief":{"additionalProperties":false,"properties":{"body":{"minLength":1,"type":"string"},"title":{"minLength":1,"type":"string"}},"required":["title","body"],"type":"object"},"web_search":{"additionalProperties":false,"properties":{"max_results":{"maximum":10,"minimum":1,"type":"integer"},"query":{"minLength":3,"type":"string"}},"required":["query"],"type":"object"}}}
|