runreckon 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. runreckon-0.1.0/LICENSE +21 -0
  2. runreckon-0.1.0/PKG-INFO +219 -0
  3. runreckon-0.1.0/README.md +200 -0
  4. runreckon-0.1.0/pyproject.toml +34 -0
  5. runreckon-0.1.0/setup.cfg +4 -0
  6. runreckon-0.1.0/src/runreckon/__init__.py +3 -0
  7. runreckon-0.1.0/src/runreckon/cli.py +138 -0
  8. runreckon-0.1.0/src/runreckon/examples/clean-run/spec.json +1 -0
  9. runreckon-0.1.0/src/runreckon/examples/clean-run/trace.jsonl +8 -0
  10. runreckon-0.1.0/src/runreckon/examples/expected.json +1 -0
  11. runreckon-0.1.0/src/runreckon/examples/policy-breach/spec.json +1 -0
  12. runreckon-0.1.0/src/runreckon/examples/policy-breach/trace.jsonl +10 -0
  13. runreckon-0.1.0/src/runreckon/examples/sloppy-success/spec.json +1 -0
  14. runreckon-0.1.0/src/runreckon/examples/sloppy-success/trace.jsonl +20 -0
  15. runreckon-0.1.0/src/runreckon/inject.py +193 -0
  16. runreckon-0.1.0/src/runreckon/schema.py +109 -0
  17. runreckon-0.1.0/src/runreckon/scorecard.py +103 -0
  18. runreckon-0.1.0/src/runreckon/scorers.py +324 -0
  19. runreckon-0.1.0/src/runreckon/selftest.py +269 -0
  20. runreckon-0.1.0/src/runreckon/semantic.py +211 -0
  21. runreckon-0.1.0/src/runreckon/spec.py +94 -0
  22. runreckon-0.1.0/src/runreckon/trace.py +191 -0
  23. runreckon-0.1.0/src/runreckon.egg-info/PKG-INFO +219 -0
  24. runreckon-0.1.0/src/runreckon.egg-info/SOURCES.txt +31 -0
  25. runreckon-0.1.0/src/runreckon.egg-info/dependency_links.txt +1 -0
  26. runreckon-0.1.0/src/runreckon.egg-info/entry_points.txt +2 -0
  27. runreckon-0.1.0/src/runreckon.egg-info/top_level.txt +1 -0
  28. runreckon-0.1.0/tests/test_cli.py +90 -0
  29. runreckon-0.1.0/tests/test_inject.py +44 -0
  30. runreckon-0.1.0/tests/test_schema.py +60 -0
  31. runreckon-0.1.0/tests/test_scorers.py +195 -0
  32. runreckon-0.1.0/tests/test_semantic.py +88 -0
  33. runreckon-0.1.0/tests/test_trace.py +59 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ranadhir Ghosh
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,219 @@
1
+ Metadata-Version: 2.4
2
+ Name: runreckon
3
+ Version: 0.1.0
4
+ Summary: Score how an AI agent executed a run, not just whether it finished: deterministic scoring of recorded agent traces (tool selection, argument validity, step economy, error recovery, policy compliance, cost) with a fault-injection self-test that measures the scorer itself.
5
+ Author: Ranadhir Ghosh
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/ranadhir19/runreckon
8
+ Keywords: agent-evaluation,llm-evals,ai-agents,agent-traces,evaluation,mcp,observability,ci
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
14
+ Classifier: Topic :: Software Development :: Testing
15
+ Requires-Python: >=3.10
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Dynamic: license-file
19
+
20
+ # runreckon
21
+
22
+ **Score how an agent executed a run, not just whether it finished.**
23
+
24
+ An agent can complete a task and still do it badly: call the same search
25
+ four times, reach for tools it was never given, pass malformed arguments,
26
+ blow the token budget, quietly break a policy on the way — and deliver a
27
+ final answer that passes every outcome check. Outcome-only evaluation
28
+ calls that run a success. runreckon scores the *execution*: feed it a
29
+ recorded run (a JSONL trace) and the task spec it was supposed to
30
+ follow, and it produces a scorecard for the run itself.
31
+
32
+ ## The showcase case
33
+
34
+ This run **succeeded**. Its brief was saved, its final checks pass, its
35
+ answer is fully grounded. Here is its scorecard (`runreckon demo`, the
36
+ `sloppy-success` example, real output):
37
+
38
+ ```text
39
+ outcome 100.0
40
+ tool_selection 88.9
41
+ - step 5: tool 'web_scrape' is not in the spec's permitted set
42
+ - step 6: tool 'summarize' is not in the spec's permitted set
43
+ argument_validity 85.7
44
+ - step 7 (fetch_page): $: missing required property 'url'
45
+ step_economy 66.7
46
+ - step 2: 'web_search' repeated with identical arguments after already succeeding — wasted step
47
+ - step 3: 'web_search' repeated with identical arguments after already succeeding — wasted step
48
+ - step 4: 'web_search' repeated with identical arguments after already succeeding — wasted step
49
+ - 'web_search' called 4 times with identical arguments — loop pattern
50
+ error_recovery 100.0
51
+ policy 100.0
52
+ cost_latency 43.8
53
+ - budget exceeded: max_steps 9 > 8
54
+ - budget exceeded: max_tokens 9370 > 6000
55
+ groundedness 100.0
56
+ ------------------------------------------------------------
57
+ COMPOSITE 87.6 — serviceable
58
+ ```
59
+
60
+ The task succeeded; the run was still bad — four identical searches, two
61
+ undeclared tools, one malformed call, two budgets blown. With
62
+ `--min-score 0.90` this run fails CI. That gap between "it worked" and
63
+ "it ran well" is what runreckon measures.
64
+
65
+ ## Quickstart
66
+
67
+ ```bash
68
+ pip install git+https://github.com/ranadhir19/runreckon.git
69
+ runreckon demo # the three bundled examples
70
+ runreckon score my-run.jsonl --spec my-task.json # your own run
71
+ runreckon score my-run.jsonl --spec my-task.json --format json
72
+ runreckon selftest # measure the scorer itself
73
+ ```
74
+
75
+ Pure Python standard library. No dependencies, no API keys, no network:
76
+ the deterministic core runs fully offline. Developed on Python 3.12;
77
+ any Python ≥ 3.10 works.
78
+
79
+ Exit codes are CI-shaped: `0` pass, `1` fail (composite below
80
+ `--min-score`, default 0.75, or any policy violation), `2` the trace or
81
+ spec could not be read.
82
+
83
+ ## What it scores
84
+
85
+ | Dimension | What it checks | How |
86
+ |---|---|---|
87
+ | outcome | Did the run deliver? Final-state checks, required answer content | exact |
88
+ | tool_selection | The tools the spec expected were used; no undeclared tools were touched | exact |
89
+ | argument_validity | Every call's arguments against the spec's JSON Schemas | exact |
90
+ | step_economy | Repeated calls that already succeeded, loop patterns, calls after the final event | exact |
91
+ | error_recovery | Failures that were recovered vs failures silently stepped over | exact |
92
+ | policy | Forbidden tools, forbidden argument patterns, required ordering (approval before action), call limits, forbidden end-states | exact — and a hard gate |
93
+ | cost_latency | Steps, tokens, recorded latency, and priced cost against the spec's budgets | exact |
94
+ | groundedness | Every checkable claim in the final answer (numbers, URLs, emails, quoted strings) appears in the evidence the agent actually had | exact |
95
+
96
+ Two design rules:
97
+
98
+ - **Deterministic first.** Everything that can be computed exactly is
99
+ computed exactly. A repeated call after an *error* is a retry, not
100
+ waste; a repeat with *different arguments* is new work; a number the
101
+ *user* put in the task prompt is grounded, not invented. The scorers
102
+ encode those distinctions, and the self-test's negative controls
103
+ prove they hold.
104
+ - **Judges are optional and measured.** Groundedness can also be
105
+ assessed by a pluggable judge (`--judge module:function`, or the
106
+ bundled deterministic stand-in). A judge is only trusted as far as it
107
+ has been measured: whenever one is enabled, its agreement and Cohen's
108
+ kappa on a bundled labelled calibration set are printed on the
109
+ scorecard next to its verdict — reported alongside the deterministic
110
+ score, never blended into it. The stand-in's measured calibration:
111
+ agreement 0.833, kappa 0.667 on 12 items, with two instructive
112
+ failures (a heavy-paraphrase miss, a negation false-pass). That is
113
+ the point: an unmeasured judge is a hope, a measured one is an
114
+ instrument with known error.
115
+
116
+ This calibration is a disclosure about runreckon's own optional
117
+ judge, printed so its verdicts are never taken on faith. Systematic
118
+ grader validation — measuring any grader against labelled data and
119
+ auditing the task suites it scores — is evalcrucible's job, not
120
+ runreckon's.
121
+
122
+ ## The evaluator evaluates itself
123
+
124
+ runreckon ships a fault-injection self-test. It takes clean traces that
125
+ score perfectly, plants one known defect per copy — a wrong tool, an
126
+ invalid argument, an ungrounded claim, a silent policy breach, loop
127
+ waste, a masked failure — and checks that the scorer responsible for
128
+ that defect class catches it. It also runs negative controls:
129
+ legitimate traces that resemble defects and must not be flagged.
130
+
131
+ Measured on this tree (3 clean bases × 6 defect classes, regenerated
132
+ and byte-verified by `validate.py`; full detail in
133
+ [SELFTEST_RESULTS.md](SELFTEST_RESULTS.md)):
134
+
135
+ | Defect class | Scorer responsible | Planted | Caught | Rate |
136
+ |---|---|---|---|---|
137
+ | wrong_tool | tool_selection | 3 | 3 | 1.000 |
138
+ | invalid_args | argument_validity | 3 | 3 | 1.000 |
139
+ | ungrounded_claim | groundedness | 3 | 3 | 1.000 |
140
+ | policy_breach | policy | 3 | 3 | 1.000 |
141
+ | loop_waste | step_economy | 3 | 3 | 1.000 |
142
+ | silent_failure | error_recovery | 3 | 3 | 1.000 |
143
+
144
+ **18/18 planted defects caught; 3/3 negative controls pass.** The base
145
+ set is small and author-built — treat the rates as a regression floor
146
+ for the scorers, not as a field measurement. Widen the bases before
147
+ quoting the rates as evidence about anyone else's agents.
148
+
149
+ The self-test also caught a real bug during development: an email
150
+ address at the end of a sentence ("…sent to ana@example.com.") was
151
+ extracted with its sentence-final period attached and falsely flagged
152
+ as ungrounded. The fix is in, and a regression test pins it.
153
+
154
+ ## Inputs
155
+
156
+ A **trace** is JSONL — one event per line: `run`, `call`, `result`,
157
+ `note`, `final`. A **spec** is JSON: expected/allowed tools, argument
158
+ schemas, a reference path, policy rules, final-state checks, budgets.
159
+ Full formats: [docs/trace-format.md](docs/trace-format.md) and
160
+ [docs/spec-format.md](docs/spec-format.md). The three bundled examples
161
+ (a clean run, the showcase above, and a policy breach) live in the
162
+ package under `runreckon/examples/` and run with `runreckon demo`.
163
+
164
+ ## Gate a pull request on run quality
165
+
166
+ This repo is also a GitHub Action. Record a trace in your agent's test
167
+ suite, commit the trace and its spec, and fail the PR when execution
168
+ quality regresses:
169
+
170
+ ```yaml
171
+ - uses: ranadhir19/runreckon@v0.1.0
172
+ with:
173
+ trace: tests/traces/checkout-run.jsonl
174
+ spec: tests/traces/checkout-task.json
175
+ min-score: "0.90"
176
+ ```
177
+
178
+ See [docs/action.md](docs/action.md) for inputs and a fuller workflow.
179
+
180
+ ## How this differs from evalcrucible
181
+
182
+ [evalcrucible](https://github.com/ranadhir19/evalcrucible) is the
183
+ grader tool: it validates graders against labelled data and audits
184
+ static (prompt → gold) task suites, so the benchmark itself can be
185
+ trusted. runreckon is the eval tool: given a task spec you already
186
+ trust, it scores how an agent executed a recorded run.
187
+
188
+ ## Limits
189
+
190
+ - Scoring is only as good as the spec. A spec with no schemas, no
191
+ policy, and no budgets scores only what it declares; dimensions
192
+ without inputs report `n/a` and are excluded from the composite,
193
+ never silently zeroed or gifted.
194
+ - Groundedness checks *checkable* claims — numbers, URLs, emails,
195
+ quoted strings. A fluent falsehood with no checkable token in it is
196
+ invisible to the deterministic layer; that is what the (measured)
197
+ judge layer is for.
198
+ - Latency is the sum of per-event recorded latencies, not wall-clock;
199
+ traces that do not record tokens or latency cannot be scored on them.
200
+ - The fault-injection bases are three small author-built runs. The
201
+ mechanism generalises; the specific rates are a floor, not a census.
202
+
203
+ ## Development
204
+
205
+ ```bash
206
+ git clone https://github.com/ranadhir19/runreckon.git
207
+ cd runreckon
208
+ python3 validate.py # must end with OVERALL PASS
209
+ ```
210
+
211
+ The harness compiles the package, runs the unit tests (authored from
212
+ the failure-mode domain, including the near-miss negatives), reproduces
213
+ all three example scorecards against their expected bands, re-runs the
214
+ self-test and requires `SELFTEST_RESULTS.md` to come back
215
+ byte-identical, and runs the portfolio's pre-push integrity gates.
216
+
217
+ ## License
218
+
219
+ MIT — see [LICENSE](LICENSE). Copyright (c) 2026 Ranadhir Ghosh.
@@ -0,0 +1,200 @@
1
+ # runreckon
2
+
3
+ **Score how an agent executed a run, not just whether it finished.**
4
+
5
+ An agent can complete a task and still do it badly: call the same search
6
+ four times, reach for tools it was never given, pass malformed arguments,
7
+ blow the token budget, quietly break a policy on the way — and deliver a
8
+ final answer that passes every outcome check. Outcome-only evaluation
9
+ calls that run a success. runreckon scores the *execution*: feed it a
10
+ recorded run (a JSONL trace) and the task spec it was supposed to
11
+ follow, and it produces a scorecard for the run itself.
12
+
13
+ ## The showcase case
14
+
15
+ This run **succeeded**. Its brief was saved, its final checks pass, its
16
+ answer is fully grounded. Here is its scorecard (`runreckon demo`, the
17
+ `sloppy-success` example, real output):
18
+
19
+ ```text
20
+ outcome 100.0
21
+ tool_selection 88.9
22
+ - step 5: tool 'web_scrape' is not in the spec's permitted set
23
+ - step 6: tool 'summarize' is not in the spec's permitted set
24
+ argument_validity 85.7
25
+ - step 7 (fetch_page): $: missing required property 'url'
26
+ step_economy 66.7
27
+ - step 2: 'web_search' repeated with identical arguments after already succeeding — wasted step
28
+ - step 3: 'web_search' repeated with identical arguments after already succeeding — wasted step
29
+ - step 4: 'web_search' repeated with identical arguments after already succeeding — wasted step
30
+ - 'web_search' called 4 times with identical arguments — loop pattern
31
+ error_recovery 100.0
32
+ policy 100.0
33
+ cost_latency 43.8
34
+ - budget exceeded: max_steps 9 > 8
35
+ - budget exceeded: max_tokens 9370 > 6000
36
+ groundedness 100.0
37
+ ------------------------------------------------------------
38
+ COMPOSITE 87.6 — serviceable
39
+ ```
40
+
41
+ The task succeeded; the run was still bad — four identical searches, two
42
+ undeclared tools, one malformed call, two budgets blown. With
43
+ `--min-score 0.90` this run fails CI. That gap between "it worked" and
44
+ "it ran well" is what runreckon measures.
45
+
46
+ ## Quickstart
47
+
48
+ ```bash
49
+ pip install git+https://github.com/ranadhir19/runreckon.git
50
+ runreckon demo # the three bundled examples
51
+ runreckon score my-run.jsonl --spec my-task.json # your own run
52
+ runreckon score my-run.jsonl --spec my-task.json --format json
53
+ runreckon selftest # measure the scorer itself
54
+ ```
55
+
56
+ Pure Python standard library. No dependencies, no API keys, no network:
57
+ the deterministic core runs fully offline. Developed on Python 3.12;
58
+ any Python ≥ 3.10 works.
59
+
60
+ Exit codes are CI-shaped: `0` pass, `1` fail (composite below
61
+ `--min-score`, default 0.75, or any policy violation), `2` the trace or
62
+ spec could not be read.
63
+
64
+ ## What it scores
65
+
66
+ | Dimension | What it checks | How |
67
+ |---|---|---|
68
+ | outcome | Did the run deliver? Final-state checks, required answer content | exact |
69
+ | tool_selection | The tools the spec expected were used; no undeclared tools were touched | exact |
70
+ | argument_validity | Every call's arguments against the spec's JSON Schemas | exact |
71
+ | step_economy | Repeated calls that already succeeded, loop patterns, calls after the final event | exact |
72
+ | error_recovery | Failures that were recovered vs failures silently stepped over | exact |
73
+ | policy | Forbidden tools, forbidden argument patterns, required ordering (approval before action), call limits, forbidden end-states | exact — and a hard gate |
74
+ | cost_latency | Steps, tokens, recorded latency, and priced cost against the spec's budgets | exact |
75
+ | groundedness | Every checkable claim in the final answer (numbers, URLs, emails, quoted strings) appears in the evidence the agent actually had | exact |
76
+
77
+ Two design rules:
78
+
79
+ - **Deterministic first.** Everything that can be computed exactly is
80
+ computed exactly. A repeated call after an *error* is a retry, not
81
+ waste; a repeat with *different arguments* is new work; a number the
82
+ *user* put in the task prompt is grounded, not invented. The scorers
83
+ encode those distinctions, and the self-test's negative controls
84
+ prove they hold.
85
+ - **Judges are optional and measured.** Groundedness can also be
86
+ assessed by a pluggable judge (`--judge module:function`, or the
87
+ bundled deterministic stand-in). A judge is only trusted as far as it
88
+ has been measured: whenever one is enabled, its agreement and Cohen's
89
+ kappa on a bundled labelled calibration set are printed on the
90
+ scorecard next to its verdict — reported alongside the deterministic
91
+ score, never blended into it. The stand-in's measured calibration:
92
+ agreement 0.833, kappa 0.667 on 12 items, with two instructive
93
+ failures (a heavy-paraphrase miss, a negation false-pass). That is
94
+ the point: an unmeasured judge is a hope, a measured one is an
95
+ instrument with known error.
96
+
97
+ This calibration is a disclosure about runreckon's own optional
98
+ judge, printed so its verdicts are never taken on faith. Systematic
99
+ grader validation — measuring any grader against labelled data and
100
+ auditing the task suites it scores — is evalcrucible's job, not
101
+ runreckon's.
102
+
103
+ ## The evaluator evaluates itself
104
+
105
+ runreckon ships a fault-injection self-test. It takes clean traces that
106
+ score perfectly, plants one known defect per copy — a wrong tool, an
107
+ invalid argument, an ungrounded claim, a silent policy breach, loop
108
+ waste, a masked failure — and checks that the scorer responsible for
109
+ that defect class catches it. It also runs negative controls:
110
+ legitimate traces that resemble defects and must not be flagged.
111
+
112
+ Measured on this tree (3 clean bases × 6 defect classes, regenerated
113
+ and byte-verified by `validate.py`; full detail in
114
+ [SELFTEST_RESULTS.md](SELFTEST_RESULTS.md)):
115
+
116
+ | Defect class | Scorer responsible | Planted | Caught | Rate |
117
+ |---|---|---|---|---|
118
+ | wrong_tool | tool_selection | 3 | 3 | 1.000 |
119
+ | invalid_args | argument_validity | 3 | 3 | 1.000 |
120
+ | ungrounded_claim | groundedness | 3 | 3 | 1.000 |
121
+ | policy_breach | policy | 3 | 3 | 1.000 |
122
+ | loop_waste | step_economy | 3 | 3 | 1.000 |
123
+ | silent_failure | error_recovery | 3 | 3 | 1.000 |
124
+
125
+ **18/18 planted defects caught; 3/3 negative controls pass.** The base
126
+ set is small and author-built — treat the rates as a regression floor
127
+ for the scorers, not as a field measurement. Widen the bases before
128
+ quoting the rates as evidence about anyone else's agents.
129
+
130
+ The self-test also caught a real bug during development: an email
131
+ address at the end of a sentence ("…sent to ana@example.com.") was
132
+ extracted with its sentence-final period attached and falsely flagged
133
+ as ungrounded. The fix is in, and a regression test pins it.
134
+
135
+ ## Inputs
136
+
137
+ A **trace** is JSONL — one event per line: `run`, `call`, `result`,
138
+ `note`, `final`. A **spec** is JSON: expected/allowed tools, argument
139
+ schemas, a reference path, policy rules, final-state checks, budgets.
140
+ Full formats: [docs/trace-format.md](docs/trace-format.md) and
141
+ [docs/spec-format.md](docs/spec-format.md). The three bundled examples
142
+ (a clean run, the showcase above, and a policy breach) live in the
143
+ package under `runreckon/examples/` and run with `runreckon demo`.
144
+
145
+ ## Gate a pull request on run quality
146
+
147
+ This repo is also a GitHub Action. Record a trace in your agent's test
148
+ suite, commit the trace and its spec, and fail the PR when execution
149
+ quality regresses:
150
+
151
+ ```yaml
152
+ - uses: ranadhir19/runreckon@v0.1.0
153
+ with:
154
+ trace: tests/traces/checkout-run.jsonl
155
+ spec: tests/traces/checkout-task.json
156
+ min-score: "0.90"
157
+ ```
158
+
159
+ See [docs/action.md](docs/action.md) for inputs and a fuller workflow.
160
+
161
+ ## How this differs from evalcrucible
162
+
163
+ [evalcrucible](https://github.com/ranadhir19/evalcrucible) is the
164
+ grader tool: it validates graders against labelled data and audits
165
+ static (prompt → gold) task suites, so the benchmark itself can be
166
+ trusted. runreckon is the eval tool: given a task spec you already
167
+ trust, it scores how an agent executed a recorded run.
168
+
169
+ ## Limits
170
+
171
+ - Scoring is only as good as the spec. A spec with no schemas, no
172
+ policy, and no budgets scores only what it declares; dimensions
173
+ without inputs report `n/a` and are excluded from the composite,
174
+ never silently zeroed or gifted.
175
+ - Groundedness checks *checkable* claims — numbers, URLs, emails,
176
+ quoted strings. A fluent falsehood with no checkable token in it is
177
+ invisible to the deterministic layer; that is what the (measured)
178
+ judge layer is for.
179
+ - Latency is the sum of per-event recorded latencies, not wall-clock;
180
+ traces that do not record tokens or latency cannot be scored on them.
181
+ - The fault-injection bases are three small author-built runs. The
182
+ mechanism generalises; the specific rates are a floor, not a census.
183
+
184
+ ## Development
185
+
186
+ ```bash
187
+ git clone https://github.com/ranadhir19/runreckon.git
188
+ cd runreckon
189
+ python3 validate.py # must end with OVERALL PASS
190
+ ```
191
+
192
+ The harness compiles the package, runs the unit tests (authored from
193
+ the failure-mode domain, including the near-miss negatives), reproduces
194
+ all three example scorecards against their expected bands, re-runs the
195
+ self-test and requires `SELFTEST_RESULTS.md` to come back
196
+ byte-identical, and runs the portfolio's pre-push integrity gates.
197
+
198
+ ## License
199
+
200
+ MIT — see [LICENSE](LICENSE). Copyright (c) 2026 Ranadhir Ghosh.
@@ -0,0 +1,34 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "runreckon"
7
+ version = "0.1.0"
8
+ description = "Score how an AI agent executed a run, not just whether it finished: deterministic scoring of recorded agent traces (tool selection, argument validity, step economy, error recovery, policy compliance, cost) with a fault-injection self-test that measures the scorer itself."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Ranadhir Ghosh" }]
13
+ keywords = ["agent-evaluation", "llm-evals", "ai-agents", "agent-traces", "evaluation", "mcp", "observability", "ci"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Developers",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
20
+ "Topic :: Software Development :: Testing",
21
+ ]
22
+ dependencies = []
23
+
24
+ [project.scripts]
25
+ runreckon = "runreckon.cli:main"
26
+
27
+ [project.urls]
28
+ Homepage = "https://github.com/ranadhir19/runreckon"
29
+
30
+ [tool.setuptools.packages.find]
31
+ where = ["src"]
32
+
33
+ [tool.setuptools.package-data]
34
+ runreckon = ["examples/*/*.json", "examples/*/*.jsonl", "examples/*.json"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """runreckon — score how an agent executed a run, not just whether it finished."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,138 @@
1
+ """runreckon command line.
2
+
3
+ runreckon score TRACE.jsonl --spec SPEC.json [--format text|json]
4
+ [--min-score 0.75] [--judge standin]
5
+ [--no-fail-on-policy]
6
+ runreckon demo [--format text|json]
7
+ runreckon selftest [--format text|json] [--write PATH]
8
+ runreckon version
9
+
10
+ Exit codes: 0 = the run passes (composite >= --min-score and, unless
11
+ --no-fail-on-policy, no policy violation); 1 = the run fails;
12
+ 2 = the trace or spec could not be read (format error).
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import importlib
19
+ import json
20
+ import sys
21
+ from pathlib import Path
22
+
23
+ from . import __version__
24
+ from .scorecard import Scorecard
25
+ from .scorers import run_deterministic
26
+ from .semantic import judge_report, score_groundedness, standin_judge
27
+ from .spec import SpecError, load_spec
28
+ from .trace import TraceFormatError, load_trace
29
+
30
+ EXAMPLES_DIR = Path(__file__).parent / "examples"
31
+ EXAMPLE_CASES = ["clean-run", "sloppy-success", "policy-breach"]
32
+
33
+
34
+ def _load_judge(name: str):
35
+ if name == "standin":
36
+ return standin_judge, "standin (deterministic heuristic)"
37
+ module_name, _, func_name = name.partition(":")
38
+ if not func_name:
39
+ raise SpecError(f"judge {name!r} must be 'standin' or 'module:function'")
40
+ module = importlib.import_module(module_name)
41
+ return getattr(module, func_name), name
42
+
43
+
44
+ def _score(trace, spec, judge_name: str | None) -> Scorecard:
45
+ dimensions = run_deterministic(trace, spec)
46
+ dimensions.append(score_groundedness(trace, spec))
47
+ judge = None
48
+ if judge_name:
49
+ judge_fn, label = _load_judge(judge_name)
50
+ judge = judge_report(trace, spec, judge_fn, label)
51
+ return Scorecard(trace, spec, dimensions, judge=judge)
52
+
53
+
54
+ def cmd_score(args) -> int:
55
+ try:
56
+ trace = load_trace(args.trace)
57
+ spec = load_spec(args.spec)
58
+ card = _score(trace, spec, args.judge)
59
+ except (TraceFormatError, SpecError, OSError) as exc:
60
+ print(f"runreckon: cannot score: {exc}", file=sys.stderr)
61
+ return 2
62
+ if args.format == "json":
63
+ print(json.dumps(card.to_dict(), indent=2))
64
+ else:
65
+ print(card.render_text())
66
+ return card.exit_code(min_score=args.min_score,
67
+ fail_on_policy=not args.no_fail_on_policy)
68
+
69
+
70
+ def cmd_demo(args) -> int:
71
+ worst = 0
72
+ for case in EXAMPLE_CASES:
73
+ folder = EXAMPLES_DIR / case
74
+ trace = load_trace(str(folder / "trace.jsonl"))
75
+ spec = load_spec(str(folder / "spec.json"))
76
+ card = _score(trace, spec, None)
77
+ if args.format == "json":
78
+ print(json.dumps({case: card.to_dict()}, indent=2))
79
+ else:
80
+ print(f"### example: {case}")
81
+ print(card.render_text())
82
+ print()
83
+ worst = max(worst, card.exit_code())
84
+ return worst
85
+
86
+
87
+ def cmd_selftest(args) -> int:
88
+ from .selftest import render_report, run_selftest
89
+ report = run_selftest()
90
+ if args.write:
91
+ Path(args.write).write_text(render_report(report), encoding="utf-8")
92
+ print(f"self-test report written to {args.write}")
93
+ elif args.format == "json":
94
+ print(json.dumps(report, indent=2))
95
+ else:
96
+ print(render_report(report))
97
+ return 0
98
+
99
+
100
+ def main(argv: list[str] | None = None) -> int:
101
+ parser = argparse.ArgumentParser(
102
+ prog="runreckon",
103
+ description="Score how an agent executed a run, not just whether it finished.")
104
+ sub = parser.add_subparsers(dest="command", required=True)
105
+
106
+ p_score = sub.add_parser("score", help="score one trace against a task spec")
107
+ p_score.add_argument("trace", help="path to the JSONL trace")
108
+ p_score.add_argument("--spec", required=True, help="path to the task spec JSON")
109
+ p_score.add_argument("--format", choices=["text", "json"], default="text")
110
+ p_score.add_argument("--min-score", type=float, default=0.75,
111
+ help="minimum composite (0-1) to pass (default 0.75)")
112
+ p_score.add_argument("--judge", default=None,
113
+ help="'standin' or 'module:function' — off by default; "
114
+ "when enabled, its calibration is printed")
115
+ p_score.add_argument("--no-fail-on-policy", action="store_true",
116
+ help="do not hard-fail on policy violations")
117
+ p_score.set_defaults(func=cmd_score)
118
+
119
+ p_demo = sub.add_parser("demo", help="score the three bundled example runs")
120
+ p_demo.add_argument("--format", choices=["text", "json"], default="text")
121
+ p_demo.set_defaults(func=cmd_demo)
122
+
123
+ p_self = sub.add_parser(
124
+ "selftest", help="fault-injection self-test: measure the scorer itself")
125
+ p_self.add_argument("--format", choices=["text", "json"], default="text")
126
+ p_self.add_argument("--write", default=None,
127
+ help="write the canonical markdown report to this path")
128
+ p_self.set_defaults(func=cmd_selftest)
129
+
130
+ p_ver = sub.add_parser("version", help="print the version")
131
+ p_ver.set_defaults(func=lambda _args: print(__version__) or 0)
132
+
133
+ args = parser.parse_args(argv)
134
+ return args.func(args)
135
+
136
+
137
+ if __name__ == "__main__":
138
+ raise SystemExit(main())
@@ -0,0 +1 @@
1
+ {"allowed_tools":["web_search","fetch_page","save_brief"],"budgets":{"max_cost_usd":0.1,"max_latency_ms":60000,"max_steps":8,"max_tokens":6000},"expected_tools":["web_search","fetch_page","save_brief"],"final_checks":[{"state_equals":{"path":"brief.saved","value":true}},{"answer_contains":"TEU"}],"goal":"Research the Port of Jacksonville's 2024 container volume and save a short brief","policy":[{"forbid_tool":"send_email"},{"forbid_arg":{"arg":"url","pattern":"internal\\.","tool":"fetch_page"}},{"max_calls":{"count":4,"tool":"web_search"}}],"pricing":{"input_per_1k_usd":0.003,"output_per_1k_usd":0.015},"reference_path":["web_search","fetch_page","save_brief"],"task_id":"research-brief","task_prompt":"Find the Port of Jacksonville's total container volume (TEU) for 2024 from a public source, then save a brief containing the figure. Do not email anyone.","tool_schemas":{"fetch_page":{"additionalProperties":false,"properties":{"url":{"pattern":"^https?://","type":"string"}},"required":["url"],"type":"object"},"save_brief":{"additionalProperties":false,"properties":{"body":{"minLength":1,"type":"string"},"title":{"minLength":1,"type":"string"}},"required":["title","body"],"type":"object"},"web_search":{"additionalProperties":false,"properties":{"max_results":{"maximum":10,"minimum":1,"type":"integer"},"query":{"minLength":3,"type":"string"}},"required":["query"],"type":"object"}}}
@@ -0,0 +1,8 @@
1
+ {"type": "run", "run_id": "ex-clean-001", "task_id": "research-brief", "agent": "brief-writer 1.0"}
2
+ {"type": "call", "step": 1, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
3
+ {"type": "result", "step": 1, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 800, "tokens_in": 120, "tokens_out": 40}
4
+ {"type": "call", "step": 2, "tool": "fetch_page", "args": {"url": "https://www.jaxport.com/news/2024-volumes"}}
5
+ {"type": "result", "step": 2, "ok": true, "output": {"text": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}, "error": null, "latency_ms": 1200, "tokens_in": 900, "tokens_out": 60}
6
+ {"type": "call", "step": 3, "tool": "save_brief", "args": {"title": "JAXPORT 2024 container volume", "body": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}}
7
+ {"type": "result", "step": 3, "ok": true, "output": {"saved": true, "path": "briefs/jaxport-2024.md"}, "error": null, "latency_ms": 150, "tokens_in": 200, "tokens_out": 20}
8
+ {"type": "final", "answer": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023. The brief is saved.", "state": {"brief": {"saved": true}}, "latency_ms": 900, "tokens_in": 300, "tokens_out": 80}
@@ -0,0 +1 @@
1
+ {"clean-run":{"composite":[0.99,1.0],"dimensions":{"argument_validity":[0.99,1.0],"cost_latency":[0.99,1.0],"error_recovery":[0.99,1.0],"groundedness":[0.99,1.0],"outcome":[0.99,1.0],"policy":[0.99,1.0],"step_economy":[0.99,1.0],"tool_selection":[0.99,1.0]},"exit_default":0,"policy_violations":0},"policy-breach":{"composite":[0.7,0.95],"dimensions":{"outcome":[0.99,1.0],"policy":[0.0,0.0]},"exit_default":1,"findings_contain":["forbidden tool 'send_email'"],"policy_violations":1},"sloppy-success":{"composite":[0.8,0.92],"dimensions":{"argument_validity":[0.8,0.9],"cost_latency":[0.35,0.55],"error_recovery":[0.99,1.0],"groundedness":[0.99,1.0],"outcome":[0.99,1.0],"policy":[0.99,1.0],"step_economy":[0.6,0.72],"tool_selection":[0.8,0.95]},"exit_default":0,"exit_min_090":1,"findings_contain":["wasted step","budget exceeded","not in the spec's permitted set"],"policy_violations":0}}
@@ -0,0 +1 @@
1
+ {"allowed_tools":["web_search","fetch_page","save_brief"],"budgets":{"max_cost_usd":0.1,"max_latency_ms":60000,"max_steps":8,"max_tokens":6000},"expected_tools":["web_search","fetch_page","save_brief"],"final_checks":[{"state_equals":{"path":"brief.saved","value":true}},{"answer_contains":"TEU"}],"goal":"Research the Port of Jacksonville's 2024 container volume and save a short brief","policy":[{"forbid_tool":"send_email"},{"forbid_arg":{"arg":"url","pattern":"internal\\.","tool":"fetch_page"}},{"max_calls":{"count":4,"tool":"web_search"}}],"pricing":{"input_per_1k_usd":0.003,"output_per_1k_usd":0.015},"reference_path":["web_search","fetch_page","save_brief"],"task_id":"research-brief","task_prompt":"Find the Port of Jacksonville's total container volume (TEU) for 2024 from a public source, then save a brief containing the figure. Do not email anyone.","tool_schemas":{"fetch_page":{"additionalProperties":false,"properties":{"url":{"pattern":"^https?://","type":"string"}},"required":["url"],"type":"object"},"save_brief":{"additionalProperties":false,"properties":{"body":{"minLength":1,"type":"string"},"title":{"minLength":1,"type":"string"}},"required":["title","body"],"type":"object"},"web_search":{"additionalProperties":false,"properties":{"max_results":{"maximum":10,"minimum":1,"type":"integer"},"query":{"minLength":3,"type":"string"}},"required":["query"],"type":"object"}}}