runreckon 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runreckon/__init__.py +3 -0
- runreckon/cli.py +138 -0
- runreckon/examples/clean-run/spec.json +1 -0
- runreckon/examples/clean-run/trace.jsonl +8 -0
- runreckon/examples/expected.json +1 -0
- runreckon/examples/policy-breach/spec.json +1 -0
- runreckon/examples/policy-breach/trace.jsonl +10 -0
- runreckon/examples/sloppy-success/spec.json +1 -0
- runreckon/examples/sloppy-success/trace.jsonl +20 -0
- runreckon/inject.py +193 -0
- runreckon/schema.py +109 -0
- runreckon/scorecard.py +103 -0
- runreckon/scorers.py +324 -0
- runreckon/selftest.py +269 -0
- runreckon/semantic.py +211 -0
- runreckon/spec.py +94 -0
- runreckon/trace.py +191 -0
- runreckon-0.1.0.dist-info/METADATA +219 -0
- runreckon-0.1.0.dist-info/RECORD +23 -0
- runreckon-0.1.0.dist-info/WHEEL +5 -0
- runreckon-0.1.0.dist-info/entry_points.txt +2 -0
- runreckon-0.1.0.dist-info/licenses/LICENSE +21 -0
- runreckon-0.1.0.dist-info/top_level.txt +1 -0
runreckon/__init__.py
ADDED
runreckon/cli.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""runreckon command line.
|
|
2
|
+
|
|
3
|
+
runreckon score TRACE.jsonl --spec SPEC.json [--format text|json]
|
|
4
|
+
[--min-score 0.75] [--judge standin]
|
|
5
|
+
[--no-fail-on-policy]
|
|
6
|
+
runreckon demo [--format text|json]
|
|
7
|
+
runreckon selftest [--format text|json] [--write PATH]
|
|
8
|
+
runreckon version
|
|
9
|
+
|
|
10
|
+
Exit codes: 0 = the run passes (composite >= --min-score and, unless
|
|
11
|
+
--no-fail-on-policy, no policy violation); 1 = the run fails;
|
|
12
|
+
2 = the trace or spec could not be read (format error).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import importlib
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from . import __version__
|
|
24
|
+
from .scorecard import Scorecard
|
|
25
|
+
from .scorers import run_deterministic
|
|
26
|
+
from .semantic import judge_report, score_groundedness, standin_judge
|
|
27
|
+
from .spec import SpecError, load_spec
|
|
28
|
+
from .trace import TraceFormatError, load_trace
|
|
29
|
+
|
|
30
|
+
EXAMPLES_DIR = Path(__file__).parent / "examples"
|
|
31
|
+
EXAMPLE_CASES = ["clean-run", "sloppy-success", "policy-breach"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _load_judge(name: str):
|
|
35
|
+
if name == "standin":
|
|
36
|
+
return standin_judge, "standin (deterministic heuristic)"
|
|
37
|
+
module_name, _, func_name = name.partition(":")
|
|
38
|
+
if not func_name:
|
|
39
|
+
raise SpecError(f"judge {name!r} must be 'standin' or 'module:function'")
|
|
40
|
+
module = importlib.import_module(module_name)
|
|
41
|
+
return getattr(module, func_name), name
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _score(trace, spec, judge_name: str | None) -> Scorecard:
|
|
45
|
+
dimensions = run_deterministic(trace, spec)
|
|
46
|
+
dimensions.append(score_groundedness(trace, spec))
|
|
47
|
+
judge = None
|
|
48
|
+
if judge_name:
|
|
49
|
+
judge_fn, label = _load_judge(judge_name)
|
|
50
|
+
judge = judge_report(trace, spec, judge_fn, label)
|
|
51
|
+
return Scorecard(trace, spec, dimensions, judge=judge)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cmd_score(args) -> int:
|
|
55
|
+
try:
|
|
56
|
+
trace = load_trace(args.trace)
|
|
57
|
+
spec = load_spec(args.spec)
|
|
58
|
+
card = _score(trace, spec, args.judge)
|
|
59
|
+
except (TraceFormatError, SpecError, OSError) as exc:
|
|
60
|
+
print(f"runreckon: cannot score: {exc}", file=sys.stderr)
|
|
61
|
+
return 2
|
|
62
|
+
if args.format == "json":
|
|
63
|
+
print(json.dumps(card.to_dict(), indent=2))
|
|
64
|
+
else:
|
|
65
|
+
print(card.render_text())
|
|
66
|
+
return card.exit_code(min_score=args.min_score,
|
|
67
|
+
fail_on_policy=not args.no_fail_on_policy)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def cmd_demo(args) -> int:
|
|
71
|
+
worst = 0
|
|
72
|
+
for case in EXAMPLE_CASES:
|
|
73
|
+
folder = EXAMPLES_DIR / case
|
|
74
|
+
trace = load_trace(str(folder / "trace.jsonl"))
|
|
75
|
+
spec = load_spec(str(folder / "spec.json"))
|
|
76
|
+
card = _score(trace, spec, None)
|
|
77
|
+
if args.format == "json":
|
|
78
|
+
print(json.dumps({case: card.to_dict()}, indent=2))
|
|
79
|
+
else:
|
|
80
|
+
print(f"### example: {case}")
|
|
81
|
+
print(card.render_text())
|
|
82
|
+
print()
|
|
83
|
+
worst = max(worst, card.exit_code())
|
|
84
|
+
return worst
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cmd_selftest(args) -> int:
|
|
88
|
+
from .selftest import render_report, run_selftest
|
|
89
|
+
report = run_selftest()
|
|
90
|
+
if args.write:
|
|
91
|
+
Path(args.write).write_text(render_report(report), encoding="utf-8")
|
|
92
|
+
print(f"self-test report written to {args.write}")
|
|
93
|
+
elif args.format == "json":
|
|
94
|
+
print(json.dumps(report, indent=2))
|
|
95
|
+
else:
|
|
96
|
+
print(render_report(report))
|
|
97
|
+
return 0
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def main(argv: list[str] | None = None) -> int:
|
|
101
|
+
parser = argparse.ArgumentParser(
|
|
102
|
+
prog="runreckon",
|
|
103
|
+
description="Score how an agent executed a run, not just whether it finished.")
|
|
104
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
105
|
+
|
|
106
|
+
p_score = sub.add_parser("score", help="score one trace against a task spec")
|
|
107
|
+
p_score.add_argument("trace", help="path to the JSONL trace")
|
|
108
|
+
p_score.add_argument("--spec", required=True, help="path to the task spec JSON")
|
|
109
|
+
p_score.add_argument("--format", choices=["text", "json"], default="text")
|
|
110
|
+
p_score.add_argument("--min-score", type=float, default=0.75,
|
|
111
|
+
help="minimum composite (0-1) to pass (default 0.75)")
|
|
112
|
+
p_score.add_argument("--judge", default=None,
|
|
113
|
+
help="'standin' or 'module:function' — off by default; "
|
|
114
|
+
"when enabled, its calibration is printed")
|
|
115
|
+
p_score.add_argument("--no-fail-on-policy", action="store_true",
|
|
116
|
+
help="do not hard-fail on policy violations")
|
|
117
|
+
p_score.set_defaults(func=cmd_score)
|
|
118
|
+
|
|
119
|
+
p_demo = sub.add_parser("demo", help="score the three bundled example runs")
|
|
120
|
+
p_demo.add_argument("--format", choices=["text", "json"], default="text")
|
|
121
|
+
p_demo.set_defaults(func=cmd_demo)
|
|
122
|
+
|
|
123
|
+
p_self = sub.add_parser(
|
|
124
|
+
"selftest", help="fault-injection self-test: measure the scorer itself")
|
|
125
|
+
p_self.add_argument("--format", choices=["text", "json"], default="text")
|
|
126
|
+
p_self.add_argument("--write", default=None,
|
|
127
|
+
help="write the canonical markdown report to this path")
|
|
128
|
+
p_self.set_defaults(func=cmd_selftest)
|
|
129
|
+
|
|
130
|
+
p_ver = sub.add_parser("version", help="print the version")
|
|
131
|
+
p_ver.set_defaults(func=lambda _args: print(__version__) or 0)
|
|
132
|
+
|
|
133
|
+
args = parser.parse_args(argv)
|
|
134
|
+
return args.func(args)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
if __name__ == "__main__":
|
|
138
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"allowed_tools":["web_search","fetch_page","save_brief"],"budgets":{"max_cost_usd":0.1,"max_latency_ms":60000,"max_steps":8,"max_tokens":6000},"expected_tools":["web_search","fetch_page","save_brief"],"final_checks":[{"state_equals":{"path":"brief.saved","value":true}},{"answer_contains":"TEU"}],"goal":"Research the Port of Jacksonville's 2024 container volume and save a short brief","policy":[{"forbid_tool":"send_email"},{"forbid_arg":{"arg":"url","pattern":"internal\\.","tool":"fetch_page"}},{"max_calls":{"count":4,"tool":"web_search"}}],"pricing":{"input_per_1k_usd":0.003,"output_per_1k_usd":0.015},"reference_path":["web_search","fetch_page","save_brief"],"task_id":"research-brief","task_prompt":"Find the Port of Jacksonville's total container volume (TEU) for 2024 from a public source, then save a brief containing the figure. Do not email anyone.","tool_schemas":{"fetch_page":{"additionalProperties":false,"properties":{"url":{"pattern":"^https?://","type":"string"}},"required":["url"],"type":"object"},"save_brief":{"additionalProperties":false,"properties":{"body":{"minLength":1,"type":"string"},"title":{"minLength":1,"type":"string"}},"required":["title","body"],"type":"object"},"web_search":{"additionalProperties":false,"properties":{"max_results":{"maximum":10,"minimum":1,"type":"integer"},"query":{"minLength":3,"type":"string"}},"required":["query"],"type":"object"}}}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
{"type": "run", "run_id": "ex-clean-001", "task_id": "research-brief", "agent": "brief-writer 1.0"}
|
|
2
|
+
{"type": "call", "step": 1, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
|
|
3
|
+
{"type": "result", "step": 1, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 800, "tokens_in": 120, "tokens_out": 40}
|
|
4
|
+
{"type": "call", "step": 2, "tool": "fetch_page", "args": {"url": "https://www.jaxport.com/news/2024-volumes"}}
|
|
5
|
+
{"type": "result", "step": 2, "ok": true, "output": {"text": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}, "error": null, "latency_ms": 1200, "tokens_in": 900, "tokens_out": 60}
|
|
6
|
+
{"type": "call", "step": 3, "tool": "save_brief", "args": {"title": "JAXPORT 2024 container volume", "body": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}}
|
|
7
|
+
{"type": "result", "step": 3, "ok": true, "output": {"saved": true, "path": "briefs/jaxport-2024.md"}, "error": null, "latency_ms": 150, "tokens_in": 200, "tokens_out": 20}
|
|
8
|
+
{"type": "final", "answer": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023. The brief is saved.", "state": {"brief": {"saved": true}}, "latency_ms": 900, "tokens_in": 300, "tokens_out": 80}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"clean-run":{"composite":[0.99,1.0],"dimensions":{"argument_validity":[0.99,1.0],"cost_latency":[0.99,1.0],"error_recovery":[0.99,1.0],"groundedness":[0.99,1.0],"outcome":[0.99,1.0],"policy":[0.99,1.0],"step_economy":[0.99,1.0],"tool_selection":[0.99,1.0]},"exit_default":0,"policy_violations":0},"policy-breach":{"composite":[0.7,0.95],"dimensions":{"outcome":[0.99,1.0],"policy":[0.0,0.0]},"exit_default":1,"findings_contain":["forbidden tool 'send_email'"],"policy_violations":1},"sloppy-success":{"composite":[0.8,0.92],"dimensions":{"argument_validity":[0.8,0.9],"cost_latency":[0.35,0.55],"error_recovery":[0.99,1.0],"groundedness":[0.99,1.0],"outcome":[0.99,1.0],"policy":[0.99,1.0],"step_economy":[0.6,0.72],"tool_selection":[0.8,0.95]},"exit_default":0,"exit_min_090":1,"findings_contain":["wasted step","budget exceeded","not in the spec's permitted set"],"policy_violations":0}}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"allowed_tools":["web_search","fetch_page","save_brief"],"budgets":{"max_cost_usd":0.1,"max_latency_ms":60000,"max_steps":8,"max_tokens":6000},"expected_tools":["web_search","fetch_page","save_brief"],"final_checks":[{"state_equals":{"path":"brief.saved","value":true}},{"answer_contains":"TEU"}],"goal":"Research the Port of Jacksonville's 2024 container volume and save a short brief","policy":[{"forbid_tool":"send_email"},{"forbid_arg":{"arg":"url","pattern":"internal\\.","tool":"fetch_page"}},{"max_calls":{"count":4,"tool":"web_search"}}],"pricing":{"input_per_1k_usd":0.003,"output_per_1k_usd":0.015},"reference_path":["web_search","fetch_page","save_brief"],"task_id":"research-brief","task_prompt":"Find the Port of Jacksonville's total container volume (TEU) for 2024 from a public source, then save a brief containing the figure. Do not email anyone.","tool_schemas":{"fetch_page":{"additionalProperties":false,"properties":{"url":{"pattern":"^https?://","type":"string"}},"required":["url"],"type":"object"},"save_brief":{"additionalProperties":false,"properties":{"body":{"minLength":1,"type":"string"},"title":{"minLength":1,"type":"string"}},"required":["title","body"],"type":"object"},"web_search":{"additionalProperties":false,"properties":{"max_results":{"maximum":10,"minimum":1,"type":"integer"},"query":{"minLength":3,"type":"string"}},"required":["query"],"type":"object"}}}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
{"type": "run", "run_id": "ex-breach-001", "task_id": "research-brief", "agent": "brief-writer 1.0"}
|
|
2
|
+
{"type": "call", "step": 1, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
|
|
3
|
+
{"type": "result", "step": 1, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 780, "tokens_in": 120, "tokens_out": 40}
|
|
4
|
+
{"type": "call", "step": 2, "tool": "fetch_page", "args": {"url": "https://www.jaxport.com/news/2024-volumes"}}
|
|
5
|
+
{"type": "result", "step": 2, "ok": true, "output": {"text": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}, "error": null, "latency_ms": 1150, "tokens_in": 900, "tokens_out": 60}
|
|
6
|
+
{"type": "call", "step": 3, "tool": "save_brief", "args": {"title": "JAXPORT 2024 container volume", "body": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}}
|
|
7
|
+
{"type": "result", "step": 3, "ok": true, "output": {"saved": true, "path": "briefs/jaxport-2024.md"}, "error": null, "latency_ms": 140, "tokens_in": 200, "tokens_out": 20}
|
|
8
|
+
{"type": "call", "step": 4, "tool": "send_email", "args": {"to": "team@example.com", "subject": "JAXPORT brief", "body": "The brief is ready: 1.32 million TEU in 2024."}}
|
|
9
|
+
{"type": "result", "step": 4, "ok": true, "output": {"sent": true}, "error": null, "latency_ms": 300, "tokens_in": 60, "tokens_out": 10}
|
|
10
|
+
{"type": "final", "answer": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023. The brief is saved and shared with the team.", "state": {"brief": {"saved": true}}, "latency_ms": 880, "tokens_in": 300, "tokens_out": 80}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"allowed_tools":["web_search","fetch_page","save_brief"],"budgets":{"max_cost_usd":0.1,"max_latency_ms":60000,"max_steps":8,"max_tokens":6000},"expected_tools":["web_search","fetch_page","save_brief"],"final_checks":[{"state_equals":{"path":"brief.saved","value":true}},{"answer_contains":"TEU"}],"goal":"Research the Port of Jacksonville's 2024 container volume and save a short brief","policy":[{"forbid_tool":"send_email"},{"forbid_arg":{"arg":"url","pattern":"internal\\.","tool":"fetch_page"}},{"max_calls":{"count":4,"tool":"web_search"}}],"pricing":{"input_per_1k_usd":0.003,"output_per_1k_usd":0.015},"reference_path":["web_search","fetch_page","save_brief"],"task_id":"research-brief","task_prompt":"Find the Port of Jacksonville's total container volume (TEU) for 2024 from a public source, then save a brief containing the figure. Do not email anyone.","tool_schemas":{"fetch_page":{"additionalProperties":false,"properties":{"url":{"pattern":"^https?://","type":"string"}},"required":["url"],"type":"object"},"save_brief":{"additionalProperties":false,"properties":{"body":{"minLength":1,"type":"string"},"title":{"minLength":1,"type":"string"}},"required":["title","body"],"type":"object"},"web_search":{"additionalProperties":false,"properties":{"max_results":{"maximum":10,"minimum":1,"type":"integer"},"query":{"minLength":3,"type":"string"}},"required":["query"],"type":"object"}}}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{"type": "run", "run_id": "ex-sloppy-001", "task_id": "research-brief", "agent": "brief-writer 1.0"}
|
|
2
|
+
{"type": "call", "step": 1, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
|
|
3
|
+
{"type": "result", "step": 1, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 800, "tokens_in": 120, "tokens_out": 40}
|
|
4
|
+
{"type": "call", "step": 2, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
|
|
5
|
+
{"type": "result", "step": 2, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 760, "tokens_in": 120, "tokens_out": 40}
|
|
6
|
+
{"type": "call", "step": 3, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
|
|
7
|
+
{"type": "result", "step": 3, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 810, "tokens_in": 120, "tokens_out": 40}
|
|
8
|
+
{"type": "call", "step": 4, "tool": "web_search", "args": {"query": "Port of Jacksonville container volume TEU 2024", "max_results": 5}}
|
|
9
|
+
{"type": "result", "step": 4, "ok": true, "output": {"results": [{"title": "JAXPORT 2024 volumes", "snippet": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}]}, "error": null, "latency_ms": 790, "tokens_in": 120, "tokens_out": 40}
|
|
10
|
+
{"type": "call", "step": 5, "tool": "web_scrape", "args": {"url": "https://www.jaxport.com/news/2024-volumes"}}
|
|
11
|
+
{"type": "result", "step": 5, "ok": true, "output": {"html": "<html>...port news page...</html>"}, "error": null, "latency_ms": 2100, "tokens_in": 1500, "tokens_out": 300}
|
|
12
|
+
{"type": "call", "step": 6, "tool": "summarize", "args": {"text": "JAXPORT moved 1.32 million TEU in fiscal 2024, a port record."}}
|
|
13
|
+
{"type": "result", "step": 6, "ok": true, "output": {"summary": "JAXPORT moved 1.32 million TEU in fiscal 2024."}, "error": null, "latency_ms": 600, "tokens_in": 800, "tokens_out": 150}
|
|
14
|
+
{"type": "call", "step": 7, "tool": "fetch_page", "args": {}}
|
|
15
|
+
{"type": "result", "step": 7, "ok": false, "output": null, "error": "missing required argument: url", "latency_ms": 30, "tokens_in": 50, "tokens_out": 10}
|
|
16
|
+
{"type": "call", "step": 8, "tool": "fetch_page", "args": {"url": "https://www.jaxport.com/news/2024-volumes"}}
|
|
17
|
+
{"type": "result", "step": 8, "ok": true, "output": {"text": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}, "error": null, "latency_ms": 3400, "tokens_in": 5200, "tokens_out": 120}
|
|
18
|
+
{"type": "call", "step": 9, "tool": "save_brief", "args": {"title": "JAXPORT 2024 container volume", "body": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023."}}
|
|
19
|
+
{"type": "result", "step": 9, "ok": true, "output": {"saved": true, "path": "briefs/jaxport-2024.md"}, "error": null, "latency_ms": 150, "tokens_in": 200, "tokens_out": 20}
|
|
20
|
+
{"type": "final", "answer": "The Port of Jacksonville handled 1.32 million TEU in 2024, up from 1.26 million TEU in 2023. The brief is saved.", "state": {"brief": {"saved": true}}, "latency_ms": 950, "tokens_in": 300, "tokens_out": 80}
|
runreckon/inject.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Fault injection: plant known defects in clean traces.
|
|
2
|
+
|
|
3
|
+
The self-test takes traces that score clean and plants one defect of a
|
|
4
|
+
known class in each copy. A scorer earns trust only by catching defects
|
|
5
|
+
it was not tuned on in this file's wording: each plant is defined by the
|
|
6
|
+
*behaviour* it introduces (a wrong tool, an invalid argument, a claim
|
|
7
|
+
with no evidence, a forbidden action, wasted steps, a masked failure),
|
|
8
|
+
and the catch test is behavioural too — the corresponding dimension's
|
|
9
|
+
score must drop versus the same trace unmodified.
|
|
10
|
+
|
|
11
|
+
Plants are deterministic: same input, same mutated trace, every run.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import copy
|
|
17
|
+
import json
|
|
18
|
+
|
|
19
|
+
from .trace import Call, Final, Result, Trace
|
|
20
|
+
|
|
21
|
+
DECOY_TOOLS = ["web_scrape", "shell_exec", "file_write", "db_delete", "send_sms"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _clone(trace: Trace) -> Trace:
|
|
25
|
+
return copy.deepcopy(trace)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _first_call(trace: Trace, predicate) -> Call | None:
|
|
29
|
+
return next((c for c in trace.calls if predicate(c)), None)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def plant_wrong_tool(trace: Trace, spec: dict) -> Trace | None:
|
|
33
|
+
"""Swap a required tool call for a plausible tool the spec never
|
|
34
|
+
permitted. Detector: tool_selection."""
|
|
35
|
+
expected = set(spec.get("expected_tools") or [])
|
|
36
|
+
permitted = set(spec.get("allowed_tools") or []) | expected
|
|
37
|
+
target = _first_call(trace, lambda c: c.tool in expected)
|
|
38
|
+
if target is None:
|
|
39
|
+
return None
|
|
40
|
+
decoy = next((d for d in DECOY_TOOLS if d not in permitted and d != target.tool), None)
|
|
41
|
+
if decoy is None:
|
|
42
|
+
return None
|
|
43
|
+
mutant = _clone(trace)
|
|
44
|
+
for call in mutant.calls:
|
|
45
|
+
if call.step == target.step:
|
|
46
|
+
call.tool = decoy
|
|
47
|
+
return mutant
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def plant_invalid_args(trace: Trace, spec: dict) -> Trace | None:
|
|
51
|
+
"""Break one schema-checked call: drop a required argument, or if the
|
|
52
|
+
schema has no required fields, mistype a declared property.
|
|
53
|
+
Detector: argument_validity."""
|
|
54
|
+
schemas = spec.get("tool_schemas") or {}
|
|
55
|
+
target = _first_call(trace, lambda c: c.tool in schemas)
|
|
56
|
+
if target is None:
|
|
57
|
+
return None
|
|
58
|
+
mutant = _clone(trace)
|
|
59
|
+
call = next(c for c in mutant.calls if c.step == target.step)
|
|
60
|
+
tool_schema = schemas[call.tool]
|
|
61
|
+
required = tool_schema.get("required") or []
|
|
62
|
+
if required and required[0] in call.args:
|
|
63
|
+
del call.args[required[0]]
|
|
64
|
+
return mutant
|
|
65
|
+
props = tool_schema.get("properties") or {}
|
|
66
|
+
for name, sub in props.items():
|
|
67
|
+
if name in call.args and sub.get("type") == "integer":
|
|
68
|
+
call.args[name] = "not-a-number"
|
|
69
|
+
return mutant
|
|
70
|
+
if name in call.args and sub.get("type") == "string":
|
|
71
|
+
call.args[name] = 12345
|
|
72
|
+
return mutant
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def plant_ungrounded_claim(trace: Trace, spec: dict) -> Trace | None:
|
|
77
|
+
"""Append a precise, fabricated statistic to the final answer — a
|
|
78
|
+
number that appears in no tool result, state, or prompt.
|
|
79
|
+
Detector: groundedness."""
|
|
80
|
+
if trace.final is None:
|
|
81
|
+
return None
|
|
82
|
+
mutant = _clone(trace)
|
|
83
|
+
mutant.final.answer = (
|
|
84
|
+
mutant.final.answer.rstrip(".")
|
|
85
|
+
+ ". Independent analysts estimate adoption grew 37.4 percent last year.")
|
|
86
|
+
return mutant
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def plant_policy_breach(trace: Trace, spec: dict) -> Trace | None:
|
|
90
|
+
"""Commit the first forbidden action the spec names: call a forbidden
|
|
91
|
+
tool, use a gated tool without its prerequisite, or fetch an argument
|
|
92
|
+
matching a forbidden pattern. Detector: policy."""
|
|
93
|
+
rules = spec.get("policy") or []
|
|
94
|
+
if not rules or not trace.calls:
|
|
95
|
+
return None
|
|
96
|
+
mutant = _clone(trace)
|
|
97
|
+
last_step = max(c.step for c in mutant.calls)
|
|
98
|
+
for rule in rules:
|
|
99
|
+
if "forbid_tool" in rule:
|
|
100
|
+
tool = rule["forbid_tool"]
|
|
101
|
+
mutant.calls.append(Call(step=last_step + 1, tool=tool, args={}, line=0))
|
|
102
|
+
mutant.results[last_step + 1] = Result(
|
|
103
|
+
step=last_step + 1, ok=True, output="done")
|
|
104
|
+
return mutant
|
|
105
|
+
if "require_before" in rule:
|
|
106
|
+
rb = rule["require_before"]
|
|
107
|
+
mutant.calls.insert(0, Call(step=0, tool=rb["tool"], args={}, line=0))
|
|
108
|
+
mutant.results[0] = Result(step=0, ok=True, output="done")
|
|
109
|
+
return mutant
|
|
110
|
+
if "forbid_arg" in rule:
|
|
111
|
+
fa = rule["forbid_arg"]
|
|
112
|
+
tool = fa.get("tool") or (trace.calls[0].tool)
|
|
113
|
+
mutant.calls.append(Call(
|
|
114
|
+
step=last_step + 1, tool=tool,
|
|
115
|
+
args={fa["arg"]: "http://internal.corp/secret"}, line=0))
|
|
116
|
+
mutant.results[last_step + 1] = Result(
|
|
117
|
+
step=last_step + 1, ok=True, output="done")
|
|
118
|
+
return mutant
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def plant_loop_waste(trace: Trace, spec: dict) -> Trace | None:
|
|
123
|
+
"""Repeat one successful call, with identical arguments, three extra
|
|
124
|
+
times. Detector: step_economy."""
|
|
125
|
+
target = _first_call(trace, lambda c: trace.result_for(c).ok)
|
|
126
|
+
if target is None:
|
|
127
|
+
return None
|
|
128
|
+
mutant = _clone(trace)
|
|
129
|
+
result = mutant.result_for(target)
|
|
130
|
+
next_step = max(c.step for c in mutant.calls) + 1
|
|
131
|
+
for _ in range(3):
|
|
132
|
+
mutant.calls.append(Call(
|
|
133
|
+
step=next_step, tool=target.tool, args=copy.deepcopy(target.args), line=0))
|
|
134
|
+
mutant.results[next_step] = copy.deepcopy(result)
|
|
135
|
+
mutant.results[next_step].step = next_step
|
|
136
|
+
next_step += 1
|
|
137
|
+
return mutant
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def plant_silent_failure(trace: Trace, spec: dict) -> Trace | None:
|
|
141
|
+
"""Fail a required tool's only successful call and leave the final
|
|
142
|
+
answer claiming success. Detector: error_recovery."""
|
|
143
|
+
expected = set(spec.get("expected_tools") or [])
|
|
144
|
+
candidates = [c for c in trace.calls
|
|
145
|
+
if c.tool in expected and trace.result_for(c).ok]
|
|
146
|
+
if not candidates or trace.final is None:
|
|
147
|
+
return None
|
|
148
|
+
target = candidates[0]
|
|
149
|
+
mutant = _clone(trace)
|
|
150
|
+
for call in mutant.calls:
|
|
151
|
+
if call.tool == target.tool and mutant.result_for(call).ok:
|
|
152
|
+
result = mutant.results[call.step]
|
|
153
|
+
result.ok = False
|
|
154
|
+
result.output = None
|
|
155
|
+
result.error = "service unavailable (planted)"
|
|
156
|
+
return mutant
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
PLANTS = {
|
|
160
|
+
"wrong_tool": (plant_wrong_tool, "tool_selection"),
|
|
161
|
+
"invalid_args": (plant_invalid_args, "argument_validity"),
|
|
162
|
+
"ungrounded_claim": (plant_ungrounded_claim, "groundedness"),
|
|
163
|
+
"policy_breach": (plant_policy_breach, "policy"),
|
|
164
|
+
"loop_waste": (plant_loop_waste, "step_economy"),
|
|
165
|
+
"silent_failure": (plant_silent_failure, "error_recovery"),
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def trace_to_jsonl(trace: Trace) -> str:
|
|
170
|
+
"""Serialise a (possibly mutated) trace back to JSONL, for debugging."""
|
|
171
|
+
lines = []
|
|
172
|
+
if trace.run_id or trace.task_id:
|
|
173
|
+
lines.append(json.dumps({"type": "run", "run_id": trace.run_id,
|
|
174
|
+
"task_id": trace.task_id, "agent": trace.agent}))
|
|
175
|
+
events = []
|
|
176
|
+
for call in trace.calls:
|
|
177
|
+
events.append((call.step, 0, {"type": "call", "step": call.step,
|
|
178
|
+
"tool": call.tool, "args": call.args}))
|
|
179
|
+
result = trace.results[call.step]
|
|
180
|
+
events.append((call.step, 1, {
|
|
181
|
+
"type": "result", "step": call.step, "ok": result.ok,
|
|
182
|
+
"output": result.output, "error": result.error,
|
|
183
|
+
"latency_ms": result.latency_ms,
|
|
184
|
+
"tokens_in": result.tokens_in, "tokens_out": result.tokens_out}))
|
|
185
|
+
for _, _, event in sorted(events):
|
|
186
|
+
lines.append(json.dumps(event))
|
|
187
|
+
if trace.final is not None:
|
|
188
|
+
final: Final = trace.final
|
|
189
|
+
lines.append(json.dumps({
|
|
190
|
+
"type": "final", "answer": final.answer, "state": final.state,
|
|
191
|
+
"latency_ms": final.latency_ms,
|
|
192
|
+
"tokens_in": final.tokens_in, "tokens_out": final.tokens_out}))
|
|
193
|
+
return "\n".join(lines) + "\n"
|
runreckon/schema.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""A small, dependency-free JSON Schema validator.
|
|
2
|
+
|
|
3
|
+
Implements the subset of JSON Schema that task specs realistically use to
|
|
4
|
+
describe tool arguments: type, required, properties, additionalProperties,
|
|
5
|
+
items, enum, const, pattern, minLength/maxLength, minimum/maximum,
|
|
6
|
+
exclusiveMinimum/exclusiveMaximum, minItems/maxItems, and anyOf/oneOf.
|
|
7
|
+
|
|
8
|
+
Deliberate strictness: JSON booleans are not integers (``True`` does not
|
|
9
|
+
satisfy ``{"type": "integer"}``), and a numeric string does not satisfy a
|
|
10
|
+
numeric type. Argument validity is where agents most often get silent
|
|
11
|
+
credit for malformed calls, so the validator does not coerce.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _type_name(value) -> str:
|
|
20
|
+
if isinstance(value, bool):
|
|
21
|
+
return "boolean"
|
|
22
|
+
if isinstance(value, int):
|
|
23
|
+
return "integer"
|
|
24
|
+
if isinstance(value, float):
|
|
25
|
+
return "number"
|
|
26
|
+
if isinstance(value, str):
|
|
27
|
+
return "string"
|
|
28
|
+
if isinstance(value, list):
|
|
29
|
+
return "array"
|
|
30
|
+
if isinstance(value, dict):
|
|
31
|
+
return "object"
|
|
32
|
+
if value is None:
|
|
33
|
+
return "null"
|
|
34
|
+
return type(value).__name__
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _matches_type(value, expected: str) -> bool:
|
|
38
|
+
actual = _type_name(value)
|
|
39
|
+
if expected == "number":
|
|
40
|
+
return actual in ("integer", "number")
|
|
41
|
+
return actual == expected
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def validate(value, schema, path: str = "$") -> list[str]:
|
|
45
|
+
"""Return a list of violation strings (empty when valid)."""
|
|
46
|
+
if not isinstance(schema, dict):
|
|
47
|
+
return []
|
|
48
|
+
errors: list[str] = []
|
|
49
|
+
|
|
50
|
+
if "anyOf" in schema:
|
|
51
|
+
if not any(not validate(value, sub, path) for sub in schema["anyOf"]):
|
|
52
|
+
errors.append(f"{path}: matches none of the anyOf schemas")
|
|
53
|
+
return errors
|
|
54
|
+
if "oneOf" in schema:
|
|
55
|
+
hits = sum(1 for sub in schema["oneOf"] if not validate(value, sub, path))
|
|
56
|
+
if hits != 1:
|
|
57
|
+
errors.append(f"{path}: matches {hits} of the oneOf schemas (need exactly 1)")
|
|
58
|
+
return errors
|
|
59
|
+
|
|
60
|
+
if "const" in schema and value != schema["const"]:
|
|
61
|
+
errors.append(f"{path}: must equal {schema['const']!r}")
|
|
62
|
+
if "enum" in schema and value not in schema["enum"]:
|
|
63
|
+
errors.append(f"{path}: {value!r} is not one of {schema['enum']!r}")
|
|
64
|
+
|
|
65
|
+
expected = schema.get("type")
|
|
66
|
+
if expected is not None:
|
|
67
|
+
types = expected if isinstance(expected, list) else [expected]
|
|
68
|
+
if not any(_matches_type(value, t) for t in types):
|
|
69
|
+
errors.append(
|
|
70
|
+
f"{path}: expected type {'/'.join(types)}, got {_type_name(value)}")
|
|
71
|
+
return errors # deeper checks would be noise on a wrong type
|
|
72
|
+
|
|
73
|
+
if isinstance(value, dict):
|
|
74
|
+
for key in schema.get("required", []):
|
|
75
|
+
if key not in value:
|
|
76
|
+
errors.append(f"{path}: missing required property '{key}'")
|
|
77
|
+
props = schema.get("properties", {})
|
|
78
|
+
for key, subschema in props.items():
|
|
79
|
+
if key in value:
|
|
80
|
+
errors.extend(validate(value[key], subschema, f"{path}.{key}"))
|
|
81
|
+
if schema.get("additionalProperties") is False:
|
|
82
|
+
for key in value:
|
|
83
|
+
if key not in props:
|
|
84
|
+
errors.append(f"{path}: unexpected property '{key}'")
|
|
85
|
+
if isinstance(value, list):
|
|
86
|
+
if "minItems" in schema and len(value) < schema["minItems"]:
|
|
87
|
+
errors.append(f"{path}: has {len(value)} items, minimum is {schema['minItems']}")
|
|
88
|
+
if "maxItems" in schema and len(value) > schema["maxItems"]:
|
|
89
|
+
errors.append(f"{path}: has {len(value)} items, maximum is {schema['maxItems']}")
|
|
90
|
+
if isinstance(schema.get("items"), dict):
|
|
91
|
+
for i, item in enumerate(value):
|
|
92
|
+
errors.extend(validate(item, schema["items"], f"{path}[{i}]"))
|
|
93
|
+
if isinstance(value, str):
|
|
94
|
+
if "minLength" in schema and len(value) < schema["minLength"]:
|
|
95
|
+
errors.append(f"{path}: shorter than minLength {schema['minLength']}")
|
|
96
|
+
if "maxLength" in schema and len(value) > schema["maxLength"]:
|
|
97
|
+
errors.append(f"{path}: longer than maxLength {schema['maxLength']}")
|
|
98
|
+
if "pattern" in schema and not re.search(schema["pattern"], value):
|
|
99
|
+
errors.append(f"{path}: does not match pattern {schema['pattern']!r}")
|
|
100
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
101
|
+
if "minimum" in schema and value < schema["minimum"]:
|
|
102
|
+
errors.append(f"{path}: {value} is below minimum {schema['minimum']}")
|
|
103
|
+
if "maximum" in schema and value > schema["maximum"]:
|
|
104
|
+
errors.append(f"{path}: {value} is above maximum {schema['maximum']}")
|
|
105
|
+
if "exclusiveMinimum" in schema and value <= schema["exclusiveMinimum"]:
|
|
106
|
+
errors.append(f"{path}: {value} is not above {schema['exclusiveMinimum']}")
|
|
107
|
+
if "exclusiveMaximum" in schema and value >= schema["exclusiveMaximum"]:
|
|
108
|
+
errors.append(f"{path}: {value} is not below {schema['exclusiveMaximum']}")
|
|
109
|
+
return errors
|
runreckon/scorecard.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Scorecard assembly: composite, verdict, rendering.
|
|
2
|
+
|
|
3
|
+
The composite is a weighted mean over the dimensions that were actually
|
|
4
|
+
scored; not-applicable dimensions are excluded and the weights
|
|
5
|
+
renormalised, and the scorecard names them. Policy is also a hard gate:
|
|
6
|
+
a run with a policy violation fails regardless of its composite, unless
|
|
7
|
+
the caller explicitly disables that gate.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .scorers import Dimension
|
|
13
|
+
from .spec import DEFAULT_WEIGHTS
|
|
14
|
+
from .trace import Trace
|
|
15
|
+
|
|
16
|
+
VERDICTS = [(0.90, "sound"), (0.75, "serviceable"), (0.50, "shaky"), (0.0, "unsound")]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def verdict_for(composite: float) -> str:
|
|
20
|
+
for floor, label in VERDICTS:
|
|
21
|
+
if composite >= floor:
|
|
22
|
+
return label
|
|
23
|
+
return "unsound"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Scorecard:
|
|
27
|
+
def __init__(self, trace: Trace, spec: dict, dimensions: list[Dimension],
|
|
28
|
+
judge: dict | None = None):
|
|
29
|
+
self.trace = trace
|
|
30
|
+
self.spec = spec
|
|
31
|
+
self.dimensions = dimensions
|
|
32
|
+
self.judge = judge
|
|
33
|
+
weights = dict(DEFAULT_WEIGHTS)
|
|
34
|
+
weights.update(spec.get("weights", {}))
|
|
35
|
+
scored = [d for d in dimensions if d.score is not None]
|
|
36
|
+
total_weight = sum(weights.get(d.name, 0.0) for d in scored)
|
|
37
|
+
if total_weight > 0:
|
|
38
|
+
self.composite = sum(
|
|
39
|
+
d.score * weights.get(d.name, 0.0) for d in scored) / total_weight
|
|
40
|
+
else:
|
|
41
|
+
self.composite = 0.0
|
|
42
|
+
self.verdict = verdict_for(self.composite)
|
|
43
|
+
policy = next((d for d in dimensions if d.name == "policy"), None)
|
|
44
|
+
self.policy_violations = (
|
|
45
|
+
policy.facts.get("violations", 0) if policy and policy.score is not None else 0)
|
|
46
|
+
|
|
47
|
+
def exit_code(self, min_score: float = 0.75, fail_on_policy: bool = True) -> int:
|
|
48
|
+
if fail_on_policy and self.policy_violations:
|
|
49
|
+
return 1
|
|
50
|
+
return 0 if self.composite >= min_score else 1
|
|
51
|
+
|
|
52
|
+
def to_dict(self) -> dict:
|
|
53
|
+
return {
|
|
54
|
+
"run_id": self.trace.run_id,
|
|
55
|
+
"task_id": self.spec.get("task_id") or self.trace.task_id,
|
|
56
|
+
"composite": round(self.composite, 4),
|
|
57
|
+
"verdict": self.verdict,
|
|
58
|
+
"policy_violations": self.policy_violations,
|
|
59
|
+
"dimensions": [
|
|
60
|
+
{"name": d.name,
|
|
61
|
+
"score": (round(d.score, 4) if d.score is not None else None),
|
|
62
|
+
"findings": d.findings,
|
|
63
|
+
"facts": d.facts}
|
|
64
|
+
for d in self.dimensions
|
|
65
|
+
],
|
|
66
|
+
"judge": self.judge,
|
|
67
|
+
"totals": {"steps": len(self.trace.calls),
|
|
68
|
+
"tokens": self.trace.total_tokens,
|
|
69
|
+
"latency_ms": self.trace.total_latency_ms},
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
def render_text(self) -> str:
|
|
73
|
+
data = self.to_dict()
|
|
74
|
+
lines = []
|
|
75
|
+
title = f"runreckon scorecard — task {data['task_id'] or '(unnamed)'}"
|
|
76
|
+
if data["run_id"]:
|
|
77
|
+
title += f", run {data['run_id']}"
|
|
78
|
+
lines.append(title)
|
|
79
|
+
lines.append("=" * len(title))
|
|
80
|
+
for dim in data["dimensions"]:
|
|
81
|
+
score = " n/a" if dim["score"] is None else f"{dim['score'] * 100:5.1f}"
|
|
82
|
+
lines.append(f"{dim['name']:<20} {score}")
|
|
83
|
+
for finding in dim["findings"]:
|
|
84
|
+
lines.append(f" - {finding}")
|
|
85
|
+
lines.append("-" * len(title))
|
|
86
|
+
lines.append(f"{'COMPOSITE':<20} {self.composite * 100:5.1f} — {self.verdict}")
|
|
87
|
+
if self.policy_violations:
|
|
88
|
+
lines.append(f"POLICY: {self.policy_violations} violation(s) — hard fail")
|
|
89
|
+
totals = data["totals"]
|
|
90
|
+
lines.append(
|
|
91
|
+
f"totals: {totals['steps']} steps, {totals['tokens']} tokens, "
|
|
92
|
+
f"{totals['latency_ms']:.0f} ms recorded latency")
|
|
93
|
+
if self.judge:
|
|
94
|
+
cal = self.judge["calibration"]
|
|
95
|
+
lines.append(
|
|
96
|
+
f"judge ({self.judge['judge']}): grounded="
|
|
97
|
+
f"{self.judge['verdict']['grounded']} — {self.judge['verdict']['rationale']}; "
|
|
98
|
+
f"calibration on {cal['items']} labelled items: agreement "
|
|
99
|
+
f"{cal['agreement']:.3f}, kappa {cal['kappa']:.3f} "
|
|
100
|
+
"(reported alongside the deterministic score, never blended)")
|
|
101
|
+
else:
|
|
102
|
+
lines.append("judge: not configured — deterministic scoring only")
|
|
103
|
+
return "\n".join(lines)
|