control-expert-mcp 0.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- control_expert_mcp/__init__.py +5 -0
- control_expert_mcp/__main__.py +3 -0
- control_expert_mcp/benchmark/__init__.py +1 -0
- control_expert_mcp/benchmark/alarms.py +221 -0
- control_expert_mcp/benchmark/archive.py +273 -0
- control_expert_mcp/benchmark/cli.py +163 -0
- control_expert_mcp/benchmark/core.py +239 -0
- control_expert_mcp/benchmark/opc.py +329 -0
- control_expert_mcp/benchmark/plot.py +139 -0
- control_expert_mcp/benchmark/provider.py +227 -0
- control_expert_mcp/benchmark/regrade.py +58 -0
- control_expert_mcp/benchmark/report.py +140 -0
- control_expert_mcp/benchmark/runner.py +349 -0
- control_expert_mcp/benchmark/tasks.py +220 -0
- control_expert_mcp/benchmark/transport.py +181 -0
- control_expert_mcp/bridge.py +2952 -0
- control_expert_mcp/constants.py +118 -0
- control_expert_mcp/diagnostics.py +60 -0
- control_expert_mcp/documentation.py +374 -0
- control_expert_mcp/lang_reference.py +411 -0
- control_expert_mcp/live_access.py +59 -0
- control_expert_mcp/modbus.py +203 -0
- control_expert_mcp/opc.py +117 -0
- control_expert_mcp/opc_worker.py +196 -0
- control_expert_mcp/prompts.py +293 -0
- control_expert_mcp/schema.py +207 -0
- control_expert_mcp/server.py +1229 -0
- control_expert_mcp-0.9.1.dist-info/METADATA +474 -0
- control_expert_mcp-0.9.1.dist-info/RECORD +32 -0
- control_expert_mcp-0.9.1.dist-info/WHEEL +5 -0
- control_expert_mcp-0.9.1.dist-info/entry_points.txt +3 -0
- control_expert_mcp-0.9.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Local, opt-in API benchmarks. No COM or provider imports at package import."""
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Restricted offline ST transition proof for the demo-derived alarm task.
|
|
2
|
+
|
|
3
|
+
Only Boolean assignments/IFs and comparisons of the converted INT temperature
|
|
4
|
+
with constants are supported. No PLC execution or ALRM_DIA emulation is claimed.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import math
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
FLAGS = {"temp_fault_1", "temp_fault_2"}
|
|
12
|
+
BOOLS = FLAGS | {"start_process"}
|
|
13
|
+
TOKEN = re.compile(r"\s+|[A-Za-z_][A-Za-z_0-9]*|\d+(?:\.\d+)?|:=|<=|>=|<>|[();=<>+\-,]")
|
|
14
|
+
|
|
15
|
+
CANONICAL = """
|
|
16
|
+
(* Demo-derived training extension: hysteretic temperature alarms. *)
|
|
17
|
+
IF NOT Start_process THEN
|
|
18
|
+
Temp_fault_1 := FALSE;
|
|
19
|
+
Temp_fault_2 := FALSE;
|
|
20
|
+
ELSE
|
|
21
|
+
IF Cmeasured_temp <= 200.0 THEN
|
|
22
|
+
Temp_fault_1 := TRUE;
|
|
23
|
+
ELSIF Cmeasured_temp >= 205.0 THEN
|
|
24
|
+
Temp_fault_1 := FALSE;
|
|
25
|
+
END_IF;
|
|
26
|
+
IF Cmeasured_temp >= 400.0 THEN
|
|
27
|
+
Temp_fault_2 := TRUE;
|
|
28
|
+
ELSIF Cmeasured_temp <= 395.0 THEN
|
|
29
|
+
Temp_fault_2 := FALSE;
|
|
30
|
+
END_IF;
|
|
31
|
+
END_IF;
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def tokens(text: str) -> list[str]:
|
|
36
|
+
"""Tokenize without silently discarding unsupported operators or syntax."""
|
|
37
|
+
text = re.sub(r"\(\*.*?\*\)|//[^\n]*", " ", text, flags=re.S)
|
|
38
|
+
result, position = [], 0
|
|
39
|
+
for match in TOKEN.finditer(text):
|
|
40
|
+
if match.start() != position:
|
|
41
|
+
raise ValueError("Unsupported ST token")
|
|
42
|
+
position = match.end()
|
|
43
|
+
if not match[0].isspace():
|
|
44
|
+
result.append(match[0].lower())
|
|
45
|
+
if position != len(text):
|
|
46
|
+
raise ValueError("Unsupported ST suffix")
|
|
47
|
+
return result
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Parser:
|
|
51
|
+
"""Small typed grammar, deliberately excluding arithmetic, calls and loops."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, values: list[str]):
|
|
54
|
+
self.values, self.position, self.constants = values, 0, set()
|
|
55
|
+
|
|
56
|
+
def peek(self):
|
|
57
|
+
return self.values[self.position] if self.position < len(self.values) else ""
|
|
58
|
+
|
|
59
|
+
def take(self, expected=None):
|
|
60
|
+
value = self.peek()
|
|
61
|
+
if not value or (expected is not None and value != expected):
|
|
62
|
+
raise ValueError("Unsupported ST structure")
|
|
63
|
+
self.position += 1
|
|
64
|
+
return value
|
|
65
|
+
|
|
66
|
+
def expression(self, precedence=0):
|
|
67
|
+
operators = {"or": 1, "and": 2, "=": 3, "<>": 3,
|
|
68
|
+
"<": 3, ">": 3, "<=": 3, ">=": 3}
|
|
69
|
+
value = self.take()
|
|
70
|
+
if value == "not":
|
|
71
|
+
node = ("not", self.expression(4))
|
|
72
|
+
elif value == "(":
|
|
73
|
+
node = self.expression()
|
|
74
|
+
self.take(")")
|
|
75
|
+
elif value in {"true", "false"}:
|
|
76
|
+
node = ("bool", value == "true")
|
|
77
|
+
elif value in BOOLS or value == "cmeasured_temp":
|
|
78
|
+
node = ("input", value)
|
|
79
|
+
else:
|
|
80
|
+
sign = -1 if value == "-" else 1
|
|
81
|
+
if value in {"-", "+"}:
|
|
82
|
+
value = self.take()
|
|
83
|
+
if not re.fullmatch(r"\d+(?:\.\d+)?", value):
|
|
84
|
+
raise ValueError("Unsupported alarm expression")
|
|
85
|
+
number = sign * float(value)
|
|
86
|
+
if not math.isfinite(number) or abs(number) > 32768:
|
|
87
|
+
raise ValueError("Alarm constant outside supported range")
|
|
88
|
+
self.constants.add(number)
|
|
89
|
+
node = ("number", number)
|
|
90
|
+
while operators.get(self.peek(), -1) >= precedence:
|
|
91
|
+
operator = self.take()
|
|
92
|
+
node = (operator, node, self.expression(operators[operator] + 1))
|
|
93
|
+
return node
|
|
94
|
+
|
|
95
|
+
def block(self):
|
|
96
|
+
statements = []
|
|
97
|
+
while self.peek() and self.peek() not in {"else", "elsif", "end_if"}:
|
|
98
|
+
if self.peek() == "if":
|
|
99
|
+
self.take("if")
|
|
100
|
+
branches = []
|
|
101
|
+
while True:
|
|
102
|
+
condition = self.expression()
|
|
103
|
+
require_boolean(condition)
|
|
104
|
+
self.take("then")
|
|
105
|
+
branches.append((condition, self.block()))
|
|
106
|
+
if self.peek() != "elsif":
|
|
107
|
+
break
|
|
108
|
+
self.take("elsif")
|
|
109
|
+
otherwise = []
|
|
110
|
+
if self.peek() == "else":
|
|
111
|
+
self.take("else")
|
|
112
|
+
otherwise = self.block()
|
|
113
|
+
self.take("end_if")
|
|
114
|
+
self.take(";")
|
|
115
|
+
statements.append(("if", branches, otherwise))
|
|
116
|
+
else:
|
|
117
|
+
target = self.take()
|
|
118
|
+
if target not in FLAGS:
|
|
119
|
+
raise ValueError("Alarm extension may only assign the two fault flags")
|
|
120
|
+
self.take(":=")
|
|
121
|
+
expression = self.expression()
|
|
122
|
+
require_boolean(expression)
|
|
123
|
+
self.take(";")
|
|
124
|
+
statements.append(("assign", target, expression))
|
|
125
|
+
return statements
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def expression_type(node) -> str:
|
|
129
|
+
op = node[0]
|
|
130
|
+
if op in {"bool", "number"}:
|
|
131
|
+
return op
|
|
132
|
+
if op == "input":
|
|
133
|
+
return "number" if node[1] == "cmeasured_temp" else "bool"
|
|
134
|
+
if op == "not":
|
|
135
|
+
require_boolean(node[1])
|
|
136
|
+
elif op in {"and", "or"}:
|
|
137
|
+
require_boolean(node[1])
|
|
138
|
+
require_boolean(node[2])
|
|
139
|
+
elif expression_type(node[1]) != "number" or expression_type(node[2]) != "number":
|
|
140
|
+
raise ValueError("Only numeric comparisons are supported")
|
|
141
|
+
return "bool"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def require_boolean(node):
|
|
145
|
+
if expression_type(node) != "bool":
|
|
146
|
+
raise ValueError("Expected Boolean alarm expression")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def value(node, state):
|
|
150
|
+
op = node[0]
|
|
151
|
+
if op in {"bool", "number"}:
|
|
152
|
+
return node[1]
|
|
153
|
+
if op == "input":
|
|
154
|
+
return state[node[1]]
|
|
155
|
+
left = value(node[1], state)
|
|
156
|
+
if op == "not":
|
|
157
|
+
return not left
|
|
158
|
+
right = value(node[2], state)
|
|
159
|
+
return {"and": lambda: left and right, "or": lambda: left or right,
|
|
160
|
+
"=": lambda: left == right, "<>": lambda: left != right,
|
|
161
|
+
"<": lambda: left < right, ">": lambda: left > right,
|
|
162
|
+
"<=": lambda: left <= right, ">=": lambda: left >= right}[op]()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def execute(statements, state):
|
|
166
|
+
for statement in statements:
|
|
167
|
+
if statement[0] == "assign":
|
|
168
|
+
state[statement[1]] = value(statement[2], state)
|
|
169
|
+
else:
|
|
170
|
+
for condition, branch in statement[1]:
|
|
171
|
+
if value(condition, state):
|
|
172
|
+
execute(branch, state)
|
|
173
|
+
break
|
|
174
|
+
else:
|
|
175
|
+
execute(statement[2], state)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def alarm_grade(baseline: str, actual: str) -> dict:
|
|
179
|
+
"""Prove every supported INT-input transition by constant partitions.
|
|
180
|
+
|
|
181
|
+
With no arithmetic, each numeric comparison is constant between its literal
|
|
182
|
+
boundaries. Checking boundary neighbors covers every INT input; enumerating
|
|
183
|
+
both previous flags and enable states proves the one-scan transition table.
|
|
184
|
+
"""
|
|
185
|
+
result = {"rubric": "feeder-alarms-1", "scope": "unverified",
|
|
186
|
+
"static_behavior": "unverified", "runtime_behavior": "unverified",
|
|
187
|
+
"method": "restricted ST interpreter + exhaustive INT comparison partitions; no PLC execution"}
|
|
188
|
+
try:
|
|
189
|
+
original, candidate = tokens(baseline), tokens(actual)
|
|
190
|
+
if candidate[:len(original)] != original:
|
|
191
|
+
return result
|
|
192
|
+
parser = Parser(candidate[len(original):])
|
|
193
|
+
statements = parser.block()
|
|
194
|
+
if parser.position != len(parser.values) or not statements:
|
|
195
|
+
return result
|
|
196
|
+
result["scope"] = "pass"
|
|
197
|
+
points = {-32768, 32767}
|
|
198
|
+
for constant in parser.constants | {200, 205, 395, 400}:
|
|
199
|
+
for point in (math.floor(constant), math.ceil(constant)):
|
|
200
|
+
points.update(x for x in (point - 1, point, point + 1) if -32768 <= x <= 32767)
|
|
201
|
+
checked = 0
|
|
202
|
+
for measured in sorted(points):
|
|
203
|
+
for enabled in (False, True):
|
|
204
|
+
for low in (False, True):
|
|
205
|
+
for high in (False, True):
|
|
206
|
+
state = {"cmeasured_temp": float(measured), "start_process": enabled,
|
|
207
|
+
"temp_fault_1": low, "temp_fault_2": high}
|
|
208
|
+
execute(statements, state)
|
|
209
|
+
expected_low = enabled and (measured <= 200 or (low and measured < 205))
|
|
210
|
+
expected_high = enabled and (measured >= 400 or (high and measured > 395))
|
|
211
|
+
checked += 1
|
|
212
|
+
if (state["temp_fault_1"], state["temp_fault_2"]) != (expected_low, expected_high):
|
|
213
|
+
result["counterexample"] = {"measured": measured, "enabled": enabled,
|
|
214
|
+
"prior_low": low, "prior_high": high,
|
|
215
|
+
"expected_low": expected_low, "expected_high": expected_high,
|
|
216
|
+
"actual_low": state["temp_fault_1"], "actual_high": state["temp_fault_2"]}
|
|
217
|
+
return result
|
|
218
|
+
result.update(static_behavior="pass", partition_cases=checked, int_domain=[-32768, 32767])
|
|
219
|
+
except (ValueError, RecursionError):
|
|
220
|
+
pass # Unsupported syntax stays unverified; never silently accept it.
|
|
221
|
+
return result
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"""Append-only, allowlisted benchmark evidence suitable for a Git repository."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import subprocess
|
|
8
|
+
import uuid
|
|
9
|
+
|
|
10
|
+
from .core import digest, read_json, write_json
|
|
11
|
+
from .report import aggregate, distribution
|
|
12
|
+
from .provider import Usage
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def source_identity(value: dict) -> dict:
|
|
16
|
+
return {key: value.get(key) for key in ("revision", "dirty", "dirty_fingerprint")}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def summarize_validation(value: dict) -> dict:
|
|
20
|
+
result = {key: value[key] for key in (
|
|
21
|
+
"rubric", "answer_pass", "scope", "static_behavior", "runtime_behavior", "compile_pass",
|
|
22
|
+
"declarations_pass", "model_built", "model_saved", "success", "obtained_source_through_mcp",
|
|
23
|
+
"evaluator_seconds", "partition_cases", "int_domain", "runtime_checks") if key in value}
|
|
24
|
+
build = value.get("build", {})
|
|
25
|
+
if build:
|
|
26
|
+
capture = build.get("diagnostics_capture", {})
|
|
27
|
+
result["build"] = {"build_state": build.get("build_state"), "error_present": bool(build.get("error")),
|
|
28
|
+
"diagnostics_available": build.get("diagnostics_available"),
|
|
29
|
+
"diagnostics_summary": {k: build.get("diagnostics_summary", {}).get(k) for k in ("process_status", "errors", "warnings")},
|
|
30
|
+
"capture": {k: capture.get(k) for k in ("freshness", "attribution", "scope", "source_tab")}}
|
|
31
|
+
return result
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def interrupted_attempt(attempt: Path, case: dict, fingerprint: str) -> dict:
|
|
35
|
+
usage = Usage()
|
|
36
|
+
requested, tool_calls, errors, retries = 0, 0, 0, 0
|
|
37
|
+
events = attempt / "events.jsonl"
|
|
38
|
+
if events.exists():
|
|
39
|
+
for line in events.read_text(encoding="utf-8").splitlines():
|
|
40
|
+
try:
|
|
41
|
+
event = json.loads(line)
|
|
42
|
+
except ValueError:
|
|
43
|
+
continue
|
|
44
|
+
if event["kind"] == "model_response":
|
|
45
|
+
raw = read_json(attempt / "blobs" / (event["response_ref"] + ".json"))
|
|
46
|
+
usage.add(raw.get("usage"))
|
|
47
|
+
elif event["kind"] == "model_request_failure":
|
|
48
|
+
usage.add(None)
|
|
49
|
+
elif event["kind"] == "model_request":
|
|
50
|
+
requested += 1
|
|
51
|
+
retries += int(event.get("connection_attempt", 0) > 0)
|
|
52
|
+
elif event["kind"] == "mcp_call" and event.get("phase") == "workflow":
|
|
53
|
+
tool_calls += 1
|
|
54
|
+
errors += int(event.get("is_error", False))
|
|
55
|
+
for _ in range(max(0, requested - usage.requests)):
|
|
56
|
+
usage.add(None)
|
|
57
|
+
return {**case, "fingerprint": fingerprint, "status": "interrupted", "success": False,
|
|
58
|
+
"durable_completed": False, "model_requests": requested, "usage": usage.result(),
|
|
59
|
+
"tool_calls": tool_calls, "tool_errors": errors, "api_retries": retries}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def interaction_metrics(attempt: Path) -> dict:
|
|
63
|
+
calls = defaultdict(list)
|
|
64
|
+
requests = []
|
|
65
|
+
event_path = attempt / "events.jsonl"
|
|
66
|
+
if event_path.exists():
|
|
67
|
+
for line in event_path.read_text(encoding="utf-8").splitlines():
|
|
68
|
+
try:
|
|
69
|
+
event = json.loads(line)
|
|
70
|
+
except ValueError:
|
|
71
|
+
continue
|
|
72
|
+
if event["kind"] == "mcp_call" and event.get("phase") == "workflow":
|
|
73
|
+
calls[event["tool"]].append(event["elapsed"])
|
|
74
|
+
elif event["kind"] in {"model_response", "model_request_failure"}:
|
|
75
|
+
requests.append(event["elapsed"])
|
|
76
|
+
return {"model_request_seconds": distribution(requests),
|
|
77
|
+
"mcp_round_trip_seconds": {name: distribution(values) for name, values in calls.items()}}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def verified_commit(server: dict, proof: dict) -> str:
|
|
81
|
+
"""Link a dirty run to a later commit only when every runtime Python file matches."""
|
|
82
|
+
root = Path(server["source"]["root"])
|
|
83
|
+
package = Path(server["runtime"]["package_path"]).relative_to(root).as_posix()
|
|
84
|
+
commit = proof["commit"]
|
|
85
|
+
resolved = subprocess.check_output(["git", "-C", str(root), "rev-parse", "--verify", commit + "^{commit}"],
|
|
86
|
+
stderr=subprocess.DEVNULL).decode().strip()
|
|
87
|
+
paths = subprocess.check_output(["git", "-C", str(root), "ls-tree", "-r", "--name-only", resolved, "--", package]).decode().splitlines()
|
|
88
|
+
hashes = {str(Path(path).relative_to(package).as_posix()): digest(subprocess.check_output(
|
|
89
|
+
["git", "-C", str(root), "show", f"{resolved}:{path}"]).replace(b"\r\n", b"\n")) for path in paths if path.endswith(".py")}
|
|
90
|
+
if hashes != proof["canonical_file_sha256"] or digest(proof["raw_file_sha256"]) != server["runtime"]["package_tree"]:
|
|
91
|
+
raise ValueError("Proposed source commit does not match the measured runtime package")
|
|
92
|
+
return resolved
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def archive_run(results: Path, run_id: str, destination: Path, *, allow_incomplete=False) -> Path:
|
|
96
|
+
"""Archive numeric/boolean metadata only; never copy raw project/model payloads."""
|
|
97
|
+
if Path(run_id).name != run_id or run_id in {".", ".."}:
|
|
98
|
+
raise ValueError("Invalid run ID")
|
|
99
|
+
run = results / run_id
|
|
100
|
+
plan = read_json(run / "plan.json")
|
|
101
|
+
snapshot = plan["snapshot"]
|
|
102
|
+
config = snapshot["config"]
|
|
103
|
+
live = snapshot.get("suite") == "opc-live"
|
|
104
|
+
if live and plan.get("mode") != "run":
|
|
105
|
+
raise ValueError("Scripted OPC validation is not an LLM benchmark run")
|
|
106
|
+
if live and not read_json(run / "outcome.json").get("restoration_verified"):
|
|
107
|
+
raise ValueError("OPC run did not verify restoration; preserve local evidence and investigate")
|
|
108
|
+
proofs_path = run / "source-verification.json"
|
|
109
|
+
proofs = read_json(proofs_path) if proofs_path.exists() else {}
|
|
110
|
+
rows = []
|
|
111
|
+
missing = []
|
|
112
|
+
for case in plan["schedule"]:
|
|
113
|
+
attempts = sorted((run / case["case_id"]).glob("attempt-*"))
|
|
114
|
+
for attempt in attempts:
|
|
115
|
+
summary_path = attempt / "summary.json"
|
|
116
|
+
row = read_json(summary_path) if summary_path.exists() else interrupted_attempt(attempt, case, plan["fingerprint"])
|
|
117
|
+
if row["fingerprint"] != plan["fingerprint"]:
|
|
118
|
+
raise ValueError("Case fingerprint differs from campaign")
|
|
119
|
+
safe = {key: row[key] for key in (
|
|
120
|
+
"case_id", "task", "repeat", "condition", "fingerprint", "status", "failure_reason",
|
|
121
|
+
"cold_start", "provider", "requested_model", "returned_models", "model_identity",
|
|
122
|
+
"startup_seconds", "discovery_seconds", "setup_seconds", "reset_seconds", "workflow_seconds",
|
|
123
|
+
"total_seconds", "tool_calls", "tool_errors", "repair_cycles", "api_retries", "transport_retries",
|
|
124
|
+
"model_requests", "usage", "success", "durable_completed", "provider_failure_code") if key in row}
|
|
125
|
+
safe["attempt_id"] = attempt.name
|
|
126
|
+
safe["interaction_metrics"] = interaction_metrics(attempt)
|
|
127
|
+
safe["validation"] = summarize_validation(row.get("validation", {}))
|
|
128
|
+
safe["artifact_sha256"] = {name: digest((attempt / name).read_bytes())
|
|
129
|
+
for name in ("project.stu", "saved-section.xml", "saved-variables.xml") if (attempt / name).exists()}
|
|
130
|
+
rows.append(safe)
|
|
131
|
+
if not any(row["case_id"] == case["case_id"] and row.get("durable_completed") for row in rows):
|
|
132
|
+
missing.append(case["case_id"])
|
|
133
|
+
if missing and not allow_incomplete:
|
|
134
|
+
raise ValueError("Campaign has unfinished cases; use --allow-incomplete to archive a labeled snapshot")
|
|
135
|
+
servers = {}
|
|
136
|
+
for name, server in snapshot["servers"].items():
|
|
137
|
+
corpus = server.get("documentation") or {}
|
|
138
|
+
servers[name] = {"source": source_identity(server.get("source", {})),
|
|
139
|
+
"runtime_package_sha256": server.get("runtime", {}).get("package_tree"),
|
|
140
|
+
"dependencies": server.get("runtime", {}).get("dependencies"),
|
|
141
|
+
"documentation_available": config["conditions"][name]["documentation"],
|
|
142
|
+
"corpus_sha256": corpus.get("sha256"), "corpus_files": len(corpus.get("files", {})),
|
|
143
|
+
"environment_sha256": digest(server.get("environment", {}))}
|
|
144
|
+
if name in proofs:
|
|
145
|
+
servers[name]["verified_source_commit"] = verified_commit(server, proofs[name])
|
|
146
|
+
servers[name]["source_verification"] = proofs[name]
|
|
147
|
+
groups = {}
|
|
148
|
+
for condition in sorted({row["condition"] for row in rows}):
|
|
149
|
+
selected = [row for row in rows if row["condition"] == condition]
|
|
150
|
+
groups[condition] = {"overall": aggregate(selected),
|
|
151
|
+
"tasks": {task: aggregate([row for row in selected if row["task"] == task])
|
|
152
|
+
for task in sorted({row["task"] for row in selected})}}
|
|
153
|
+
pairs = defaultdict(dict)
|
|
154
|
+
for row in rows:
|
|
155
|
+
pairs[(row["task"], row["repeat"])][row["condition"]] = row
|
|
156
|
+
differences = defaultdict(list)
|
|
157
|
+
for (task, _), arms in pairs.items():
|
|
158
|
+
if "A" not in arms or "B" not in arms:
|
|
159
|
+
continue
|
|
160
|
+
for metric in ("workflow_seconds", "total_tokens"):
|
|
161
|
+
values = [arms[c].get(metric) if metric == "workflow_seconds" else
|
|
162
|
+
arms[c].get("usage", {}).get("tokens", {}).get(metric) for c in ("A", "B")]
|
|
163
|
+
delta = values[1] - values[0] if all(v is not None for v in values) else None
|
|
164
|
+
differences[f"{task}:{metric}:B_minus_A"].append(delta)
|
|
165
|
+
record = {"archive_version": 1, "run_id": run_id, "fingerprint": plan["fingerprint"],
|
|
166
|
+
"complete": not missing, "unfinished_cases": missing, "scheduled_cases": len(plan["schedule"]),
|
|
167
|
+
"model": config["model"], "provider": config.get("provider", {}).get("name", "openai"),
|
|
168
|
+
"cpu": config["cpu"], "ce_version": config.get("ce_version"),
|
|
169
|
+
"machine_sha256": digest(snapshot["machine"]), "harness": source_identity(snapshot["harness"]),
|
|
170
|
+
"dependencies": snapshot["dependencies"], "servers": servers,
|
|
171
|
+
"fixture_sha256": snapshot["fixture_sha256"], "rubric_version": snapshot["rubric_version"],
|
|
172
|
+
"prompt_sha256": {task: digest(prompt) for task, prompt in snapshot["prompts"].items()},
|
|
173
|
+
"orientation_sha256": digest(snapshot["orientation"]), "contexts": plan.get("contexts"),
|
|
174
|
+
"provider_retry_policy": snapshot.get("provider_retry_policy"),
|
|
175
|
+
"seed": plan["seed"], "schedule": plan["schedule"], "cases": rows, "conditions": groups,
|
|
176
|
+
"paired_differences": {key: distribution(values) for key, values in differences.items()},
|
|
177
|
+
"suite": snapshot.get("suite", "offline-feeder"),
|
|
178
|
+
"limitations": ["Small repetition counts are exploratory, not statistically strong comparisons.",
|
|
179
|
+
("Live scalar OPC input/output checks on PLC Simulator; no physical hardware validation."
|
|
180
|
+
if live else "Static recognized-subset acceptance; PLC runtime behavior remains unverified."),
|
|
181
|
+
"API client/model experiment, distinct from ChatGPT UI.",
|
|
182
|
+
"Dirty source fingerprints identify uncommitted code; do not label them a clean revision."]}
|
|
183
|
+
if live:
|
|
184
|
+
outcome = read_json(run / "outcome.json")
|
|
185
|
+
record["runtime_environment"] = {"ofs": {k: outcome.get("ofs", {}).get(k) for k in
|
|
186
|
+
("version", "worker_bits", "server_state")}, "restoration_verified": True,
|
|
187
|
+
"alias_sha256": digest(plan["alias"])}
|
|
188
|
+
regrades = []
|
|
189
|
+
for path in sorted((run / "regrades").glob("*/*/*.json")):
|
|
190
|
+
from .regrade import regraded_row
|
|
191
|
+
grade = read_json(path)
|
|
192
|
+
summary = run / grade["case_id"] / grade["attempt_id"] / "summary.json"
|
|
193
|
+
regraded_row(summary, grade["rubric"]) # Validate source evidence hashes.
|
|
194
|
+
regrades.append(grade)
|
|
195
|
+
record["inspection_regrades"] = regrades
|
|
196
|
+
record["regraded_conditions"] = {}
|
|
197
|
+
for rubric in sorted({grade["rubric"] for grade in regrades}):
|
|
198
|
+
revised = []
|
|
199
|
+
for row in rows:
|
|
200
|
+
value = dict(row)
|
|
201
|
+
grade = next((g for g in regrades if g["rubric"] == rubric and g["case_id"] == row["case_id"] and g["attempt_id"] == row["attempt_id"]), None)
|
|
202
|
+
if grade:
|
|
203
|
+
value.update(success=grade["success"], validation=grade["validation"])
|
|
204
|
+
revised.append(value)
|
|
205
|
+
record["regraded_conditions"][rubric] = {condition: aggregate([row for row in revised if row["condition"] == condition])
|
|
206
|
+
for condition in sorted({row["condition"] for row in rows})}
|
|
207
|
+
if config["model"].get("identity") == "alias":
|
|
208
|
+
record["limitations"].append("Mutable provider alias: matching model IDs do not guarantee identical weights over time.")
|
|
209
|
+
target = destination / (run_id + ("-partial-" + digest(record)[:10] if missing else ""))
|
|
210
|
+
if target.exists():
|
|
211
|
+
if read_json(target / "record.json") != record:
|
|
212
|
+
raise ValueError("Archive is immutable; use a new run ID for changed evidence")
|
|
213
|
+
checksums = read_json(target / "checksums.json")
|
|
214
|
+
if set(checksums) != {"record.json", "report.md"} or any(
|
|
215
|
+
digest((target / name).read_bytes()) != expected for name, expected in checksums.items()):
|
|
216
|
+
raise ValueError("Archive checksum mismatch; preserve and investigate existing evidence")
|
|
217
|
+
update_index(destination)
|
|
218
|
+
return target
|
|
219
|
+
staging = destination.resolve() / (".staging-" + uuid.uuid4().hex)
|
|
220
|
+
staging.mkdir(parents=True, exist_ok=False)
|
|
221
|
+
write_json(staging / "record.json", record)
|
|
222
|
+
lines = ["# " + run_id, "", f"Provider/model: `{record['provider']}` / `{record['model']['id']}`.",
|
|
223
|
+
f"Harness base revision: `{record['harness']['revision']}`; dirty: `{record['harness']['dirty']}`.", "",
|
|
224
|
+
*["- " + value for value in record["limitations"]], "",
|
|
225
|
+
"| Condition | Task | Attempts | Accepted | Build passes | Median workflow s | Median total tokens |",
|
|
226
|
+
"|---|---|---:|---:|---:|---:|---:|"]
|
|
227
|
+
for condition, group in groups.items():
|
|
228
|
+
for task, value in {**group["tasks"], "overall": group["overall"]}.items():
|
|
229
|
+
lines.append(f"| {condition} | {task} | {value['cases']} | {value['accepted']['pass']} | {value['build']['pass']} | "
|
|
230
|
+
f"{value['all_attempts']['workflow_seconds']['median']} | {value['all_attempts']['total_tokens']['median']} |")
|
|
231
|
+
lines += ["", "Original results above remain unchanged. Explicit inspection regrades:", ""]
|
|
232
|
+
for rubric, conditions in record["regraded_conditions"].items():
|
|
233
|
+
for condition, value in conditions.items():
|
|
234
|
+
lines.append(f"- `{rubric}`, {condition}: {value['accepted']['pass']}/{value['cases']} accepted.")
|
|
235
|
+
lines += ["", "Measured Python source verified against Git (CRLF/LF normalized):", ""]
|
|
236
|
+
for condition, server in servers.items():
|
|
237
|
+
if server.get("verified_source_commit"):
|
|
238
|
+
lines.append(f"- {condition}: `{server['verified_source_commit']}` (original dirty-state metadata retained).")
|
|
239
|
+
(staging / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
240
|
+
write_json(staging / "checksums.json", {name: digest((staging / name).read_bytes()) for name in ("record.json", "report.md")})
|
|
241
|
+
if not staging.resolve().is_relative_to(destination.resolve()) or not target.resolve().is_relative_to(destination.resolve()):
|
|
242
|
+
raise ValueError("Archive destination escaped intended directory")
|
|
243
|
+
staging.rename(target.resolve())
|
|
244
|
+
update_index(destination)
|
|
245
|
+
return target
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def update_index(destination: Path) -> None:
|
|
249
|
+
# The index is derived; individual run directories are append-only.
|
|
250
|
+
entries = []
|
|
251
|
+
for path in sorted(destination.glob("*/record.json")):
|
|
252
|
+
if path.parent.name.startswith("."):
|
|
253
|
+
continue
|
|
254
|
+
value = read_json(path)
|
|
255
|
+
entries.append({"path": path.parent.name, "run_id": value["run_id"], "provider": value["provider"],
|
|
256
|
+
"model": value["model"], "harness": value["harness"], "servers": value["servers"],
|
|
257
|
+
"complete": value["complete"], "conditions": value["conditions"],
|
|
258
|
+
"regraded_conditions": value.get("regraded_conditions", {})})
|
|
259
|
+
write_json(destination / "index.json", entries)
|
|
260
|
+
index = ["# Archived benchmark runs", "", "Sanitized results intended for Git history. Raw evidence stays local.", "",
|
|
261
|
+
"", "",
|
|
262
|
+
"Regenerate from this archive with `ce-bench visualize --archive benchmarks/archive --output benchmarks/archive/overview.png`.", "",
|
|
263
|
+
"| Run | Provider | Model | Measured source | Original accepted | Regraded accepted | Complete |", "|---|---|---|---|---|---|---|"]
|
|
264
|
+
for value in entries:
|
|
265
|
+
revisions = sorted({server.get("verified_source_commit") or server.get("source", {}).get("revision") or "unknown"
|
|
266
|
+
for server in value["servers"].values()})
|
|
267
|
+
original = sum(arm["overall"]["accepted"]["pass"] for arm in value["conditions"].values())
|
|
268
|
+
total = sum(arm["overall"]["cases"] for arm in value["conditions"].values())
|
|
269
|
+
revised = "; ".join(f"{rubric}: {sum(arm['accepted']['pass'] for arm in arms.values())}/{sum(arm['cases'] for arm in arms.values())}"
|
|
270
|
+
for rubric, arms in value.get("regraded_conditions", {}).items()) or "none"
|
|
271
|
+
index.append(f"| [{value['run_id']}]({value['path']}/report.md) | {value['provider']} | {value['model']['id']} | "
|
|
272
|
+
f"{', '.join(rev[:12] for rev in revisions)} | {original}/{total} | {revised} | {value['complete']} |")
|
|
273
|
+
(destination / "README.md").write_text("\n".join(index) + "\n", encoding="utf-8")
|