control-expert-mcp 0.9.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ """MCP server for EcoStruxure Control Expert (Unity Pro)."""
2
+
3
+ from importlib.metadata import version
4
+
5
+ __version__ = version("control-expert-mcp")
@@ -0,0 +1,3 @@
1
+ from .server import main
2
+
3
+ main()
@@ -0,0 +1 @@
1
+ """Local, opt-in API benchmarks. No COM or provider imports at package import."""
@@ -0,0 +1,221 @@
1
+ """Restricted offline ST transition proof for the demo-derived alarm task.
2
+
3
+ Only Boolean assignments/IFs and comparisons of the converted INT temperature
4
+ with constants are supported. No PLC execution or ALRM_DIA emulation is claimed.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import math
9
+ import re
10
+
11
+ FLAGS = {"temp_fault_1", "temp_fault_2"}
12
+ BOOLS = FLAGS | {"start_process"}
13
+ TOKEN = re.compile(r"\s+|[A-Za-z_][A-Za-z_0-9]*|\d+(?:\.\d+)?|:=|<=|>=|<>|[();=<>+\-,]")
14
+
15
+ CANONICAL = """
16
+ (* Demo-derived training extension: hysteretic temperature alarms. *)
17
+ IF NOT Start_process THEN
18
+ Temp_fault_1 := FALSE;
19
+ Temp_fault_2 := FALSE;
20
+ ELSE
21
+ IF Cmeasured_temp <= 200.0 THEN
22
+ Temp_fault_1 := TRUE;
23
+ ELSIF Cmeasured_temp >= 205.0 THEN
24
+ Temp_fault_1 := FALSE;
25
+ END_IF;
26
+ IF Cmeasured_temp >= 400.0 THEN
27
+ Temp_fault_2 := TRUE;
28
+ ELSIF Cmeasured_temp <= 395.0 THEN
29
+ Temp_fault_2 := FALSE;
30
+ END_IF;
31
+ END_IF;
32
+ """
33
+
34
+
35
+ def tokens(text: str) -> list[str]:
36
+ """Tokenize without silently discarding unsupported operators or syntax."""
37
+ text = re.sub(r"\(\*.*?\*\)|//[^\n]*", " ", text, flags=re.S)
38
+ result, position = [], 0
39
+ for match in TOKEN.finditer(text):
40
+ if match.start() != position:
41
+ raise ValueError("Unsupported ST token")
42
+ position = match.end()
43
+ if not match[0].isspace():
44
+ result.append(match[0].lower())
45
+ if position != len(text):
46
+ raise ValueError("Unsupported ST suffix")
47
+ return result
48
+
49
+
50
+ class Parser:
51
+ """Small typed grammar, deliberately excluding arithmetic, calls and loops."""
52
+
53
+ def __init__(self, values: list[str]):
54
+ self.values, self.position, self.constants = values, 0, set()
55
+
56
+ def peek(self):
57
+ return self.values[self.position] if self.position < len(self.values) else ""
58
+
59
+ def take(self, expected=None):
60
+ value = self.peek()
61
+ if not value or (expected is not None and value != expected):
62
+ raise ValueError("Unsupported ST structure")
63
+ self.position += 1
64
+ return value
65
+
66
+ def expression(self, precedence=0):
67
+ operators = {"or": 1, "and": 2, "=": 3, "<>": 3,
68
+ "<": 3, ">": 3, "<=": 3, ">=": 3}
69
+ value = self.take()
70
+ if value == "not":
71
+ node = ("not", self.expression(4))
72
+ elif value == "(":
73
+ node = self.expression()
74
+ self.take(")")
75
+ elif value in {"true", "false"}:
76
+ node = ("bool", value == "true")
77
+ elif value in BOOLS or value == "cmeasured_temp":
78
+ node = ("input", value)
79
+ else:
80
+ sign = -1 if value == "-" else 1
81
+ if value in {"-", "+"}:
82
+ value = self.take()
83
+ if not re.fullmatch(r"\d+(?:\.\d+)?", value):
84
+ raise ValueError("Unsupported alarm expression")
85
+ number = sign * float(value)
86
+ if not math.isfinite(number) or abs(number) > 32768:
87
+ raise ValueError("Alarm constant outside supported range")
88
+ self.constants.add(number)
89
+ node = ("number", number)
90
+ while operators.get(self.peek(), -1) >= precedence:
91
+ operator = self.take()
92
+ node = (operator, node, self.expression(operators[operator] + 1))
93
+ return node
94
+
95
+ def block(self):
96
+ statements = []
97
+ while self.peek() and self.peek() not in {"else", "elsif", "end_if"}:
98
+ if self.peek() == "if":
99
+ self.take("if")
100
+ branches = []
101
+ while True:
102
+ condition = self.expression()
103
+ require_boolean(condition)
104
+ self.take("then")
105
+ branches.append((condition, self.block()))
106
+ if self.peek() != "elsif":
107
+ break
108
+ self.take("elsif")
109
+ otherwise = []
110
+ if self.peek() == "else":
111
+ self.take("else")
112
+ otherwise = self.block()
113
+ self.take("end_if")
114
+ self.take(";")
115
+ statements.append(("if", branches, otherwise))
116
+ else:
117
+ target = self.take()
118
+ if target not in FLAGS:
119
+ raise ValueError("Alarm extension may only assign the two fault flags")
120
+ self.take(":=")
121
+ expression = self.expression()
122
+ require_boolean(expression)
123
+ self.take(";")
124
+ statements.append(("assign", target, expression))
125
+ return statements
126
+
127
+
128
+ def expression_type(node) -> str:
129
+ op = node[0]
130
+ if op in {"bool", "number"}:
131
+ return op
132
+ if op == "input":
133
+ return "number" if node[1] == "cmeasured_temp" else "bool"
134
+ if op == "not":
135
+ require_boolean(node[1])
136
+ elif op in {"and", "or"}:
137
+ require_boolean(node[1])
138
+ require_boolean(node[2])
139
+ elif expression_type(node[1]) != "number" or expression_type(node[2]) != "number":
140
+ raise ValueError("Only numeric comparisons are supported")
141
+ return "bool"
142
+
143
+
144
+ def require_boolean(node):
145
+ if expression_type(node) != "bool":
146
+ raise ValueError("Expected Boolean alarm expression")
147
+
148
+
149
+ def value(node, state):
150
+ op = node[0]
151
+ if op in {"bool", "number"}:
152
+ return node[1]
153
+ if op == "input":
154
+ return state[node[1]]
155
+ left = value(node[1], state)
156
+ if op == "not":
157
+ return not left
158
+ right = value(node[2], state)
159
+ return {"and": lambda: left and right, "or": lambda: left or right,
160
+ "=": lambda: left == right, "<>": lambda: left != right,
161
+ "<": lambda: left < right, ">": lambda: left > right,
162
+ "<=": lambda: left <= right, ">=": lambda: left >= right}[op]()
163
+
164
+
165
+ def execute(statements, state):
166
+ for statement in statements:
167
+ if statement[0] == "assign":
168
+ state[statement[1]] = value(statement[2], state)
169
+ else:
170
+ for condition, branch in statement[1]:
171
+ if value(condition, state):
172
+ execute(branch, state)
173
+ break
174
+ else:
175
+ execute(statement[2], state)
176
+
177
+
178
+ def alarm_grade(baseline: str, actual: str) -> dict:
179
+ """Prove every supported INT-input transition by constant partitions.
180
+
181
+ With no arithmetic, each numeric comparison is constant between its literal
182
+ boundaries. Checking boundary neighbors covers every INT input; enumerating
183
+ both previous flags and enable states proves the one-scan transition table.
184
+ """
185
+ result = {"rubric": "feeder-alarms-1", "scope": "unverified",
186
+ "static_behavior": "unverified", "runtime_behavior": "unverified",
187
+ "method": "restricted ST interpreter + exhaustive INT comparison partitions; no PLC execution"}
188
+ try:
189
+ original, candidate = tokens(baseline), tokens(actual)
190
+ if candidate[:len(original)] != original:
191
+ return result
192
+ parser = Parser(candidate[len(original):])
193
+ statements = parser.block()
194
+ if parser.position != len(parser.values) or not statements:
195
+ return result
196
+ result["scope"] = "pass"
197
+ points = {-32768, 32767}
198
+ for constant in parser.constants | {200, 205, 395, 400}:
199
+ for point in (math.floor(constant), math.ceil(constant)):
200
+ points.update(x for x in (point - 1, point, point + 1) if -32768 <= x <= 32767)
201
+ checked = 0
202
+ for measured in sorted(points):
203
+ for enabled in (False, True):
204
+ for low in (False, True):
205
+ for high in (False, True):
206
+ state = {"cmeasured_temp": float(measured), "start_process": enabled,
207
+ "temp_fault_1": low, "temp_fault_2": high}
208
+ execute(statements, state)
209
+ expected_low = enabled and (measured <= 200 or (low and measured < 205))
210
+ expected_high = enabled and (measured >= 400 or (high and measured > 395))
211
+ checked += 1
212
+ if (state["temp_fault_1"], state["temp_fault_2"]) != (expected_low, expected_high):
213
+ result["counterexample"] = {"measured": measured, "enabled": enabled,
214
+ "prior_low": low, "prior_high": high,
215
+ "expected_low": expected_low, "expected_high": expected_high,
216
+ "actual_low": state["temp_fault_1"], "actual_high": state["temp_fault_2"]}
217
+ return result
218
+ result.update(static_behavior="pass", partition_cases=checked, int_domain=[-32768, 32767])
219
+ except (ValueError, RecursionError):
220
+ pass # Unsupported syntax stays unverified; never silently accept it.
221
+ return result
@@ -0,0 +1,273 @@
1
+ """Append-only, allowlisted benchmark evidence suitable for a Git repository."""
2
+ from __future__ import annotations
3
+
4
+ from collections import defaultdict
5
+ import json
6
+ from pathlib import Path
7
+ import subprocess
8
+ import uuid
9
+
10
+ from .core import digest, read_json, write_json
11
+ from .report import aggregate, distribution
12
+ from .provider import Usage
13
+
14
+
15
+ def source_identity(value: dict) -> dict:
16
+ return {key: value.get(key) for key in ("revision", "dirty", "dirty_fingerprint")}
17
+
18
+
19
+ def summarize_validation(value: dict) -> dict:
20
+ result = {key: value[key] for key in (
21
+ "rubric", "answer_pass", "scope", "static_behavior", "runtime_behavior", "compile_pass",
22
+ "declarations_pass", "model_built", "model_saved", "success", "obtained_source_through_mcp",
23
+ "evaluator_seconds", "partition_cases", "int_domain", "runtime_checks") if key in value}
24
+ build = value.get("build", {})
25
+ if build:
26
+ capture = build.get("diagnostics_capture", {})
27
+ result["build"] = {"build_state": build.get("build_state"), "error_present": bool(build.get("error")),
28
+ "diagnostics_available": build.get("diagnostics_available"),
29
+ "diagnostics_summary": {k: build.get("diagnostics_summary", {}).get(k) for k in ("process_status", "errors", "warnings")},
30
+ "capture": {k: capture.get(k) for k in ("freshness", "attribution", "scope", "source_tab")}}
31
+ return result
32
+
33
+
34
+ def interrupted_attempt(attempt: Path, case: dict, fingerprint: str) -> dict:
35
+ usage = Usage()
36
+ requested, tool_calls, errors, retries = 0, 0, 0, 0
37
+ events = attempt / "events.jsonl"
38
+ if events.exists():
39
+ for line in events.read_text(encoding="utf-8").splitlines():
40
+ try:
41
+ event = json.loads(line)
42
+ except ValueError:
43
+ continue
44
+ if event["kind"] == "model_response":
45
+ raw = read_json(attempt / "blobs" / (event["response_ref"] + ".json"))
46
+ usage.add(raw.get("usage"))
47
+ elif event["kind"] == "model_request_failure":
48
+ usage.add(None)
49
+ elif event["kind"] == "model_request":
50
+ requested += 1
51
+ retries += int(event.get("connection_attempt", 0) > 0)
52
+ elif event["kind"] == "mcp_call" and event.get("phase") == "workflow":
53
+ tool_calls += 1
54
+ errors += int(event.get("is_error", False))
55
+ for _ in range(max(0, requested - usage.requests)):
56
+ usage.add(None)
57
+ return {**case, "fingerprint": fingerprint, "status": "interrupted", "success": False,
58
+ "durable_completed": False, "model_requests": requested, "usage": usage.result(),
59
+ "tool_calls": tool_calls, "tool_errors": errors, "api_retries": retries}
60
+
61
+
62
+ def interaction_metrics(attempt: Path) -> dict:
63
+ calls = defaultdict(list)
64
+ requests = []
65
+ event_path = attempt / "events.jsonl"
66
+ if event_path.exists():
67
+ for line in event_path.read_text(encoding="utf-8").splitlines():
68
+ try:
69
+ event = json.loads(line)
70
+ except ValueError:
71
+ continue
72
+ if event["kind"] == "mcp_call" and event.get("phase") == "workflow":
73
+ calls[event["tool"]].append(event["elapsed"])
74
+ elif event["kind"] in {"model_response", "model_request_failure"}:
75
+ requests.append(event["elapsed"])
76
+ return {"model_request_seconds": distribution(requests),
77
+ "mcp_round_trip_seconds": {name: distribution(values) for name, values in calls.items()}}
78
+
79
+
80
+ def verified_commit(server: dict, proof: dict) -> str:
81
+ """Link a dirty run to a later commit only when every runtime Python file matches."""
82
+ root = Path(server["source"]["root"])
83
+ package = Path(server["runtime"]["package_path"]).relative_to(root).as_posix()
84
+ commit = proof["commit"]
85
+ resolved = subprocess.check_output(["git", "-C", str(root), "rev-parse", "--verify", commit + "^{commit}"],
86
+ stderr=subprocess.DEVNULL).decode().strip()
87
+ paths = subprocess.check_output(["git", "-C", str(root), "ls-tree", "-r", "--name-only", resolved, "--", package]).decode().splitlines()
88
+ hashes = {str(Path(path).relative_to(package).as_posix()): digest(subprocess.check_output(
89
+ ["git", "-C", str(root), "show", f"{resolved}:{path}"]).replace(b"\r\n", b"\n")) for path in paths if path.endswith(".py")}
90
+ if hashes != proof["canonical_file_sha256"] or digest(proof["raw_file_sha256"]) != server["runtime"]["package_tree"]:
91
+ raise ValueError("Proposed source commit does not match the measured runtime package")
92
+ return resolved
93
+
94
+
95
+ def archive_run(results: Path, run_id: str, destination: Path, *, allow_incomplete=False) -> Path:
96
+ """Archive numeric/boolean metadata only; never copy raw project/model payloads."""
97
+ if Path(run_id).name != run_id or run_id in {".", ".."}:
98
+ raise ValueError("Invalid run ID")
99
+ run = results / run_id
100
+ plan = read_json(run / "plan.json")
101
+ snapshot = plan["snapshot"]
102
+ config = snapshot["config"]
103
+ live = snapshot.get("suite") == "opc-live"
104
+ if live and plan.get("mode") != "run":
105
+ raise ValueError("Scripted OPC validation is not an LLM benchmark run")
106
+ if live and not read_json(run / "outcome.json").get("restoration_verified"):
107
+ raise ValueError("OPC run did not verify restoration; preserve local evidence and investigate")
108
+ proofs_path = run / "source-verification.json"
109
+ proofs = read_json(proofs_path) if proofs_path.exists() else {}
110
+ rows = []
111
+ missing = []
112
+ for case in plan["schedule"]:
113
+ attempts = sorted((run / case["case_id"]).glob("attempt-*"))
114
+ for attempt in attempts:
115
+ summary_path = attempt / "summary.json"
116
+ row = read_json(summary_path) if summary_path.exists() else interrupted_attempt(attempt, case, plan["fingerprint"])
117
+ if row["fingerprint"] != plan["fingerprint"]:
118
+ raise ValueError("Case fingerprint differs from campaign")
119
+ safe = {key: row[key] for key in (
120
+ "case_id", "task", "repeat", "condition", "fingerprint", "status", "failure_reason",
121
+ "cold_start", "provider", "requested_model", "returned_models", "model_identity",
122
+ "startup_seconds", "discovery_seconds", "setup_seconds", "reset_seconds", "workflow_seconds",
123
+ "total_seconds", "tool_calls", "tool_errors", "repair_cycles", "api_retries", "transport_retries",
124
+ "model_requests", "usage", "success", "durable_completed", "provider_failure_code") if key in row}
125
+ safe["attempt_id"] = attempt.name
126
+ safe["interaction_metrics"] = interaction_metrics(attempt)
127
+ safe["validation"] = summarize_validation(row.get("validation", {}))
128
+ safe["artifact_sha256"] = {name: digest((attempt / name).read_bytes())
129
+ for name in ("project.stu", "saved-section.xml", "saved-variables.xml") if (attempt / name).exists()}
130
+ rows.append(safe)
131
+ if not any(row["case_id"] == case["case_id"] and row.get("durable_completed") for row in rows):
132
+ missing.append(case["case_id"])
133
+ if missing and not allow_incomplete:
134
+ raise ValueError("Campaign has unfinished cases; use --allow-incomplete to archive a labeled snapshot")
135
+ servers = {}
136
+ for name, server in snapshot["servers"].items():
137
+ corpus = server.get("documentation") or {}
138
+ servers[name] = {"source": source_identity(server.get("source", {})),
139
+ "runtime_package_sha256": server.get("runtime", {}).get("package_tree"),
140
+ "dependencies": server.get("runtime", {}).get("dependencies"),
141
+ "documentation_available": config["conditions"][name]["documentation"],
142
+ "corpus_sha256": corpus.get("sha256"), "corpus_files": len(corpus.get("files", {})),
143
+ "environment_sha256": digest(server.get("environment", {}))}
144
+ if name in proofs:
145
+ servers[name]["verified_source_commit"] = verified_commit(server, proofs[name])
146
+ servers[name]["source_verification"] = proofs[name]
147
+ groups = {}
148
+ for condition in sorted({row["condition"] for row in rows}):
149
+ selected = [row for row in rows if row["condition"] == condition]
150
+ groups[condition] = {"overall": aggregate(selected),
151
+ "tasks": {task: aggregate([row for row in selected if row["task"] == task])
152
+ for task in sorted({row["task"] for row in selected})}}
153
+ pairs = defaultdict(dict)
154
+ for row in rows:
155
+ pairs[(row["task"], row["repeat"])][row["condition"]] = row
156
+ differences = defaultdict(list)
157
+ for (task, _), arms in pairs.items():
158
+ if "A" not in arms or "B" not in arms:
159
+ continue
160
+ for metric in ("workflow_seconds", "total_tokens"):
161
+ values = [arms[c].get(metric) if metric == "workflow_seconds" else
162
+ arms[c].get("usage", {}).get("tokens", {}).get(metric) for c in ("A", "B")]
163
+ delta = values[1] - values[0] if all(v is not None for v in values) else None
164
+ differences[f"{task}:{metric}:B_minus_A"].append(delta)
165
+ record = {"archive_version": 1, "run_id": run_id, "fingerprint": plan["fingerprint"],
166
+ "complete": not missing, "unfinished_cases": missing, "scheduled_cases": len(plan["schedule"]),
167
+ "model": config["model"], "provider": config.get("provider", {}).get("name", "openai"),
168
+ "cpu": config["cpu"], "ce_version": config.get("ce_version"),
169
+ "machine_sha256": digest(snapshot["machine"]), "harness": source_identity(snapshot["harness"]),
170
+ "dependencies": snapshot["dependencies"], "servers": servers,
171
+ "fixture_sha256": snapshot["fixture_sha256"], "rubric_version": snapshot["rubric_version"],
172
+ "prompt_sha256": {task: digest(prompt) for task, prompt in snapshot["prompts"].items()},
173
+ "orientation_sha256": digest(snapshot["orientation"]), "contexts": plan.get("contexts"),
174
+ "provider_retry_policy": snapshot.get("provider_retry_policy"),
175
+ "seed": plan["seed"], "schedule": plan["schedule"], "cases": rows, "conditions": groups,
176
+ "paired_differences": {key: distribution(values) for key, values in differences.items()},
177
+ "suite": snapshot.get("suite", "offline-feeder"),
178
+ "limitations": ["Small repetition counts are exploratory, not statistically strong comparisons.",
179
+ ("Live scalar OPC input/output checks on PLC Simulator; no physical hardware validation."
180
+ if live else "Static recognized-subset acceptance; PLC runtime behavior remains unverified."),
181
+ "API client/model experiment, distinct from ChatGPT UI.",
182
+ "Dirty source fingerprints identify uncommitted code; do not label them a clean revision."]}
183
+ if live:
184
+ outcome = read_json(run / "outcome.json")
185
+ record["runtime_environment"] = {"ofs": {k: outcome.get("ofs", {}).get(k) for k in
186
+ ("version", "worker_bits", "server_state")}, "restoration_verified": True,
187
+ "alias_sha256": digest(plan["alias"])}
188
+ regrades = []
189
+ for path in sorted((run / "regrades").glob("*/*/*.json")):
190
+ from .regrade import regraded_row
191
+ grade = read_json(path)
192
+ summary = run / grade["case_id"] / grade["attempt_id"] / "summary.json"
193
+ regraded_row(summary, grade["rubric"]) # Validate source evidence hashes.
194
+ regrades.append(grade)
195
+ record["inspection_regrades"] = regrades
196
+ record["regraded_conditions"] = {}
197
+ for rubric in sorted({grade["rubric"] for grade in regrades}):
198
+ revised = []
199
+ for row in rows:
200
+ value = dict(row)
201
+ grade = next((g for g in regrades if g["rubric"] == rubric and g["case_id"] == row["case_id"] and g["attempt_id"] == row["attempt_id"]), None)
202
+ if grade:
203
+ value.update(success=grade["success"], validation=grade["validation"])
204
+ revised.append(value)
205
+ record["regraded_conditions"][rubric] = {condition: aggregate([row for row in revised if row["condition"] == condition])
206
+ for condition in sorted({row["condition"] for row in rows})}
207
+ if config["model"].get("identity") == "alias":
208
+ record["limitations"].append("Mutable provider alias: matching model IDs do not guarantee identical weights over time.")
209
+ target = destination / (run_id + ("-partial-" + digest(record)[:10] if missing else ""))
210
+ if target.exists():
211
+ if read_json(target / "record.json") != record:
212
+ raise ValueError("Archive is immutable; use a new run ID for changed evidence")
213
+ checksums = read_json(target / "checksums.json")
214
+ if set(checksums) != {"record.json", "report.md"} or any(
215
+ digest((target / name).read_bytes()) != expected for name, expected in checksums.items()):
216
+ raise ValueError("Archive checksum mismatch; preserve and investigate existing evidence")
217
+ update_index(destination)
218
+ return target
219
+ staging = destination.resolve() / (".staging-" + uuid.uuid4().hex)
220
+ staging.mkdir(parents=True, exist_ok=False)
221
+ write_json(staging / "record.json", record)
222
+ lines = ["# " + run_id, "", f"Provider/model: `{record['provider']}` / `{record['model']['id']}`.",
223
+ f"Harness base revision: `{record['harness']['revision']}`; dirty: `{record['harness']['dirty']}`.", "",
224
+ *["- " + value for value in record["limitations"]], "",
225
+ "| Condition | Task | Attempts | Accepted | Build passes | Median workflow s | Median total tokens |",
226
+ "|---|---|---:|---:|---:|---:|---:|"]
227
+ for condition, group in groups.items():
228
+ for task, value in {**group["tasks"], "overall": group["overall"]}.items():
229
+ lines.append(f"| {condition} | {task} | {value['cases']} | {value['accepted']['pass']} | {value['build']['pass']} | "
230
+ f"{value['all_attempts']['workflow_seconds']['median']} | {value['all_attempts']['total_tokens']['median']} |")
231
+ lines += ["", "Original results above remain unchanged. Explicit inspection regrades:", ""]
232
+ for rubric, conditions in record["regraded_conditions"].items():
233
+ for condition, value in conditions.items():
234
+ lines.append(f"- `{rubric}`, {condition}: {value['accepted']['pass']}/{value['cases']} accepted.")
235
+ lines += ["", "Measured Python source verified against Git (CRLF/LF normalized):", ""]
236
+ for condition, server in servers.items():
237
+ if server.get("verified_source_commit"):
238
+ lines.append(f"- {condition}: `{server['verified_source_commit']}` (original dirty-state metadata retained).")
239
+ (staging / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
240
+ write_json(staging / "checksums.json", {name: digest((staging / name).read_bytes()) for name in ("record.json", "report.md")})
241
+ if not staging.resolve().is_relative_to(destination.resolve()) or not target.resolve().is_relative_to(destination.resolve()):
242
+ raise ValueError("Archive destination escaped intended directory")
243
+ staging.rename(target.resolve())
244
+ update_index(destination)
245
+ return target
246
+
247
+
248
+ def update_index(destination: Path) -> None:
249
+ # The index is derived; individual run directories are append-only.
250
+ entries = []
251
+ for path in sorted(destination.glob("*/record.json")):
252
+ if path.parent.name.startswith("."):
253
+ continue
254
+ value = read_json(path)
255
+ entries.append({"path": path.parent.name, "run_id": value["run_id"], "provider": value["provider"],
256
+ "model": value["model"], "harness": value["harness"], "servers": value["servers"],
257
+ "complete": value["complete"], "conditions": value["conditions"],
258
+ "regraded_conditions": value.get("regraded_conditions", {})})
259
+ write_json(destination / "index.json", entries)
260
+ index = ["# Archived benchmark runs", "", "Sanitized results intended for Git history. Raw evidence stays local.", "",
261
+ "![Graphical benchmark overview](overview.png)", "",
262
+ "Regenerate from this archive with `ce-bench visualize --archive benchmarks/archive --output benchmarks/archive/overview.png`.", "",
263
+ "| Run | Provider | Model | Measured source | Original accepted | Regraded accepted | Complete |", "|---|---|---|---|---|---|---|"]
264
+ for value in entries:
265
+ revisions = sorted({server.get("verified_source_commit") or server.get("source", {}).get("revision") or "unknown"
266
+ for server in value["servers"].values()})
267
+ original = sum(arm["overall"]["accepted"]["pass"] for arm in value["conditions"].values())
268
+ total = sum(arm["overall"]["cases"] for arm in value["conditions"].values())
269
+ revised = "; ".join(f"{rubric}: {sum(arm['accepted']['pass'] for arm in arms.values())}/{sum(arm['cases'] for arm in arms.values())}"
270
+ for rubric, arms in value.get("regraded_conditions", {}).items()) or "none"
271
+ index.append(f"| [{value['run_id']}]({value['path']}/report.md) | {value['provider']} | {value['model']['id']} | "
272
+ f"{', '.join(rev[:12] for rev in revisions)} | {original}/{total} | {revised} | {value['complete']} |")
273
+ (destination / "README.md").write_text("\n".join(index) + "\n", encoding="utf-8")