agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,2908 @@
1
+ """Checksum-locked multi-arm driver for stateful external arenas.
2
+
3
+ The public atomic replay contract is intentionally pairwise: one external
4
+ runtime operation restores the same protected origin for a baseline and one
5
+ treatment. This driver lifts that contract to a frozen multi-arm experiment
6
+ cell by constructing every payload before execution and then running one
7
+ atomic pair per treatment. Every pair is bound to the same checkpoint,
8
+ origin, replay identity, seed, and dependency set. The repeated NO_OP result
9
+ must also have identical official metrics, trace, usage, and logical receipt
10
+ fields or the whole cell aborts without a terminal result.
11
+
12
+ No upstream runtime, model, selector, or intervention is selected here.
13
+ Operators provide checksum-locked zero-argument builders in one strict JSON
14
+ configuration. Loading and preflight never download assets or execute a
15
+ benchmark cell.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import hashlib
21
+ import json
22
+ import math
23
+ import os
24
+ import re
25
+ from collections.abc import Mapping, Sequence
26
+ from dataclasses import dataclass, field
27
+ from pathlib import Path
28
+ from types import MappingProxyType
29
+ from typing import Any, Protocol, runtime_checkable
30
+
31
+ from agentinputlab.benchmarks import (
32
+ ATOMIC_EXTERNAL_PAIR_CONTRACT,
33
+ AtomicExternalReplayBridge,
34
+ AtomicExternalReplayRunner,
35
+ AtomicPairedReplayRecord,
36
+ AtomicReplayArmRequest,
37
+ AtomicReplayRequest,
38
+ BenchmarkRegistry,
39
+ interactive_benchmark_specs,
40
+ )
41
+ from agentinputlab.contracts.enums import ProfileKind, SemanticFamily
42
+ from agentinputlab.contracts.errors import ArtifactError, ValidationError
43
+ from agentinputlab.contracts.serialization import (
44
+ content_hash,
45
+ deep_freeze_jsonable,
46
+ freeze_mapping,
47
+ to_jsonable,
48
+ )
49
+ from agentinputlab.contracts.state import FrozenAgentState
50
+ from agentinputlab.experiments.executor import ExperimentCellResult
51
+ from agentinputlab.experiments.loader import resolve_driver_entrypoint
52
+ from agentinputlab.experiments.plan import ExperimentCell
53
+ from agentinputlab.experiments.schema import (
54
+ ExperimentArmKind,
55
+ ExperimentArmSpec,
56
+ ExperimentBenchmarkTarget,
57
+ ExperimentManifest,
58
+ PrerequisiteKind,
59
+ )
60
+ from agentinputlab.selectors import (
61
+ SelectorArtifactBundle,
62
+ SelectorArtifactStage,
63
+ VisibleStateFeatureExtractor,
64
+ )
65
+
66
+ CONFIG_PATH_ENV = "AILAB_EXTERNAL_ARENA_DRIVER_CONFIG"
67
+ CONFIG_SHA256_ENV = "AILAB_EXTERNAL_ARENA_DRIVER_CONFIG_SHA256"
68
+
69
+ _CONFIG_SCHEMA_VERSION = 3
70
+ _DRIVER_IMPLEMENTATION_ID = "agentinputlab.external_arena.atomic_multi_arm"
71
+ _VALID_SUCCESS_FORMULA_ID = "agentinputlab.valid_success.v1"
72
+ _CONTRACT_PRESERVED_METRIC = "contract_preserved"
73
+ _PRIVILEGED_LEAKAGE_METRIC = "privileged_leakage"
74
+ _VALID_SUCCESS_METRIC = "valid_success"
75
+ _SHA256 = re.compile(r"^[0-9a-f]{64}$")
76
+ _ANONYMIZED_GROUP = re.compile(r"^group_[0-9a-f]{64}$")
77
+ _ENTRYPOINT = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*:[A-Za-z_][A-Za-z0-9_.]*$")
78
+ _ENVIRONMENT_NAME = re.compile(r"^[A-Z_][A-Z0-9_]*$")
79
+ _MAX_CONFIG_BYTES = 4 * 1024 * 1024
80
+ _MAX_POLICY_ARTIFACT_BYTES = 32 * 1024 * 1024
81
+ _BUILDER_KINDS = frozenset({"arena_checkpoint", "atomic_bridge", "arm_payload"})
82
+ _CORE_FAMILIES = frozenset(
83
+ {
84
+ SemanticFamily.REFORMULATE,
85
+ SemanticFamily.DECOMPOSE,
86
+ SemanticFamily.ACQUIRE,
87
+ SemanticFamily.CLARIFY,
88
+ SemanticFamily.CURATE,
89
+ }
90
+ )
91
+ _EXECUTABLE_RELEASE_FAMILIES = frozenset(
92
+ {
93
+ *_CORE_FAMILIES,
94
+ SemanticFamily.COMPOSITE,
95
+ }
96
+ )
97
+ _SUPPORTED_ARENAS = frozenset(
98
+ {
99
+ "tau3.banking_knowledge",
100
+ "tau3.retail",
101
+ "state_bench.common_support",
102
+ "toolsandbox.common_support",
103
+ "appworld_ul.common_support",
104
+ "webarena.external",
105
+ }
106
+ )
107
+ _REGISTERED_OFFICIAL_SUCCESS_METRICS = {
108
+ "tau3.banking_knowledge": "official_reward",
109
+ # Native RewardInfo is retained separately. A completed native evaluator
110
+ # must not turn an episode timeout or shared-budget stop into paper success.
111
+ "tau3.retail": "terminal_success",
112
+ "state_bench.common_support": "task_completion_pass",
113
+ "toolsandbox.common_support": "official_similarity",
114
+ "appworld_ul.common_support": "task_success",
115
+ }
116
+ _POLICY_KIND_BY_RELEASE = {
117
+ "policy.locked_best_eligible_global": "locked_best_eligible_global",
118
+ "policy.eligibility_aware_selector": "frozen_selector",
119
+ }
120
+ _REQUIRED_SHARED_DEPENDENCY_SLOTS = frozenset(
121
+ {
122
+ "agent_model",
123
+ "official_runtime",
124
+ "corpus",
125
+ "preprocessing",
126
+ "index",
127
+ "embedder",
128
+ "retriever",
129
+ "reranker",
130
+ "reader",
131
+ "scorer",
132
+ }
133
+ )
134
+ _RETAIL_ENVIRONMENT_DEPENDENCY_SLOTS = frozenset({"user_simulator", "nl_assertion_judge"})
135
+ _NOT_APPLICABLE_DEPENDENCY_HASHES = MappingProxyType(
136
+ {
137
+ slot: content_hash(
138
+ {
139
+ "agentinputlab_dependency": "not_applicable",
140
+ "schema_version": 1,
141
+ "slot": slot,
142
+ }
143
+ )
144
+ for slot in _REQUIRED_SHARED_DEPENDENCY_SLOTS
145
+ }
146
+ )
147
+ _GATE_BINDINGS = {
148
+ "toolsandbox.common_support": (
149
+ "AILAB_TOOLSANDBOX_GATE_RECEIPT",
150
+ "AILAB_TOOLSANDBOX_GATE_RECEIPT_SHA256",
151
+ "atomic_restore",
152
+ ),
153
+ "appworld_ul.common_support": (
154
+ "AILAB_APPWORLD_UL_GATE_RECEIPT",
155
+ "AILAB_APPWORLD_UL_GATE_RECEIPT_SHA256",
156
+ "atomic_restore",
157
+ ),
158
+ "webarena.external": (
159
+ "AILAB_WEBARENA_STACK_RECEIPT",
160
+ "AILAB_WEBARENA_STACK_RECEIPT_SHA256",
161
+ "independent_stack_or_verified_reset",
162
+ ),
163
+ }
164
+ _SECRET_TEXT_PATTERNS = (
165
+ re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{8,}"),
166
+ re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"),
167
+ re.compile(r"\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b"),
168
+ )
169
+
170
+
171
+ @dataclass(frozen=True, slots=True)
172
+ class ExternalArenaCheckpoint:
173
+ """Operator-captured visible state plus opaque protected-runtime identities."""
174
+
175
+ benchmark_id: str
176
+ checkpoint_id: str
177
+ group_id: str
178
+ source_revision: str
179
+ origin_state_hash: str
180
+ replay_identity: str
181
+ locked_dependency_hashes: Mapping[str, str]
182
+ visible_state: FrozenAgentState
183
+ eligible_families: tuple[SemanticFamily, ...]
184
+
185
+ def __post_init__(self) -> None:
186
+ for label in ("benchmark_id", "checkpoint_id", "group_id", "source_revision"):
187
+ _require_text(getattr(self, label), label=f"external checkpoint {label}")
188
+ _require_sha256(
189
+ self.origin_state_hash,
190
+ label="external checkpoint origin_state_hash",
191
+ )
192
+ _require_sha256(
193
+ self.replay_identity,
194
+ label="external checkpoint replay_identity",
195
+ )
196
+ if not isinstance(self.visible_state, FrozenAgentState):
197
+ raise TypeError("external checkpoint visible_state must be FrozenAgentState")
198
+ families = tuple(self.eligible_families)
199
+ if len(families) != len(set(families)) or any(
200
+ family not in _EXECUTABLE_RELEASE_FAMILIES for family in families
201
+ ):
202
+ raise ValidationError(
203
+ "external checkpoint eligible_families must be unique executable families"
204
+ )
205
+ object.__setattr__(
206
+ self,
207
+ "locked_dependency_hashes",
208
+ _validated_dependency_hashes(self.locked_dependency_hashes, benchmark_id=self.benchmark_id),
209
+ )
210
+ object.__setattr__(self, "eligible_families", families)
211
+
212
+ @property
213
+ def identity_hash(self) -> str:
214
+ return content_hash(
215
+ {
216
+ "benchmark_id": self.benchmark_id,
217
+ "checkpoint_id": self.checkpoint_id,
218
+ "group_id": self.group_id,
219
+ "source_revision": self.source_revision,
220
+ "origin_state_hash": self.origin_state_hash,
221
+ "replay_identity": self.replay_identity,
222
+ "locked_dependency_hashes": self.locked_dependency_hashes,
223
+ "visible_state_hash": self.visible_state.state_hash,
224
+ "eligible_families": self.eligible_families,
225
+ }
226
+ )
227
+
228
+
229
+ @dataclass(frozen=True, slots=True)
230
+ class ExternalArmPayload:
231
+ """One typed, canonically serializable artifact sent to the external runtime."""
232
+
233
+ arm_id: str
234
+ release_id: str
235
+ family: SemanticFamily
236
+ profile_kind: ProfileKind
237
+ payload: Any = field(repr=False)
238
+ selector_decision: Mapping[str, Any] | None = None
239
+ selector_input_state_hash: str | None = None
240
+ selector_feature_hash: str | None = None
241
+
242
+ def __post_init__(self) -> None:
243
+ _require_text(self.arm_id, label="external arm payload arm_id")
244
+ _require_text(self.release_id, label="external arm payload release_id")
245
+ if not isinstance(self.family, SemanticFamily):
246
+ raise ValidationError("external arm payload family must be typed")
247
+ if not isinstance(self.profile_kind, ProfileKind):
248
+ raise ValidationError("external arm payload profile_kind must be typed")
249
+ try:
250
+ payload = deep_freeze_jsonable(self.payload)
251
+ content_hash(payload)
252
+ except (TypeError, ValueError) as exc:
253
+ raise ValidationError(f"external arm payload must be canonical JSON: {exc}") from exc
254
+ if payload is None:
255
+ raise ValidationError("external arm payload cannot be null")
256
+ decision = self.selector_decision
257
+ if decision is not None:
258
+ decision = _validated_policy_decision(decision)
259
+ selector_hashes = (
260
+ self.selector_input_state_hash,
261
+ self.selector_feature_hash,
262
+ )
263
+ if all(value is None for value in selector_hashes):
264
+ pass
265
+ elif any(value is None for value in selector_hashes):
266
+ raise ValidationError(
267
+ "selector input state_hash and feature_hash must be supplied together"
268
+ )
269
+ else:
270
+ _require_sha256(
271
+ self.selector_input_state_hash,
272
+ label="selector input state_hash",
273
+ )
274
+ _require_sha256(
275
+ self.selector_feature_hash,
276
+ label="selector input feature_hash",
277
+ )
278
+ object.__setattr__(self, "payload", payload)
279
+ object.__setattr__(self, "selector_decision", decision)
280
+
281
+ @property
282
+ def payload_sha256(self) -> str:
283
+ return content_hash(
284
+ {
285
+ "arm_id": self.arm_id,
286
+ "release_id": self.release_id,
287
+ "family": self.family,
288
+ "profile_kind": self.profile_kind,
289
+ "payload": self.payload,
290
+ "selector_decision": self.selector_decision,
291
+ "selector_input_state_hash": self.selector_input_state_hash,
292
+ "selector_feature_hash": self.selector_feature_hash,
293
+ }
294
+ )
295
+
296
+
297
+ @dataclass(frozen=True, slots=True)
298
+ class ExternalCheckpointBuildRequest:
299
+ benchmark_id: str
300
+ split: str
301
+ checkpoint_id: str
302
+ source_revision: str
303
+ model_slot: str
304
+ seed: int
305
+ builder_parameters: Mapping[str, Any]
306
+
307
+ def __post_init__(self) -> None:
308
+ for label in (
309
+ "benchmark_id",
310
+ "split",
311
+ "checkpoint_id",
312
+ "source_revision",
313
+ "model_slot",
314
+ ):
315
+ _require_text(getattr(self, label), label=f"checkpoint build {label}")
316
+ _require_seed(self.seed)
317
+ object.__setattr__(
318
+ self,
319
+ "builder_parameters",
320
+ freeze_mapping(self.builder_parameters),
321
+ )
322
+
323
+
324
+ @dataclass(frozen=True, slots=True)
325
+ class ExternalUnitPreflightRequest:
326
+ """Exact resolved unit inventory presented to a read-only checkpoint builder."""
327
+
328
+ benchmark_id: str
329
+ split: str
330
+ source_revision: str
331
+ unit_ids: tuple[str, ...]
332
+ builder_parameters: Mapping[str, Any]
333
+
334
+ def __post_init__(self) -> None:
335
+ for label in ("benchmark_id", "split", "source_revision"):
336
+ _require_text(
337
+ getattr(self, label),
338
+ label=f"unit preflight {label}",
339
+ )
340
+ unit_ids = tuple(self.unit_ids)
341
+ if (
342
+ not unit_ids
343
+ or len(unit_ids) != len(set(unit_ids))
344
+ or any(
345
+ not isinstance(item, str) or not item.strip() or item != item.strip()
346
+ for item in unit_ids
347
+ )
348
+ ):
349
+ raise ValidationError("unit preflight requires unique trimmed non-empty unit IDs")
350
+ object.__setattr__(self, "unit_ids", unit_ids)
351
+ object.__setattr__(
352
+ self,
353
+ "builder_parameters",
354
+ freeze_mapping(self.builder_parameters),
355
+ )
356
+
357
+
358
+ @dataclass(frozen=True, slots=True)
359
+ class ExternalBridgeBuildRequest:
360
+ checkpoint: ExternalArenaCheckpoint
361
+ model_slot: str
362
+ seed: int
363
+ builder_parameters: Mapping[str, Any]
364
+
365
+ def __post_init__(self) -> None:
366
+ if not isinstance(self.checkpoint, ExternalArenaCheckpoint):
367
+ raise TypeError("bridge build request requires ExternalArenaCheckpoint")
368
+ _require_text(self.model_slot, label="bridge build model_slot")
369
+ _require_seed(self.seed)
370
+ object.__setattr__(
371
+ self,
372
+ "builder_parameters",
373
+ freeze_mapping(self.builder_parameters),
374
+ )
375
+
376
+
377
+ @dataclass(frozen=True, slots=True)
378
+ class ExternalArmPayloadBuildRequest:
379
+ checkpoint: ExternalArenaCheckpoint
380
+ arm_id: str
381
+ release_id: str
382
+ family: SemanticFamily
383
+ profile_kind: ProfileKind
384
+ arm_kind: ExperimentArmKind
385
+ model_slot: str
386
+ seed: int
387
+ builder_parameters: Mapping[str, Any]
388
+ arm_parameters: Mapping[str, Any]
389
+ pre_action: Mapping[str, Any]
390
+
391
+ def __post_init__(self) -> None:
392
+ if not isinstance(self.checkpoint, ExternalArenaCheckpoint):
393
+ raise TypeError("payload build request requires ExternalArenaCheckpoint")
394
+ for label in ("arm_id", "release_id", "model_slot"):
395
+ _require_text(getattr(self, label), label=f"payload build {label}")
396
+ if not isinstance(self.family, SemanticFamily):
397
+ raise ValidationError("payload build family must be typed")
398
+ if not isinstance(self.profile_kind, ProfileKind):
399
+ raise ValidationError("payload build profile_kind must be typed")
400
+ if not isinstance(self.arm_kind, ExperimentArmKind):
401
+ raise ValidationError("payload build arm_kind must be typed")
402
+ _require_seed(self.seed)
403
+ object.__setattr__(
404
+ self,
405
+ "builder_parameters",
406
+ freeze_mapping(self.builder_parameters),
407
+ )
408
+ object.__setattr__(self, "arm_parameters", freeze_mapping(self.arm_parameters))
409
+ frozen_pre_action = freeze_mapping(self.pre_action)
410
+ for key in ("state_hash", "feature_hash"):
411
+ _require_sha256(
412
+ frozen_pre_action.get(key),
413
+ label=f"payload build pre_action {key}",
414
+ )
415
+ object.__setattr__(self, "pre_action", frozen_pre_action)
416
+
417
+
418
+ @runtime_checkable
419
+ class LockedExternalArenaBuilder(Protocol):
420
+ """Shared contract for operator-owned, checksum-locked component builders."""
421
+
422
+ builder_kind: str
423
+
424
+ def fingerprint(self) -> Mapping[str, Any]: ...
425
+
426
+ def preflight(self, parameters: Mapping[str, Any]) -> Sequence[str]: ...
427
+
428
+ def build(self, request: Any) -> Any: ...
429
+
430
+
431
+ @dataclass(frozen=True, slots=True)
432
+ class _LockedBuilderSpec:
433
+ entrypoint: str
434
+ fingerprint_sha256: str
435
+ parameters: Mapping[str, Any] = field(default_factory=dict)
436
+
437
+ def __post_init__(self) -> None:
438
+ if not isinstance(self.entrypoint, str) or _ENTRYPOINT.fullmatch(self.entrypoint) is None:
439
+ raise ValidationError("external builder entrypoint must use module:factory syntax")
440
+ _require_sha256(
441
+ self.fingerprint_sha256,
442
+ label="external builder fingerprint_sha256",
443
+ )
444
+ object.__setattr__(self, "parameters", freeze_mapping(self.parameters))
445
+
446
+ @classmethod
447
+ def from_mapping(cls, value: Any, *, path: str) -> _LockedBuilderSpec:
448
+ data = _strict_mapping(
449
+ value,
450
+ path=path,
451
+ keys={"entrypoint", "fingerprint_sha256", "parameters"},
452
+ required={"entrypoint", "fingerprint_sha256"},
453
+ )
454
+ parameters = data.get("parameters", {})
455
+ if not isinstance(parameters, Mapping):
456
+ raise ValidationError(f"{path}.parameters must be an object")
457
+ return cls(
458
+ entrypoint=_text(data["entrypoint"], f"{path}.entrypoint"),
459
+ fingerprint_sha256=_text(
460
+ data["fingerprint_sha256"],
461
+ f"{path}.fingerprint_sha256",
462
+ ),
463
+ parameters=parameters,
464
+ )
465
+
466
+ @property
467
+ def binding_identity(self) -> Mapping[str, Any]:
468
+ return freeze_mapping(
469
+ {
470
+ "entrypoint": self.entrypoint,
471
+ "fingerprint_sha256": self.fingerprint_sha256,
472
+ "parameters_sha256": content_hash(self.parameters),
473
+ }
474
+ )
475
+
476
+
477
+ @dataclass(frozen=True, slots=True)
478
+ class _BridgeIdentityLock:
479
+ bridge_id: str
480
+ bridge_revision: str
481
+ scorer_id: str
482
+ scorer_revision: str
483
+ dependency_hashes_by_model_slot: Mapping[str, Mapping[str, str]]
484
+ benchmark_id: str = ""
485
+
486
+ def __post_init__(self) -> None:
487
+ for label in ("bridge_id", "scorer_id"):
488
+ _require_text(getattr(self, label), label=f"bridge lock {label}")
489
+ for label in ("bridge_revision", "scorer_revision"):
490
+ _require_sha256(getattr(self, label), label=f"bridge lock {label}")
491
+ raw = self.dependency_hashes_by_model_slot
492
+ if not isinstance(raw, Mapping) or not raw:
493
+ raise ValidationError("bridge lock dependency_hashes_by_model_slot must be non-empty")
494
+ normalized: dict[str, Mapping[str, str]] = {}
495
+ for model_slot, dependencies in raw.items():
496
+ _require_text(model_slot, label="bridge lock model_slot")
497
+ normalized[model_slot] = _validated_dependency_hashes(dependencies, benchmark_id=self.benchmark_id)
498
+ object.__setattr__(
499
+ self,
500
+ "dependency_hashes_by_model_slot",
501
+ MappingProxyType(normalized),
502
+ )
503
+
504
+ @classmethod
505
+ def from_mapping(cls, value: Any, *, path: str, benchmark_id: str = "") -> _BridgeIdentityLock:
506
+ data = _strict_mapping(
507
+ value,
508
+ path=path,
509
+ keys={
510
+ "bridge_id",
511
+ "bridge_revision",
512
+ "scorer_id",
513
+ "scorer_revision",
514
+ "dependency_hashes_by_model_slot",
515
+ },
516
+ required={
517
+ "bridge_id",
518
+ "bridge_revision",
519
+ "scorer_id",
520
+ "scorer_revision",
521
+ "dependency_hashes_by_model_slot",
522
+ },
523
+ )
524
+ dependencies = data["dependency_hashes_by_model_slot"]
525
+ if not isinstance(dependencies, Mapping):
526
+ raise ValidationError(f"{path}.dependency_hashes_by_model_slot must be an object")
527
+ return cls(
528
+ bridge_id=_text(data["bridge_id"], f"{path}.bridge_id"),
529
+ bridge_revision=_text(
530
+ data["bridge_revision"],
531
+ f"{path}.bridge_revision",
532
+ ),
533
+ scorer_id=_text(data["scorer_id"], f"{path}.scorer_id"),
534
+ scorer_revision=_text(
535
+ data["scorer_revision"],
536
+ f"{path}.scorer_revision",
537
+ ),
538
+ dependency_hashes_by_model_slot=dependencies,
539
+ benchmark_id=benchmark_id,
540
+ )
541
+
542
+ def dependencies(self, model_slot: str) -> Mapping[str, str]:
543
+ try:
544
+ return self.dependency_hashes_by_model_slot[model_slot]
545
+ except KeyError as exc:
546
+ raise ValidationError(
547
+ f"bridge lock has no dependency identity for model slot {model_slot!r}"
548
+ ) from exc
549
+
550
+
551
+ @dataclass(frozen=True, slots=True)
552
+ class _OutcomeContract:
553
+ formula_id: str
554
+ official_success_metric: str
555
+
556
+ def __post_init__(self) -> None:
557
+ if self.formula_id != _VALID_SUCCESS_FORMULA_ID:
558
+ raise ValidationError(
559
+ f"external outcome formula_id must be {_VALID_SUCCESS_FORMULA_ID!r}"
560
+ )
561
+ _require_text(
562
+ self.official_success_metric,
563
+ label="external outcome official_success_metric",
564
+ )
565
+ if self.official_success_metric in {
566
+ _CONTRACT_PRESERVED_METRIC,
567
+ _PRIVILEGED_LEAKAGE_METRIC,
568
+ _VALID_SUCCESS_METRIC,
569
+ }:
570
+ raise ValidationError(
571
+ "external official_success_metric conflicts with a host outcome field"
572
+ )
573
+
574
+ @classmethod
575
+ def from_mapping(cls, value: Any, *, path: str) -> _OutcomeContract:
576
+ data = _strict_mapping(
577
+ value,
578
+ path=path,
579
+ keys={"formula_id", "official_success_metric"},
580
+ required={"formula_id", "official_success_metric"},
581
+ )
582
+ return cls(
583
+ formula_id=_text(data["formula_id"], f"{path}.formula_id"),
584
+ official_success_metric=_text(
585
+ data["official_success_metric"],
586
+ f"{path}.official_success_metric",
587
+ ),
588
+ )
589
+
590
+
591
+ @dataclass(frozen=True, slots=True)
592
+ class _TargetBinding:
593
+ benchmark_id: str
594
+ split: str
595
+ source_revision: str
596
+ checkpoint_builder: _LockedBuilderSpec
597
+ bridge_builder: _LockedBuilderSpec
598
+ bridge_lock: _BridgeIdentityLock
599
+ outcome_contract: _OutcomeContract
600
+
601
+ @property
602
+ def target_key(self) -> str:
603
+ return f"{self.benchmark_id}:{self.split}"
604
+
605
+
606
+ @dataclass(frozen=True, slots=True)
607
+ class _ArmBinding:
608
+ release_id: str
609
+ family: SemanticFamily
610
+ profile_kind: ProfileKind
611
+ arm_kind: ExperimentArmKind
612
+ payload_builder: _LockedBuilderSpec
613
+ policy_kind: str | None = None
614
+
615
+ def __post_init__(self) -> None:
616
+ if self.arm_kind is ExperimentArmKind.POLICY:
617
+ expected = _POLICY_KIND_BY_RELEASE.get(self.release_id)
618
+ if expected is None or self.policy_kind != expected:
619
+ raise ValidationError(
620
+ "policy arm binding kind differs from its supported policy release"
621
+ )
622
+ elif self.policy_kind is not None:
623
+ raise ValidationError("policy_kind is valid only for policy arm bindings")
624
+
625
+
626
+ @dataclass(frozen=True, slots=True)
627
+ class _GroupPartitionCommitment:
628
+ members_by_split: Mapping[str, tuple[str, ...]]
629
+ commitments_by_split: Mapping[str, str]
630
+
631
+ def __post_init__(self) -> None:
632
+ members = dict(self.members_by_split)
633
+ commitments = dict(self.commitments_by_split)
634
+ if set(members) != {"train", "calibration"} or set(commitments) != set(members):
635
+ raise ValidationError(
636
+ "policy group partition must contain exact train and calibration splits"
637
+ )
638
+ normalized_members: dict[str, tuple[str, ...]] = {}
639
+ observed: set[str] = set()
640
+ for split in ("train", "calibration"):
641
+ values = tuple(members[split])
642
+ if (
643
+ not values
644
+ or values != tuple(sorted(values))
645
+ or len(values) != len(set(values))
646
+ or any(
647
+ not isinstance(item, str) or _ANONYMIZED_GROUP.fullmatch(item) is None
648
+ for item in values
649
+ )
650
+ ):
651
+ raise ValidationError(
652
+ f"policy group partition {split} members must be sorted unique "
653
+ "privacy-safe group hashes"
654
+ )
655
+ if observed.intersection(values):
656
+ raise ValidationError("policy group partition members cross splits")
657
+ observed.update(values)
658
+ expected = content_hash({"split": split, "group_ids": list(values)})
659
+ if commitments[split] != expected:
660
+ raise ValidationError(
661
+ f"policy group partition {split} commitment differs from its members"
662
+ )
663
+ normalized_members[split] = values
664
+ object.__setattr__(
665
+ self,
666
+ "members_by_split",
667
+ MappingProxyType(normalized_members),
668
+ )
669
+ object.__setattr__(
670
+ self,
671
+ "commitments_by_split",
672
+ MappingProxyType(commitments),
673
+ )
674
+
675
+ @classmethod
676
+ def from_mapping(
677
+ cls,
678
+ value: Any,
679
+ *,
680
+ path: str,
681
+ ) -> _GroupPartitionCommitment:
682
+ data = _strict_mapping(
683
+ value,
684
+ path=path,
685
+ keys={"schema_version", "hash_scheme", "splits"},
686
+ required={"schema_version", "hash_scheme", "splits"},
687
+ )
688
+ if data["schema_version"] != 1 or data["hash_scheme"] != "benchmark_group.content_hash.v1":
689
+ raise ValidationError(f"{path} has an unsupported group commitment schema")
690
+ splits = _strict_mapping(
691
+ data["splits"],
692
+ path=f"{path}.splits",
693
+ keys={"train", "calibration"},
694
+ required={"train", "calibration"},
695
+ )
696
+ members: dict[str, tuple[str, ...]] = {}
697
+ commitments: dict[str, str] = {}
698
+ for split in ("train", "calibration"):
699
+ row = _strict_mapping(
700
+ splits[split],
701
+ path=f"{path}.splits.{split}",
702
+ keys={"member_hashes", "commitment_sha256"},
703
+ required={"member_hashes", "commitment_sha256"},
704
+ )
705
+ raw_members = row["member_hashes"]
706
+ if isinstance(raw_members, (str, bytes)) or not isinstance(
707
+ raw_members,
708
+ Sequence,
709
+ ):
710
+ raise ValidationError(f"{path}.splits.{split}.member_hashes must be an array")
711
+ members[split] = tuple(raw_members)
712
+ commitments[split] = _text(
713
+ row["commitment_sha256"],
714
+ f"{path}.splits.{split}.commitment_sha256",
715
+ )
716
+ return cls(members, commitments)
717
+
718
+ @property
719
+ def excluded_members(self) -> frozenset[str]:
720
+ return frozenset(member for values in self.members_by_split.values() for member in values)
721
+
722
+ @property
723
+ def split_provenance_hash(self) -> str:
724
+ return content_hash({"group_split_digests": dict(self.commitments_by_split)})
725
+
726
+ def to_receipt(self) -> Mapping[str, Any]:
727
+ return freeze_mapping(
728
+ {
729
+ "hash_scheme": "benchmark_group.content_hash.v1",
730
+ "split_commitments": dict(self.commitments_by_split),
731
+ "excluded_member_count": len(self.excluded_members),
732
+ "excluded_members_sha256": content_hash(
733
+ {"member_hashes": sorted(self.excluded_members)}
734
+ ),
735
+ }
736
+ )
737
+
738
+
739
+ @dataclass(frozen=True, slots=True)
740
+ class _SelectorPolicyManifest:
741
+ discovery_revision: str
742
+ discovery_sha256: str
743
+ training_table_sha256: str
744
+ split_provenance_hash: str
745
+ comparison_profile_hash: str
746
+ representative_by_family: Mapping[SemanticFamily, str]
747
+ selected_family: SemanticFamily
748
+ selected_release_id: str
749
+ calibration_split_digest: str
750
+ group_partition: _GroupPartitionCommitment
751
+
752
+ def __post_init__(self) -> None:
753
+ _require_text(self.discovery_revision, label="policy manifest discovery revision")
754
+ for label in (
755
+ "discovery_sha256",
756
+ "training_table_sha256",
757
+ "split_provenance_hash",
758
+ "comparison_profile_hash",
759
+ "calibration_split_digest",
760
+ ):
761
+ _require_sha256(getattr(self, label), label=f"policy manifest {label}")
762
+ representatives = _validated_policy_representatives(self.representative_by_family)
763
+ if self.selected_family not in _CORE_FAMILIES:
764
+ raise ValidationError("policy manifest selected_family must be a core family")
765
+ _require_text(
766
+ self.selected_release_id,
767
+ label="policy manifest selected_release_id",
768
+ )
769
+ if representatives.get(self.selected_family) != self.selected_release_id:
770
+ raise ValidationError(
771
+ "policy manifest selected release differs from its frozen representative"
772
+ )
773
+ if not isinstance(self.group_partition, _GroupPartitionCommitment):
774
+ raise ValidationError("policy manifest requires a typed group partition")
775
+ if self.group_partition.split_provenance_hash != self.split_provenance_hash:
776
+ raise ValidationError("policy group partition differs from split_provenance_hash")
777
+ if (
778
+ self.group_partition.commitments_by_split["calibration"]
779
+ != self.calibration_split_digest
780
+ ):
781
+ raise ValidationError(
782
+ "policy group partition calibration commitment differs from selection receipt"
783
+ )
784
+ object.__setattr__(
785
+ self,
786
+ "representative_by_family",
787
+ MappingProxyType(representatives),
788
+ )
789
+
790
+ @classmethod
791
+ def from_mapping(cls, value: Any, *, path: str) -> _SelectorPolicyManifest:
792
+ data = _strict_mapping(
793
+ value,
794
+ path=path,
795
+ keys={
796
+ "schema_version",
797
+ "artifact",
798
+ "stage",
799
+ "discovery",
800
+ "training_table_sha256",
801
+ "split_provenance_hash",
802
+ "comparison_profile_hash",
803
+ "representative_by_family",
804
+ "group_partition",
805
+ "locked_best_eligible_global",
806
+ },
807
+ required={
808
+ "schema_version",
809
+ "artifact",
810
+ "stage",
811
+ "discovery",
812
+ "training_table_sha256",
813
+ "split_provenance_hash",
814
+ "comparison_profile_hash",
815
+ "representative_by_family",
816
+ "group_partition",
817
+ "locked_best_eligible_global",
818
+ },
819
+ )
820
+ if (
821
+ data["schema_version"] != 1
822
+ or data["artifact"] != "agentinputlab.selector_policy_manifest"
823
+ or data["stage"] != "frozen"
824
+ ):
825
+ raise ValidationError(f"{path} must be a frozen selector policy manifest schema v1")
826
+ discovery = _strict_mapping(
827
+ data["discovery"],
828
+ path=f"{path}.discovery",
829
+ keys={"revision", "sha256"},
830
+ required={"revision", "sha256"},
831
+ )
832
+ locked = _strict_mapping(
833
+ data["locked_best_eligible_global"],
834
+ path=f"{path}.locked_best_eligible_global",
835
+ keys={"selected_family", "selected_release_id", "selection_receipt"},
836
+ required={"selected_family", "selected_release_id", "selection_receipt"},
837
+ )
838
+ receipt = _strict_mapping(
839
+ locked["selection_receipt"],
840
+ path=f"{path}.locked_best_eligible_global.selection_receipt",
841
+ keys={
842
+ "schema_version",
843
+ "rule_id",
844
+ "selection_split",
845
+ "training_table_sha256",
846
+ "calibration_split_digest",
847
+ },
848
+ required={
849
+ "schema_version",
850
+ "rule_id",
851
+ "selection_split",
852
+ "training_table_sha256",
853
+ "calibration_split_digest",
854
+ },
855
+ )
856
+ if (
857
+ receipt["schema_version"] != 1
858
+ or receipt["rule_id"] != "best_eligible_global.v1"
859
+ or receipt["selection_split"] != "calibration"
860
+ or receipt["training_table_sha256"] != data["training_table_sha256"]
861
+ ):
862
+ raise ValidationError(f"{path} locked-best selection receipt is inconsistent")
863
+ representatives = _policy_representatives_from_mapping(
864
+ data["representative_by_family"],
865
+ path=f"{path}.representative_by_family",
866
+ )
867
+ try:
868
+ selected_family = SemanticFamily(
869
+ _text(
870
+ locked["selected_family"],
871
+ f"{path}.locked_best_eligible_global.selected_family",
872
+ )
873
+ )
874
+ except ValueError as exc:
875
+ raise ValidationError(f"{path} locked-best selected_family is unknown") from exc
876
+ return cls(
877
+ discovery_revision=_text(
878
+ discovery["revision"],
879
+ f"{path}.discovery.revision",
880
+ ),
881
+ discovery_sha256=_text(
882
+ discovery["sha256"],
883
+ f"{path}.discovery.sha256",
884
+ ),
885
+ training_table_sha256=_text(
886
+ data["training_table_sha256"],
887
+ f"{path}.training_table_sha256",
888
+ ),
889
+ split_provenance_hash=_text(
890
+ data["split_provenance_hash"],
891
+ f"{path}.split_provenance_hash",
892
+ ),
893
+ comparison_profile_hash=_text(
894
+ data["comparison_profile_hash"],
895
+ f"{path}.comparison_profile_hash",
896
+ ),
897
+ representative_by_family=representatives,
898
+ selected_family=selected_family,
899
+ selected_release_id=_text(
900
+ locked["selected_release_id"],
901
+ f"{path}.locked_best_eligible_global.selected_release_id",
902
+ ),
903
+ calibration_split_digest=_text(
904
+ receipt["calibration_split_digest"],
905
+ (f"{path}.locked_best_eligible_global.selection_receipt.calibration_split_digest"),
906
+ ),
907
+ group_partition=_GroupPartitionCommitment.from_mapping(
908
+ data["group_partition"],
909
+ path=f"{path}.group_partition",
910
+ ),
911
+ )
912
+
913
+ @property
914
+ def manifest_hash(self) -> str:
915
+ return content_hash(self)
916
+
917
+
918
+ @dataclass(frozen=True, slots=True)
919
+ class _DriverConfig:
920
+ driver_id: str
921
+ driver_revision_sha256: str
922
+ baseline_arm_id: str
923
+ benchmark_targets: Mapping[str, _TargetBinding]
924
+ arms: Mapping[str, _ArmBinding]
925
+
926
+ def __post_init__(self) -> None:
927
+ _require_text(self.driver_id, label="external driver_id")
928
+ _require_sha256(
929
+ self.driver_revision_sha256,
930
+ label="external driver_revision_sha256",
931
+ )
932
+ _require_text(self.baseline_arm_id, label="external baseline_arm_id")
933
+ targets = dict(self.benchmark_targets)
934
+ arms = dict(self.arms)
935
+ if not targets or any(key != binding.target_key for key, binding in targets.items()):
936
+ raise ValidationError(
937
+ "external benchmark target keys must be exact cell_id:split identities"
938
+ )
939
+ if not arms or self.baseline_arm_id not in arms:
940
+ raise ValidationError("external driver requires its configured baseline arm")
941
+ baseline = arms[self.baseline_arm_id]
942
+ if (
943
+ baseline.release_id != "baseline.no_op"
944
+ or baseline.family is not SemanticFamily.NO_OP
945
+ or baseline.arm_kind is not ExperimentArmKind.RELEASE
946
+ ):
947
+ raise ValidationError(
948
+ "external driver baseline must bind baseline.no_op in the no_op family"
949
+ )
950
+ object.__setattr__(self, "benchmark_targets", MappingProxyType(targets))
951
+ object.__setattr__(self, "arms", MappingProxyType(arms))
952
+
953
+ @classmethod
954
+ def from_mapping(cls, value: Any) -> _DriverConfig:
955
+ data = _strict_mapping(
956
+ value,
957
+ path="external_driver_config",
958
+ keys={
959
+ "schema_version",
960
+ "driver_id",
961
+ "driver_revision_sha256",
962
+ "baseline_arm_id",
963
+ "benchmark_targets",
964
+ "arms",
965
+ },
966
+ required={
967
+ "schema_version",
968
+ "driver_id",
969
+ "driver_revision_sha256",
970
+ "baseline_arm_id",
971
+ "benchmark_targets",
972
+ "arms",
973
+ },
974
+ )
975
+ if data["schema_version"] != _CONFIG_SCHEMA_VERSION:
976
+ raise ValidationError("unsupported external arena config schema_version")
977
+ raw_targets = _nonempty_object(
978
+ data["benchmark_targets"],
979
+ path="external_driver_config.benchmark_targets",
980
+ )
981
+ raw_arms = _nonempty_object(
982
+ data["arms"],
983
+ path="external_driver_config.arms",
984
+ )
985
+ targets = {
986
+ key: _target_binding(
987
+ item,
988
+ path=f"external_driver_config.benchmark_targets[{key!r}]",
989
+ )
990
+ for key, item in raw_targets.items()
991
+ }
992
+ arms = {
993
+ key: _arm_binding(
994
+ item,
995
+ path=f"external_driver_config.arms[{key!r}]",
996
+ )
997
+ for key, item in raw_arms.items()
998
+ }
999
+ return cls(
1000
+ driver_id=_text(data["driver_id"], "external_driver_config.driver_id"),
1001
+ driver_revision_sha256=_text(
1002
+ data["driver_revision_sha256"],
1003
+ "external_driver_config.driver_revision_sha256",
1004
+ ),
1005
+ baseline_arm_id=_text(
1006
+ data["baseline_arm_id"],
1007
+ "external_driver_config.baseline_arm_id",
1008
+ ),
1009
+ benchmark_targets=targets,
1010
+ arms=arms,
1011
+ )
1012
+
1013
+
1014
+ class ExternalArenaExperimentDriver:
1015
+ """Execute exact multi-arm cells through locked atomic external bridges."""
1016
+
1017
+ def __init__(self, config: _DriverConfig, *, config_sha256: str) -> None:
1018
+ if not isinstance(config, _DriverConfig):
1019
+ raise TypeError("external arena driver requires parsed configuration")
1020
+ _require_sha256(config_sha256, label="external driver config_sha256")
1021
+ self._config = config
1022
+ self._config_sha256 = config_sha256
1023
+ self._feature_extractor = VisibleStateFeatureExtractor()
1024
+ self._preflight_manifest_hash: str | None = None
1025
+ self._policy_binding_receipts: Mapping[str, Mapping[str, Any]] = MappingProxyType({})
1026
+ self._gate_binding_receipts: Mapping[str, Mapping[str, Any]] = MappingProxyType({})
1027
+
1028
+ @classmethod
1029
+ def from_file(
1030
+ cls,
1031
+ path: str | Path,
1032
+ *,
1033
+ expected_sha256: str,
1034
+ ) -> ExternalArenaExperimentDriver:
1035
+ """Load exact UTF-8 JSON bytes after checking their caller-supplied hash."""
1036
+
1037
+ _require_sha256(expected_sha256, label="expected external config SHA-256")
1038
+ source = Path(path).expanduser()
1039
+ if not source.is_absolute():
1040
+ raise ArtifactError("external arena config path must be absolute")
1041
+ try:
1042
+ if source.stat().st_size > _MAX_CONFIG_BYTES:
1043
+ raise ArtifactError("external arena config exceeds the size limit")
1044
+ raw = source.read_bytes()
1045
+ except OSError as exc:
1046
+ raise ArtifactError("cannot read external arena driver config") from exc
1047
+ observed = hashlib.sha256(raw).hexdigest()
1048
+ if observed != expected_sha256:
1049
+ raise ArtifactError("external arena driver config checksum mismatch")
1050
+ return cls(_DriverConfig.from_mapping(_strict_json(raw)), config_sha256=observed)
1051
+
1052
+ def fingerprint(self) -> Mapping[str, Any]:
1053
+ return freeze_mapping(
1054
+ {
1055
+ "implementation_id": _DRIVER_IMPLEMENTATION_ID,
1056
+ "schema_version": _CONFIG_SCHEMA_VERSION,
1057
+ "driver_id": self._config.driver_id,
1058
+ "driver_revision_sha256": self._config.driver_revision_sha256,
1059
+ "config_sha256": self._config_sha256,
1060
+ "baseline_arm_id": self._config.baseline_arm_id,
1061
+ "benchmark_targets": {
1062
+ key: {
1063
+ "benchmark_id": binding.benchmark_id,
1064
+ "split": binding.split,
1065
+ "source_revision": binding.source_revision,
1066
+ "checkpoint_builder": binding.checkpoint_builder.binding_identity,
1067
+ "bridge_builder": binding.bridge_builder.binding_identity,
1068
+ "bridge_lock": to_jsonable(binding.bridge_lock),
1069
+ "outcome_contract": to_jsonable(binding.outcome_contract),
1070
+ }
1071
+ for key, binding in self._config.benchmark_targets.items()
1072
+ },
1073
+ "arms": {
1074
+ arm_id: {
1075
+ "release_id": binding.release_id,
1076
+ "family": binding.family.value,
1077
+ "profile_kind": binding.profile_kind.value,
1078
+ "arm_kind": binding.arm_kind.value,
1079
+ "payload_builder": binding.payload_builder.binding_identity,
1080
+ "policy_kind": binding.policy_kind,
1081
+ }
1082
+ for arm_id, binding in self._config.arms.items()
1083
+ },
1084
+ "resolved_policy_bindings": self._policy_binding_receipts,
1085
+ "resolved_gate_bindings": self._gate_binding_receipts,
1086
+ "feature_schema_id": self._feature_extractor.feature_schema_id,
1087
+ "feature_schema_hash": self._feature_extractor.feature_schema_hash,
1088
+ # Static capability attestation for Studio's second,
1089
+ # execution-confirmation gate. Successful preflight still has
1090
+ # to produce the concrete policy/gate receipts below; these
1091
+ # booleans only state which fail-closed checks this driver
1092
+ # implementation enforces.
1093
+ "studio_campaign_attestation": {
1094
+ "schema_version": 1,
1095
+ "shared_dependency_slots": True,
1096
+ "selector_group_exclusion": True,
1097
+ "typed_gate_receipts": True,
1098
+ },
1099
+ }
1100
+ )
1101
+
1102
+ def preflight(self, manifest: ExperimentManifest) -> Sequence[str]:
1103
+ """Validate locks and builders without constructing checkpoints or bridges."""
1104
+
1105
+ self._preflight_manifest_hash = None
1106
+ self._policy_binding_receipts = MappingProxyType({})
1107
+ self._gate_binding_receipts = MappingProxyType({})
1108
+ blockers = self._manifest_blockers(manifest)
1109
+ if blockers:
1110
+ return tuple(blockers)
1111
+ used_specs: dict[tuple[str, str, str, str], _LockedBuilderSpec] = {}
1112
+ for target in manifest.benchmarks:
1113
+ binding = self._config.benchmark_targets[_target_key(target)]
1114
+ for kind, spec in (
1115
+ ("arena_checkpoint", binding.checkpoint_builder),
1116
+ ("atomic_bridge", binding.bridge_builder),
1117
+ ):
1118
+ used_specs[
1119
+ (
1120
+ kind,
1121
+ spec.entrypoint,
1122
+ spec.fingerprint_sha256,
1123
+ content_hash(spec.parameters),
1124
+ )
1125
+ ] = spec
1126
+ selected_arm_ids = {
1127
+ arm_id
1128
+ for target in manifest.benchmarks
1129
+ for arm_id in (target.arm_ids or tuple(arm.arm_id for arm in manifest.arms))
1130
+ }
1131
+ for arm_id in selected_arm_ids:
1132
+ spec = self._config.arms[arm_id].payload_builder
1133
+ used_specs[
1134
+ (
1135
+ "arm_payload",
1136
+ spec.entrypoint,
1137
+ spec.fingerprint_sha256,
1138
+ content_hash(spec.parameters),
1139
+ )
1140
+ ] = spec
1141
+ for (kind, _entrypoint, _fingerprint, _parameters), spec in used_specs.items():
1142
+ try:
1143
+ builder = _resolve_locked_builder(spec, expected_kind=kind)
1144
+ messages = _builder_preflight(builder, spec.parameters)
1145
+ except Exception as exc:
1146
+ blockers.append(f"{kind} builder: {_safe_message(exc)}")
1147
+ continue
1148
+ blockers.extend(f"{kind} builder: {_safe_text(item)}" for item in messages)
1149
+ try:
1150
+ policy_receipts = self._resolve_policy_bindings(manifest)
1151
+ except Exception as exc:
1152
+ blockers.append("policy binding: " + _safe_message(exc))
1153
+ policy_receipts = MappingProxyType({})
1154
+ try:
1155
+ gate_receipts = self._resolve_gate_bindings(manifest)
1156
+ except Exception as exc:
1157
+ blockers.append("external gate binding: " + _safe_message(exc))
1158
+ gate_receipts = MappingProxyType({})
1159
+ if blockers:
1160
+ return tuple(blockers)
1161
+ self._policy_binding_receipts = policy_receipts
1162
+ self._gate_binding_receipts = gate_receipts
1163
+ self._preflight_manifest_hash = manifest.manifest_hash
1164
+ return ()
1165
+
1166
+ def preflight_units(
1167
+ self,
1168
+ manifest: ExperimentManifest,
1169
+ resolved_units: Mapping[str, Sequence[str]],
1170
+ ) -> Sequence[str]:
1171
+ """Ask each locked checkpoint builder to recognize the exact planned IDs."""
1172
+
1173
+ if not isinstance(manifest, ExperimentManifest):
1174
+ return ("unit preflight requires an ExperimentManifest",)
1175
+ requirement = manifest.metadata.get("require_driver_unit_preflight")
1176
+ if requirement is not None and not isinstance(requirement, bool):
1177
+ return ("require_driver_unit_preflight metadata must be boolean when supplied",)
1178
+ required = requirement is True
1179
+ if not isinstance(resolved_units, Mapping):
1180
+ return ("resolved unit inventory must be a mapping",)
1181
+ expected_keys = {_target_key(target) for target in manifest.benchmarks}
1182
+ if set(resolved_units) != expected_keys:
1183
+ return ("resolved unit inventories differ from the manifest targets",)
1184
+ blockers: list[str] = []
1185
+ forbidden_group_hashes = self._policy_excluded_group_hashes()
1186
+ for target in manifest.benchmarks:
1187
+ key = _target_key(target)
1188
+ binding = self._config.benchmark_targets.get(key)
1189
+ if binding is None:
1190
+ blockers.append(f"benchmark {key}: no locked external target binding")
1191
+ continue
1192
+ try:
1193
+ unit_ids = tuple(sorted(resolved_units[key]))
1194
+ request = ExternalUnitPreflightRequest(
1195
+ benchmark_id=target.cell_id,
1196
+ split=target.split,
1197
+ source_revision=binding.source_revision,
1198
+ unit_ids=unit_ids,
1199
+ builder_parameters=binding.checkpoint_builder.parameters,
1200
+ )
1201
+ builder = _resolve_locked_builder(
1202
+ binding.checkpoint_builder,
1203
+ expected_kind="arena_checkpoint",
1204
+ )
1205
+ callback = getattr(builder, "preflight_units", None)
1206
+ if callback is None:
1207
+ if required:
1208
+ blockers.append(
1209
+ f"benchmark {key}: checkpoint builder lacks required "
1210
+ "read-only preflight_units"
1211
+ )
1212
+ continue
1213
+ if not callable(callback):
1214
+ raise ValidationError("checkpoint builder preflight_units must be callable")
1215
+ messages = callback(request)
1216
+ if (
1217
+ isinstance(messages, (str, bytes))
1218
+ or not isinstance(messages, Sequence)
1219
+ or any(not isinstance(item, str) or not item.strip() for item in messages)
1220
+ ):
1221
+ raise ValidationError(
1222
+ "checkpoint builder preflight_units must return a "
1223
+ "sequence of non-empty blocker strings"
1224
+ )
1225
+ blockers.extend(f"benchmark {key}: {_safe_text(item)}" for item in messages)
1226
+ if forbidden_group_hashes:
1227
+ group_callback = getattr(builder, "preflight_group_ids", None)
1228
+ if not callable(group_callback):
1229
+ blockers.append(
1230
+ f"benchmark {key}: checkpoint builder lacks required "
1231
+ "read-only preflight_group_ids for selector leakage exclusion"
1232
+ )
1233
+ continue
1234
+ raw_groups = group_callback(request)
1235
+ if not isinstance(raw_groups, Mapping) or set(raw_groups) != set(unit_ids):
1236
+ raise ValidationError(
1237
+ "checkpoint builder preflight_group_ids must return an "
1238
+ "exact unit_id-to-group_id mapping"
1239
+ )
1240
+ for unit_id, group_id in raw_groups.items():
1241
+ _require_text(
1242
+ group_id,
1243
+ label=f"checkpoint group for unit {unit_id!r}",
1244
+ )
1245
+ group_hash = _anonymized_group_hash(
1246
+ target.cell_id,
1247
+ group_id,
1248
+ )
1249
+ if group_hash in forbidden_group_hashes:
1250
+ blockers.append(
1251
+ f"benchmark {key}: unit {unit_id!r} belongs to a "
1252
+ "selector discovery train/calibration group"
1253
+ )
1254
+ except Exception as exc:
1255
+ blockers.append(f"benchmark {key}: {_safe_message(exc)}")
1256
+ return tuple(blockers)
1257
+
1258
+ def execute_cell(
1259
+ self,
1260
+ manifest: ExperimentManifest,
1261
+ cell: ExperimentCell,
1262
+ ) -> ExperimentCellResult:
1263
+ """Execute a complete cell or raise before any terminal result is written."""
1264
+
1265
+ if self._preflight_manifest_hash != manifest.manifest_hash:
1266
+ raise ArtifactError(
1267
+ "external arena driver requires successful preflight for this manifest"
1268
+ )
1269
+ if self._resolve_policy_bindings(manifest) != self._policy_binding_receipts:
1270
+ raise ArtifactError("external policy artifact identity changed after preflight")
1271
+ if self._resolve_gate_bindings(manifest) != self._gate_binding_receipts:
1272
+ raise ArtifactError("external gate artifact identity changed after preflight")
1273
+ target, target_binding = self._validate_cell(manifest, cell)
1274
+ expected_dependencies = target_binding.bridge_lock.dependencies(cell.model_slot)
1275
+ checkpoint = self._build_checkpoint(cell, target_binding)
1276
+ self._validate_checkpoint(
1277
+ checkpoint,
1278
+ cell=cell,
1279
+ binding=target_binding,
1280
+ expected_dependencies=expected_dependencies,
1281
+ )
1282
+ checkpoint_group_hash = _anonymized_group_hash(
1283
+ checkpoint.benchmark_id,
1284
+ checkpoint.group_id,
1285
+ )
1286
+ if checkpoint_group_hash in self._policy_excluded_group_hashes():
1287
+ raise ValidationError(
1288
+ "external checkpoint belongs to a selector discovery train/calibration group"
1289
+ )
1290
+ pre_action = self._pre_action_receipt(checkpoint.visible_state)
1291
+ bridge = self._build_bridge(
1292
+ checkpoint,
1293
+ cell=cell,
1294
+ binding=target_binding,
1295
+ )
1296
+ self._validate_bridge(
1297
+ bridge,
1298
+ binding=target_binding,
1299
+ expected_dependencies=expected_dependencies,
1300
+ )
1301
+
1302
+ arm_specs = {arm.arm_id: arm for arm in manifest.arms}
1303
+ artifacts = {
1304
+ arm_id: self._build_arm_payload(
1305
+ checkpoint,
1306
+ cell=cell,
1307
+ arm_spec=arm_specs[arm_id],
1308
+ pre_action=pre_action,
1309
+ )
1310
+ for arm_id in cell.arm_ids
1311
+ }
1312
+ self._validate_artifact_set(
1313
+ cell,
1314
+ checkpoint=checkpoint,
1315
+ arm_specs=arm_specs,
1316
+ artifacts=artifacts,
1317
+ )
1318
+ baseline_id = self._config.baseline_arm_id
1319
+ baseline_artifact = artifacts[baseline_id]
1320
+ pair_records: list[tuple[str, AtomicReplayRequest, AtomicPairedReplayRecord]] = []
1321
+ baseline_consistency: str | None = None
1322
+ for treatment_id in cell.arm_ids:
1323
+ if treatment_id == baseline_id:
1324
+ continue
1325
+ request = AtomicReplayRequest(
1326
+ benchmark_id=cell.benchmark_id,
1327
+ checkpoint_id=cell.unit_id,
1328
+ source_revision=target_binding.source_revision,
1329
+ origin_state_hash=checkpoint.origin_state_hash,
1330
+ replay_identity=checkpoint.replay_identity,
1331
+ seed=cell.seed,
1332
+ locked_dependency_hashes=expected_dependencies,
1333
+ arms=(
1334
+ AtomicReplayArmRequest(
1335
+ role="baseline",
1336
+ arm_id=baseline_id,
1337
+ payload=baseline_artifact.payload,
1338
+ ),
1339
+ AtomicReplayArmRequest(
1340
+ role="treatment",
1341
+ arm_id=treatment_id,
1342
+ payload=artifacts[treatment_id].payload,
1343
+ ),
1344
+ ),
1345
+ )
1346
+ record = AtomicExternalReplayRunner().run(bridge, request)
1347
+ observed_consistency = _baseline_consistency_hash(record)
1348
+ if baseline_consistency is None:
1349
+ baseline_consistency = observed_consistency
1350
+ elif observed_consistency != baseline_consistency:
1351
+ raise ValidationError(
1352
+ "atomic multi-arm NO_OP metrics, trace, usage, or logical "
1353
+ "receipt changed across treatment pairs"
1354
+ )
1355
+ pair_records.append((treatment_id, request, record))
1356
+ if len(pair_records) != len(cell.arm_ids) - 1 or baseline_consistency is None:
1357
+ raise ValidationError("external arena driver produced a partial pair inventory")
1358
+
1359
+ arms = self._arm_outcomes(
1360
+ cell,
1361
+ arm_specs=arm_specs,
1362
+ artifacts=artifacts,
1363
+ pair_records=pair_records,
1364
+ target_binding=target_binding,
1365
+ )
1366
+ if tuple(item["arm_id"] for item in arms) != cell.arm_ids:
1367
+ raise ValidationError("external arena driver produced a partial arm inventory")
1368
+ atomic_pairs = tuple(
1369
+ freeze_mapping(
1370
+ {
1371
+ "treatment_arm_id": treatment_id,
1372
+ "request_sha256": content_hash(request),
1373
+ "receipt": to_jsonable(record.receipt),
1374
+ "receipt_sha256": content_hash(record.receipt),
1375
+ "baseline_consistency_sha256": baseline_consistency,
1376
+ "metric_deltas": record.metric_deltas,
1377
+ "resource_receipt_sha256s": tuple(
1378
+ content_hash(receipt) for receipt in record.resource_receipts
1379
+ ),
1380
+ }
1381
+ )
1382
+ for treatment_id, request, record in pair_records
1383
+ )
1384
+ resource_receipts = tuple(
1385
+ receipt
1386
+ for _treatment_id, _request, record in pair_records
1387
+ for receipt in record.resource_receipts
1388
+ )
1389
+ payload: dict[str, Any] = {
1390
+ "benchmark_id": cell.benchmark_id,
1391
+ "split": cell.split,
1392
+ "unit_id": cell.unit_id,
1393
+ "model_slot": cell.model_slot,
1394
+ "seed": cell.seed,
1395
+ "baseline_arm_id": baseline_id,
1396
+ "planned_arm_ids": cell.arm_ids,
1397
+ "checkpoint_identity": {
1398
+ "identity_sha256": checkpoint.identity_hash,
1399
+ "group_sha256": checkpoint_group_hash,
1400
+ "visible_state_sha256": checkpoint.visible_state.state_hash,
1401
+ "origin_state_sha256": checkpoint.origin_state_hash,
1402
+ "replay_identity_sha256": checkpoint.replay_identity,
1403
+ "source_revision": checkpoint.source_revision,
1404
+ },
1405
+ "pre_action": pre_action,
1406
+ "arms": arms,
1407
+ "atomic_pairs": atomic_pairs,
1408
+ "baseline_consistency_sha256": baseline_consistency,
1409
+ "observed_dependency_hashes": expected_dependencies,
1410
+ "not_applicable_dependency_slots": tuple(
1411
+ sorted(
1412
+ slot
1413
+ for slot, digest in expected_dependencies.items()
1414
+ if digest == _NOT_APPLICABLE_DEPENDENCY_HASHES[slot]
1415
+ )
1416
+ ),
1417
+ "target_source_revision": target.source_revision,
1418
+ "atomic_replay_contract": ATOMIC_EXTERNAL_PAIR_CONTRACT,
1419
+ }
1420
+ gate_receipt = self._gate_binding_receipts.get(_target_key(target))
1421
+ if gate_receipt is not None:
1422
+ payload["external_gate_receipt"] = gate_receipt
1423
+ if manifest.metadata.get("selector_training_allowed") is True:
1424
+ payload["selector_record"] = {
1425
+ "schema_version": 1,
1426
+ "state": checkpoint.visible_state.canonical_dict(),
1427
+ "group_id": checkpoint.group_id,
1428
+ "eligible_families": tuple(
1429
+ sorted(
1430
+ {
1431
+ arm_specs[arm_id].family.value
1432
+ for arm_id in cell.arm_ids
1433
+ if arm_specs[arm_id].family in _CORE_FAMILIES
1434
+ }
1435
+ )
1436
+ ),
1437
+ }
1438
+ return ExperimentCellResult(
1439
+ cell_id=cell.cell_id,
1440
+ status="succeeded",
1441
+ payload=payload,
1442
+ resource_receipts=resource_receipts,
1443
+ )
1444
+
1445
+ def _policy_excluded_group_hashes(self) -> frozenset[str]:
1446
+ members: set[str] = set()
1447
+ for receipt in self._policy_binding_receipts.values():
1448
+ partition = receipt.get("group_partition")
1449
+ if not isinstance(partition, Mapping):
1450
+ continue
1451
+ raw = partition.get("excluded_member_hashes")
1452
+ if isinstance(raw, (str, bytes)) or not isinstance(raw, Sequence):
1453
+ raise ArtifactError("resolved policy group partition is invalid")
1454
+ for item in raw:
1455
+ if not isinstance(item, str) or _ANONYMIZED_GROUP.fullmatch(item) is None:
1456
+ raise ArtifactError("resolved policy group member hash is invalid")
1457
+ members.add(item)
1458
+ return frozenset(members)
1459
+
1460
+ def _resolve_policy_bindings(
1461
+ self,
1462
+ manifest: ExperimentManifest,
1463
+ ) -> Mapping[str, Mapping[str, Any]]:
1464
+ receipts: dict[str, Mapping[str, Any]] = {}
1465
+ configured_releases = {
1466
+ binding.release_id: binding.family
1467
+ for binding in self._config.arms.values()
1468
+ if binding.arm_kind is ExperimentArmKind.RELEASE
1469
+ }
1470
+ for arm in manifest.arms:
1471
+ if arm.arm_kind is not ExperimentArmKind.POLICY:
1472
+ continue
1473
+ binding = self._config.arms[arm.arm_id]
1474
+ parameters = dict(arm.parameters)
1475
+ common_fields = {
1476
+ "policy_kind",
1477
+ "policy_manifest_env",
1478
+ "policy_manifest_sha256_env",
1479
+ }
1480
+ selector_fields = {
1481
+ "selector_identity_env",
1482
+ "selector_bundle_env",
1483
+ "selector_bundle_sha256_env",
1484
+ }
1485
+ required = (
1486
+ common_fields | selector_fields
1487
+ if binding.policy_kind == "frozen_selector"
1488
+ else common_fields
1489
+ )
1490
+ if set(parameters) != required:
1491
+ raise ValidationError(f"policy arm {arm.arm_id!r} binding fields are invalid")
1492
+ if parameters["policy_kind"] != binding.policy_kind:
1493
+ raise ValidationError(
1494
+ f"policy arm {arm.arm_id!r} kind differs from the locked config"
1495
+ )
1496
+ policy_bytes, policy_file_sha256 = _read_manifest_file_binding(
1497
+ manifest,
1498
+ environment_name=parameters["policy_manifest_env"],
1499
+ checksum_environment_name=parameters["policy_manifest_sha256_env"],
1500
+ label=f"policy arm {arm.arm_id!r} manifest",
1501
+ )
1502
+ policy_manifest = _SelectorPolicyManifest.from_mapping(
1503
+ _strict_json(policy_bytes),
1504
+ path=f"policy arm {arm.arm_id!r} manifest",
1505
+ )
1506
+ for family, release_id in policy_manifest.representative_by_family.items():
1507
+ if configured_releases.get(release_id) is not family:
1508
+ raise ValidationError(
1509
+ f"policy arm {arm.arm_id!r} representative {release_id!r} "
1510
+ "is outside the locked release bindings"
1511
+ )
1512
+ receipt: dict[str, Any] = {
1513
+ "policy_kind": binding.policy_kind,
1514
+ "policy_manifest_file_sha256": policy_file_sha256,
1515
+ "policy_manifest_hash": policy_manifest.manifest_hash,
1516
+ "discovery": {
1517
+ "revision": policy_manifest.discovery_revision,
1518
+ "sha256": policy_manifest.discovery_sha256,
1519
+ },
1520
+ "training_table_sha256": policy_manifest.training_table_sha256,
1521
+ "split_provenance_hash": policy_manifest.split_provenance_hash,
1522
+ "comparison_profile_hash": (policy_manifest.comparison_profile_hash),
1523
+ "group_partition": {
1524
+ **dict(policy_manifest.group_partition.to_receipt()),
1525
+ "excluded_member_hashes": tuple(
1526
+ sorted(policy_manifest.group_partition.excluded_members)
1527
+ ),
1528
+ },
1529
+ "representative_by_family": {
1530
+ family.value: release_id
1531
+ for family, release_id in sorted(
1532
+ policy_manifest.representative_by_family.items(),
1533
+ key=lambda item: item[0].value,
1534
+ )
1535
+ },
1536
+ }
1537
+ if binding.policy_kind == "locked_best_eligible_global":
1538
+ receipt["selected_representative"] = {
1539
+ "family": policy_manifest.selected_family.value,
1540
+ "release_id": policy_manifest.selected_release_id,
1541
+ "calibration_split_digest": (policy_manifest.calibration_split_digest),
1542
+ }
1543
+ else:
1544
+ bundle_bytes, bundle_file_sha256 = _read_manifest_file_binding(
1545
+ manifest,
1546
+ environment_name=parameters["selector_bundle_env"],
1547
+ checksum_environment_name=parameters["selector_bundle_sha256_env"],
1548
+ label=f"policy arm {arm.arm_id!r} selector bundle",
1549
+ )
1550
+ bundle = _load_frozen_selector_bundle(
1551
+ bundle_bytes,
1552
+ identity_environment_name=parameters["selector_identity_env"],
1553
+ manifest=manifest,
1554
+ label=f"policy arm {arm.arm_id!r} selector bundle",
1555
+ )
1556
+ _validate_selector_policy_provenance(
1557
+ bundle,
1558
+ policy_manifest=policy_manifest,
1559
+ label=f"policy arm {arm.arm_id!r}",
1560
+ )
1561
+ receipt["selector_bundle"] = {
1562
+ "file_sha256": bundle_file_sha256,
1563
+ "selector_id": bundle.selector_card_id,
1564
+ "revision": bundle.revision,
1565
+ "bundle_hash": bundle.bundle_hash,
1566
+ "model_hash": bundle.model_hash,
1567
+ "stage": bundle.stage.value,
1568
+ }
1569
+ receipts[arm.arm_id] = freeze_mapping(receipt)
1570
+ return MappingProxyType(receipts)
1571
+
1572
+ def _resolve_gate_bindings(
1573
+ self,
1574
+ manifest: ExperimentManifest,
1575
+ ) -> Mapping[str, Mapping[str, Any]]:
1576
+ receipts: dict[str, Mapping[str, Any]] = {}
1577
+ for target in manifest.benchmarks:
1578
+ gate_binding = _GATE_BINDINGS.get(target.cell_id)
1579
+ if gate_binding is None:
1580
+ continue
1581
+ environment_name, checksum_environment_name, reset_mode = gate_binding
1582
+ raw, file_sha256 = _read_manifest_file_binding(
1583
+ manifest,
1584
+ environment_name=environment_name,
1585
+ checksum_environment_name=checksum_environment_name,
1586
+ label=f"benchmark {target.cell_id!r} external gate receipt",
1587
+ )
1588
+ binding = self._config.benchmark_targets[_target_key(target)]
1589
+ inventory_sha256 = os.environ.get(target.unit_inventory_sha256_env)
1590
+ if not inventory_sha256:
1591
+ raise ArtifactError(
1592
+ f"benchmark {target.cell_id!r} unit inventory SHA-256 is missing"
1593
+ )
1594
+ _require_sha256(
1595
+ inventory_sha256,
1596
+ label=f"benchmark {target.cell_id!r} unit inventory SHA-256",
1597
+ )
1598
+ receipt = _validate_external_gate_receipt(
1599
+ _strict_json(raw),
1600
+ benchmark_id=target.cell_id,
1601
+ split=target.split,
1602
+ source_revision=binding.source_revision,
1603
+ manifest_hash=manifest.manifest_hash,
1604
+ driver_config_sha256=self._config_sha256,
1605
+ unit_inventory_sha256=inventory_sha256,
1606
+ bridge_lock=binding.bridge_lock,
1607
+ reset_mode=reset_mode,
1608
+ )
1609
+ receipts[_target_key(target)] = freeze_mapping(
1610
+ {
1611
+ "file_sha256": file_sha256,
1612
+ "receipt_sha256": content_hash(receipt),
1613
+ "scope_sha256": receipt["approval"]["scope_sha256"],
1614
+ "benchmark_id": target.cell_id,
1615
+ "split": target.split,
1616
+ "source_revision": binding.source_revision,
1617
+ "manifest_hash": manifest.manifest_hash,
1618
+ "driver_config_sha256": self._config_sha256,
1619
+ "bridge": receipt["bridge"],
1620
+ "scorer": receipt["scorer"],
1621
+ "approval": receipt["approval"],
1622
+ "verification": receipt["verification"],
1623
+ "unit_inventory_sha256": inventory_sha256,
1624
+ }
1625
+ )
1626
+ return MappingProxyType(receipts)
1627
+
1628
+ def _manifest_blockers(self, manifest: ExperimentManifest) -> list[str]:
1629
+ if not isinstance(manifest, ExperimentManifest):
1630
+ return ["manifest must be an ExperimentManifest"]
1631
+ blockers: list[str] = []
1632
+ if manifest.estimand_profile_kind is ProfileKind.OFFICIAL_NATIVE:
1633
+ blockers.append(
1634
+ "external arena driver supports controlled_adapter or "
1635
+ "factorial_adapter estimands, not official_native"
1636
+ )
1637
+ manifest_arms = {arm.arm_id: arm for arm in manifest.arms}
1638
+ if self._config.baseline_arm_id not in manifest_arms:
1639
+ blockers.append("manifest omits the configured NO_OP baseline")
1640
+ elif manifest_arms[self._config.baseline_arm_id].family is not SemanticFamily.NO_OP:
1641
+ blockers.append("configured baseline arm is not no_op in the manifest")
1642
+ required_payload_builder = manifest.metadata.get("required_payload_builder")
1643
+ if required_payload_builder is not None and (
1644
+ not isinstance(required_payload_builder, str)
1645
+ or _ENTRYPOINT.fullmatch(required_payload_builder) is None
1646
+ ):
1647
+ blockers.append("manifest required_payload_builder is not an entrypoint")
1648
+ for arm_id, arm in manifest_arms.items():
1649
+ binding = self._config.arms.get(arm_id)
1650
+ if binding is None:
1651
+ blockers.append(f"arm {arm_id}: no locked payload binding")
1652
+ continue
1653
+ if (
1654
+ binding.release_id != arm.release_id
1655
+ or binding.family is not arm.family
1656
+ or binding.profile_kind is not arm.profile_kind
1657
+ or binding.arm_kind is not arm.arm_kind
1658
+ ):
1659
+ blockers.append(
1660
+ f"arm {arm_id}: manifest identity differs from locked payload binding"
1661
+ )
1662
+ elif (
1663
+ arm.arm_kind is ExperimentArmKind.POLICY
1664
+ and arm.parameters.get("policy_kind") != binding.policy_kind
1665
+ ):
1666
+ blockers.append(
1667
+ f"arm {arm_id}: manifest policy kind differs from locked payload binding"
1668
+ )
1669
+ elif (
1670
+ arm.release_id == "baseline.no_op_cost_shadow"
1671
+ and required_payload_builder is not None
1672
+ and binding.payload_builder.entrypoint != required_payload_builder
1673
+ ):
1674
+ blockers.append(
1675
+ f"arm {arm_id}: payload builder differs from the manifest-required "
1676
+ "cost-shadow builder"
1677
+ )
1678
+ for target in manifest.benchmarks:
1679
+ key = _target_key(target)
1680
+ binding = self._config.benchmark_targets.get(key)
1681
+ if binding is None:
1682
+ blockers.append(f"benchmark {key}: no locked external target binding")
1683
+ continue
1684
+ if target.cell_id not in _SUPPORTED_ARENAS:
1685
+ blockers.append(f"benchmark {target.cell_id}: unsupported external arena")
1686
+ if target.cell_id in _GATE_BINDINGS and (
1687
+ target.external is not True
1688
+ or target.unit_inventory_env is None
1689
+ or target.unit_inventory_sha256_env is None
1690
+ ):
1691
+ blockers.append(
1692
+ f"benchmark {target.cell_id}: gated external arena requires "
1693
+ "external=true and a checksum-bound unit inventory"
1694
+ )
1695
+ if (
1696
+ target.source_revision is not None
1697
+ and target.source_revision != binding.source_revision
1698
+ ):
1699
+ blockers.append(
1700
+ f"benchmark {target.cell_id}: manifest source revision differs "
1701
+ "from the external driver lock"
1702
+ )
1703
+ selected = target.arm_ids or tuple(manifest_arms)
1704
+ if self._config.baseline_arm_id not in selected:
1705
+ blockers.append(
1706
+ f"benchmark {target.cell_id}: selected arms omit the NO_OP baseline"
1707
+ )
1708
+ unknown = sorted(set(selected) - set(self._config.arms))
1709
+ if unknown:
1710
+ blockers.append(
1711
+ f"benchmark {target.cell_id}: selected arms lack payload bindings: "
1712
+ + ", ".join(unknown)
1713
+ )
1714
+ for model_slot in manifest.model_slots:
1715
+ if model_slot not in binding.bridge_lock.dependency_hashes_by_model_slot:
1716
+ blockers.append(
1717
+ f"benchmark {target.cell_id}: bridge lock has no model slot {model_slot}"
1718
+ )
1719
+ return blockers
1720
+
1721
+ def _validate_cell(
1722
+ self,
1723
+ manifest: ExperimentManifest,
1724
+ cell: ExperimentCell,
1725
+ ) -> tuple[ExperimentBenchmarkTarget, _TargetBinding]:
1726
+ if not isinstance(cell, ExperimentCell):
1727
+ raise TypeError("external arena driver requires ExperimentCell")
1728
+ if cell.manifest_hash != manifest.manifest_hash:
1729
+ raise ValidationError("external arena cell belongs to another manifest")
1730
+ matches = tuple(
1731
+ target
1732
+ for target in manifest.benchmarks
1733
+ if target.cell_id == cell.benchmark_id and target.split == cell.split
1734
+ )
1735
+ if len(matches) != 1:
1736
+ raise ValidationError("external arena cell has no unique manifest target")
1737
+ target = matches[0]
1738
+ expected_arms = target.arm_ids or tuple(arm.arm_id for arm in manifest.arms)
1739
+ if cell.arm_ids != expected_arms:
1740
+ raise ValidationError("external arena cell arms differ from the target")
1741
+ if cell.model_slot not in manifest.model_slots or cell.seed not in manifest.seeds:
1742
+ raise ValidationError("external arena cell model slot or seed is not frozen")
1743
+ return target, self._config.benchmark_targets[_target_key(target)]
1744
+
1745
+ def _build_checkpoint(
1746
+ self,
1747
+ cell: ExperimentCell,
1748
+ binding: _TargetBinding,
1749
+ ) -> ExternalArenaCheckpoint:
1750
+ spec = binding.checkpoint_builder
1751
+ builder = _resolve_locked_builder(spec, expected_kind="arena_checkpoint")
1752
+ checkpoint = builder.build(
1753
+ ExternalCheckpointBuildRequest(
1754
+ benchmark_id=cell.benchmark_id,
1755
+ split=cell.split,
1756
+ checkpoint_id=cell.unit_id,
1757
+ source_revision=binding.source_revision,
1758
+ model_slot=cell.model_slot,
1759
+ seed=cell.seed,
1760
+ builder_parameters=spec.parameters,
1761
+ )
1762
+ )
1763
+ if not isinstance(checkpoint, ExternalArenaCheckpoint):
1764
+ raise ValidationError("arena checkpoint builder must return ExternalArenaCheckpoint")
1765
+ return checkpoint
1766
+
1767
+ def _validate_checkpoint(
1768
+ self,
1769
+ checkpoint: ExternalArenaCheckpoint,
1770
+ *,
1771
+ cell: ExperimentCell,
1772
+ binding: _TargetBinding,
1773
+ expected_dependencies: Mapping[str, str],
1774
+ ) -> None:
1775
+ if (
1776
+ checkpoint.benchmark_id != cell.benchmark_id
1777
+ or checkpoint.checkpoint_id != cell.unit_id
1778
+ or checkpoint.source_revision != binding.source_revision
1779
+ or checkpoint.locked_dependency_hashes != expected_dependencies
1780
+ ):
1781
+ raise ValidationError(
1782
+ "external checkpoint identity or dependencies differ from the frozen cell"
1783
+ )
1784
+
1785
+ def _build_bridge(
1786
+ self,
1787
+ checkpoint: ExternalArenaCheckpoint,
1788
+ *,
1789
+ cell: ExperimentCell,
1790
+ binding: _TargetBinding,
1791
+ ) -> AtomicExternalReplayBridge:
1792
+ spec = binding.bridge_builder
1793
+ builder = _resolve_locked_builder(spec, expected_kind="atomic_bridge")
1794
+ bridge = builder.build(
1795
+ ExternalBridgeBuildRequest(
1796
+ checkpoint=checkpoint,
1797
+ model_slot=cell.model_slot,
1798
+ seed=cell.seed,
1799
+ builder_parameters=spec.parameters,
1800
+ )
1801
+ )
1802
+ if not isinstance(bridge, AtomicExternalReplayBridge):
1803
+ raise ValidationError("atomic bridge builder must return AtomicExternalReplayBridge")
1804
+ return bridge
1805
+
1806
+ def _validate_bridge(
1807
+ self,
1808
+ bridge: AtomicExternalReplayBridge,
1809
+ *,
1810
+ binding: _TargetBinding,
1811
+ expected_dependencies: Mapping[str, str],
1812
+ ) -> None:
1813
+ lock = binding.bridge_lock
1814
+ if (
1815
+ bridge.atomic_replay_contract != ATOMIC_EXTERNAL_PAIR_CONTRACT
1816
+ or bridge.benchmark_id != binding.benchmark_id
1817
+ or bridge.source_revision != binding.source_revision
1818
+ or bridge.bridge_id != lock.bridge_id
1819
+ or bridge.bridge_revision != lock.bridge_revision
1820
+ or bridge.scorer_id != lock.scorer_id
1821
+ or bridge.scorer_revision != lock.scorer_revision
1822
+ or _validated_dependency_hashes(bridge.locked_dependency_hashes, benchmark_id=binding.benchmark_id)
1823
+ != expected_dependencies
1824
+ ):
1825
+ raise ValidationError(
1826
+ "constructed atomic bridge differs from its checksum-locked identity"
1827
+ )
1828
+
1829
+ def _build_arm_payload(
1830
+ self,
1831
+ checkpoint: ExternalArenaCheckpoint,
1832
+ *,
1833
+ cell: ExperimentCell,
1834
+ arm_spec: ExperimentArmSpec,
1835
+ pre_action: Mapping[str, Any],
1836
+ ) -> ExternalArmPayload:
1837
+ binding = self._config.arms[arm_spec.arm_id]
1838
+ spec = binding.payload_builder
1839
+ builder = _resolve_locked_builder(spec, expected_kind="arm_payload")
1840
+ artifact = builder.build(
1841
+ ExternalArmPayloadBuildRequest(
1842
+ checkpoint=checkpoint,
1843
+ arm_id=arm_spec.arm_id,
1844
+ release_id=arm_spec.release_id,
1845
+ family=arm_spec.family,
1846
+ profile_kind=arm_spec.profile_kind,
1847
+ arm_kind=arm_spec.arm_kind,
1848
+ model_slot=cell.model_slot,
1849
+ seed=cell.seed,
1850
+ builder_parameters=spec.parameters,
1851
+ arm_parameters=arm_spec.parameters,
1852
+ pre_action=pre_action,
1853
+ )
1854
+ )
1855
+ if not isinstance(artifact, ExternalArmPayload):
1856
+ raise ValidationError("arm payload builder must return ExternalArmPayload")
1857
+ if (
1858
+ artifact.arm_id != arm_spec.arm_id
1859
+ or artifact.release_id != arm_spec.release_id
1860
+ or artifact.family is not arm_spec.family
1861
+ or artifact.profile_kind is not arm_spec.profile_kind
1862
+ ):
1863
+ raise ValidationError(
1864
+ f"arm payload builder returned another identity for {arm_spec.arm_id!r}"
1865
+ )
1866
+ if arm_spec.arm_kind is ExperimentArmKind.POLICY:
1867
+ if artifact.selector_decision is None:
1868
+ raise ValidationError(
1869
+ f"policy arm {arm_spec.arm_id!r} requires selector decision receipt"
1870
+ )
1871
+ if (
1872
+ artifact.selector_input_state_hash != pre_action["state_hash"]
1873
+ or artifact.selector_feature_hash != pre_action["feature_hash"]
1874
+ ):
1875
+ raise ValidationError(
1876
+ f"policy arm {arm_spec.arm_id!r} selector input identity differs "
1877
+ "from the visible pre_action receipt"
1878
+ )
1879
+ try:
1880
+ policy_receipt = self._policy_binding_receipts[arm_spec.arm_id]
1881
+ except KeyError as exc: # pragma: no cover - preflight invariant
1882
+ raise ArtifactError(
1883
+ f"policy arm {arm_spec.arm_id!r} lacks a resolved binding receipt"
1884
+ ) from exc
1885
+ if binding.policy_kind == "locked_best_eligible_global":
1886
+ if (
1887
+ set(artifact.selector_decision)
1888
+ != {
1889
+ "selected_family",
1890
+ "selected_release_id",
1891
+ "abstained",
1892
+ "predicted_gain",
1893
+ "confidence",
1894
+ "policy_manifest_sha256",
1895
+ }
1896
+ or artifact.selector_decision["policy_manifest_sha256"]
1897
+ != policy_receipt["policy_manifest_file_sha256"]
1898
+ ):
1899
+ raise ValidationError(
1900
+ f"policy arm {arm_spec.arm_id!r} decision differs from the "
1901
+ "independently hashed policy manifest"
1902
+ )
1903
+ selected = policy_receipt["selected_representative"]
1904
+ decision_family = artifact.selector_decision["selected_family"]
1905
+ if decision_family != SemanticFamily.NO_OP.value and (
1906
+ decision_family != selected["family"]
1907
+ or artifact.selector_decision["selected_release_id"] != selected["release_id"]
1908
+ ):
1909
+ raise ValidationError(
1910
+ f"policy arm {arm_spec.arm_id!r} selected a release other "
1911
+ "than the frozen global representative"
1912
+ )
1913
+ if (
1914
+ decision_family == SemanticFamily.NO_OP.value
1915
+ and SemanticFamily(selected["family"]) in checkpoint.eligible_families
1916
+ ):
1917
+ raise ValidationError(
1918
+ f"policy arm {arm_spec.arm_id!r} abstained even though the "
1919
+ "frozen global representative is eligible"
1920
+ )
1921
+ elif (
1922
+ set(artifact.selector_decision)
1923
+ != {
1924
+ "selected_family",
1925
+ "selected_release_id",
1926
+ "abstained",
1927
+ "predicted_gain",
1928
+ "confidence",
1929
+ "selector_bundle_hash",
1930
+ }
1931
+ or artifact.selector_decision["selector_bundle_hash"]
1932
+ != policy_receipt["selector_bundle"]["bundle_hash"]
1933
+ ):
1934
+ raise ValidationError(
1935
+ f"policy arm {arm_spec.arm_id!r} selector bundle hash differs "
1936
+ "from the independently parsed frozen bundle"
1937
+ )
1938
+ elif artifact.selector_decision["selected_family"] != SemanticFamily.NO_OP.value:
1939
+ representatives = policy_receipt["representative_by_family"]
1940
+ if (
1941
+ representatives.get(artifact.selector_decision["selected_family"])
1942
+ != (artifact.selector_decision["selected_release_id"])
1943
+ ):
1944
+ raise ValidationError(
1945
+ f"policy arm {arm_spec.arm_id!r} selected a release outside "
1946
+ "the frozen selector representatives"
1947
+ )
1948
+ elif (
1949
+ artifact.selector_decision is not None
1950
+ or artifact.selector_input_state_hash is not None
1951
+ or artifact.selector_feature_hash is not None
1952
+ ):
1953
+ raise ValidationError(f"release arm {arm_spec.arm_id!r} cannot carry selector evidence")
1954
+ return artifact
1955
+
1956
+ def _validate_artifact_set(
1957
+ self,
1958
+ cell: ExperimentCell,
1959
+ *,
1960
+ checkpoint: ExternalArenaCheckpoint,
1961
+ arm_specs: Mapping[str, ExperimentArmSpec],
1962
+ artifacts: Mapping[str, ExternalArmPayload],
1963
+ ) -> None:
1964
+ if set(artifacts) != set(cell.arm_ids) or len(artifacts) != len(cell.arm_ids):
1965
+ raise ValidationError("external arm payload inventory is partial")
1966
+ eligible = set(checkpoint.eligible_families)
1967
+ configured_release_families = {
1968
+ binding.release_id: binding.family
1969
+ for binding in self._config.arms.values()
1970
+ if binding.arm_kind is ExperimentArmKind.RELEASE
1971
+ }
1972
+ for arm_id in cell.arm_ids:
1973
+ arm_spec = arm_specs[arm_id]
1974
+ artifact = artifacts[arm_id]
1975
+ if (
1976
+ arm_spec.arm_kind is ExperimentArmKind.RELEASE
1977
+ and arm_spec.family in _EXECUTABLE_RELEASE_FAMILIES
1978
+ and arm_spec.family not in eligible
1979
+ ):
1980
+ raise ValidationError(
1981
+ f"checkpoint is not eligible for arm family {arm_spec.family.value}"
1982
+ )
1983
+ decision = artifact.selector_decision
1984
+ if decision is None:
1985
+ continue
1986
+ selected_family = SemanticFamily(decision["selected_family"])
1987
+ selected_release_id = decision["selected_release_id"]
1988
+ if selected_family is not SemanticFamily.NO_OP and selected_family not in eligible:
1989
+ raise ValidationError(f"policy arm {arm_id!r} selected an ineligible family")
1990
+ expected_family = configured_release_families.get(selected_release_id)
1991
+ if expected_family is None or expected_family is not selected_family:
1992
+ raise ValidationError(
1993
+ f"policy arm {arm_id!r} selected a release outside the locked release bindings"
1994
+ )
1995
+
1996
+ def _arm_outcomes(
1997
+ self,
1998
+ cell: ExperimentCell,
1999
+ *,
2000
+ arm_specs: Mapping[str, ExperimentArmSpec],
2001
+ artifacts: Mapping[str, ExternalArmPayload],
2002
+ pair_records: Sequence[tuple[str, AtomicReplayRequest, AtomicPairedReplayRecord]],
2003
+ target_binding: _TargetBinding,
2004
+ ) -> tuple[Mapping[str, Any], ...]:
2005
+ baseline_id = self._config.baseline_arm_id
2006
+ baseline = pair_records[0][2].baseline
2007
+ receipt_hashes = tuple(
2008
+ content_hash(record.receipt) for _arm_id, _request, record in pair_records
2009
+ )
2010
+ records = {arm_id: record for arm_id, _request, record in pair_records}
2011
+ outcomes: list[Mapping[str, Any]] = []
2012
+ for arm_id in cell.arm_ids:
2013
+ spec = arm_specs[arm_id]
2014
+ artifact = artifacts[arm_id]
2015
+ if arm_id == baseline_id:
2016
+ result = baseline
2017
+ receipt_data: Mapping[str, Any] = {
2018
+ "atomic_pair_receipt_sha256s": receipt_hashes,
2019
+ }
2020
+ else:
2021
+ result = records[arm_id].treatment
2022
+ receipt_data = {
2023
+ "atomic_pair_receipt_sha256": content_hash(records[arm_id].receipt),
2024
+ }
2025
+ metrics, composition_receipt = _compose_valid_success(
2026
+ result.metrics,
2027
+ benchmark_id=cell.benchmark_id,
2028
+ arm_id=arm_id,
2029
+ trace_hash=result.trace_hash,
2030
+ bridge_lock=target_binding.bridge_lock,
2031
+ outcome_contract=target_binding.outcome_contract,
2032
+ )
2033
+ outcome: dict[str, Any] = {
2034
+ "arm_id": arm_id,
2035
+ "release_id": spec.release_id,
2036
+ "family": spec.family.value,
2037
+ "profile_kind": spec.profile_kind.value,
2038
+ "status": "succeeded",
2039
+ "metrics": metrics,
2040
+ "trace_hash": result.trace_hash,
2041
+ "usage": result.usage.as_dict(),
2042
+ "payload_sha256": artifact.payload_sha256,
2043
+ "valid_success_composition": composition_receipt,
2044
+ **receipt_data,
2045
+ }
2046
+ if artifact.selector_decision is not None:
2047
+ outcome["decision"] = artifact.selector_decision
2048
+ outcome["selector_decision_sha256"] = content_hash(artifact.selector_decision)
2049
+ outcome["selector_input_receipt"] = {
2050
+ "state_hash": artifact.selector_input_state_hash,
2051
+ "feature_hash": artifact.selector_feature_hash,
2052
+ }
2053
+ outcomes.append(freeze_mapping(outcome))
2054
+ return tuple(outcomes)
2055
+
2056
+ def _pre_action_receipt(self, state: FrozenAgentState) -> Mapping[str, Any]:
2057
+ observation = self._feature_extractor.extract(state)
2058
+ features = observation.features
2059
+ return freeze_mapping(
2060
+ {
2061
+ "state_hash": observation.state_hash,
2062
+ "feature_hash": observation.feature_hash,
2063
+ "feature_schema_id": self._feature_extractor.feature_schema_id,
2064
+ "feature_schema_hash": self._feature_extractor.feature_schema_hash,
2065
+ "provenance_hash": content_hash(observation.provenance),
2066
+ "defect_scores": {
2067
+ SemanticFamily.REFORMULATE.value: (features.reformulation_need_score),
2068
+ SemanticFamily.DECOMPOSE.value: features.multi_step_score,
2069
+ SemanticFamily.ACQUIRE.value: features.retrieval_need_score,
2070
+ SemanticFamily.CLARIFY.value: features.ambiguity_score,
2071
+ SemanticFamily.CURATE.value: features.context_pressure,
2072
+ },
2073
+ }
2074
+ )
2075
+
2076
+
2077
+ def factory() -> ExternalArenaExperimentDriver:
2078
+ """Construct the driver from explicit path and exact-byte SHA-256 bindings."""
2079
+
2080
+ path = os.environ.get(CONFIG_PATH_ENV)
2081
+ expected_sha256 = os.environ.get(CONFIG_SHA256_ENV)
2082
+ if not path or not expected_sha256:
2083
+ raise ArtifactError(
2084
+ f"set both {CONFIG_PATH_ENV} and {CONFIG_SHA256_ENV} before loading "
2085
+ "the external arena driver"
2086
+ )
2087
+ return ExternalArenaExperimentDriver.from_file(
2088
+ path,
2089
+ expected_sha256=expected_sha256,
2090
+ )
2091
+
2092
+
2093
+ def _target_binding(value: Any, *, path: str) -> _TargetBinding:
2094
+ data = _strict_mapping(
2095
+ value,
2096
+ path=path,
2097
+ keys={
2098
+ "benchmark_id",
2099
+ "split",
2100
+ "source_revision",
2101
+ "checkpoint_builder",
2102
+ "bridge_builder",
2103
+ "bridge_lock",
2104
+ "outcome_contract",
2105
+ },
2106
+ required={
2107
+ "benchmark_id",
2108
+ "split",
2109
+ "source_revision",
2110
+ "checkpoint_builder",
2111
+ "bridge_builder",
2112
+ "bridge_lock",
2113
+ "outcome_contract",
2114
+ },
2115
+ )
2116
+ benchmark_id = _text(data["benchmark_id"], f"{path}.benchmark_id")
2117
+ source_revision = _text(data["source_revision"], f"{path}.source_revision")
2118
+ if benchmark_id not in _SUPPORTED_ARENAS:
2119
+ raise ValidationError(f"{path}.benchmark_id is not a supported external arena")
2120
+ official = {spec.cell_id: spec.source_revision for spec in interactive_benchmark_specs()}.get(
2121
+ benchmark_id
2122
+ )
2123
+ if benchmark_id == "tau3.retail":
2124
+ # Retail has its own checkpoint/provider path. Registering metadata is
2125
+ # not certification by the historical five-family interactive factory.
2126
+ official = BenchmarkRegistry.default().get(benchmark_id).source_revision
2127
+ if official is not None and source_revision != official:
2128
+ raise ValidationError(
2129
+ f"{path}.source_revision differs from the packaged official source lock"
2130
+ )
2131
+ outcome_contract = _OutcomeContract.from_mapping(
2132
+ data["outcome_contract"],
2133
+ path=f"{path}.outcome_contract",
2134
+ )
2135
+ expected_success_metric = _REGISTERED_OFFICIAL_SUCCESS_METRICS.get(benchmark_id)
2136
+ if (
2137
+ expected_success_metric is not None
2138
+ and outcome_contract.official_success_metric != expected_success_metric
2139
+ ):
2140
+ raise ValidationError(
2141
+ f"{path}.outcome_contract.official_success_metric must match the "
2142
+ f"registered adapter metric {expected_success_metric!r}"
2143
+ )
2144
+ return _TargetBinding(
2145
+ benchmark_id=benchmark_id,
2146
+ split=_text(data["split"], f"{path}.split"),
2147
+ source_revision=source_revision,
2148
+ checkpoint_builder=_LockedBuilderSpec.from_mapping(
2149
+ data["checkpoint_builder"],
2150
+ path=f"{path}.checkpoint_builder",
2151
+ ),
2152
+ bridge_builder=_LockedBuilderSpec.from_mapping(
2153
+ data["bridge_builder"],
2154
+ path=f"{path}.bridge_builder",
2155
+ ),
2156
+ bridge_lock=_BridgeIdentityLock.from_mapping(
2157
+ data["bridge_lock"],
2158
+ path=f"{path}.bridge_lock",
2159
+ benchmark_id=benchmark_id,
2160
+ ),
2161
+ outcome_contract=outcome_contract,
2162
+ )
2163
+
2164
+
2165
+ def _arm_binding(value: Any, *, path: str) -> _ArmBinding:
2166
+ data = _strict_mapping(
2167
+ value,
2168
+ path=path,
2169
+ keys={
2170
+ "release_id",
2171
+ "family",
2172
+ "profile_kind",
2173
+ "arm_kind",
2174
+ "payload_builder",
2175
+ "policy_kind",
2176
+ },
2177
+ required={"release_id", "family", "profile_kind", "payload_builder"},
2178
+ )
2179
+ try:
2180
+ family = SemanticFamily(_text(data["family"], f"{path}.family"))
2181
+ profile_kind = ProfileKind(_text(data["profile_kind"], f"{path}.profile_kind"))
2182
+ arm_kind = ExperimentArmKind(
2183
+ _text(
2184
+ data.get("arm_kind", ExperimentArmKind.RELEASE.value),
2185
+ f"{path}.arm_kind",
2186
+ )
2187
+ )
2188
+ except ValueError as exc:
2189
+ raise ValidationError(f"{path} contains an unknown enum value") from exc
2190
+ policy_kind = data.get("policy_kind")
2191
+ if policy_kind is not None:
2192
+ policy_kind = _text(policy_kind, f"{path}.policy_kind")
2193
+ return _ArmBinding(
2194
+ release_id=_text(data["release_id"], f"{path}.release_id"),
2195
+ family=family,
2196
+ profile_kind=profile_kind,
2197
+ arm_kind=arm_kind,
2198
+ payload_builder=_LockedBuilderSpec.from_mapping(
2199
+ data["payload_builder"],
2200
+ path=f"{path}.payload_builder",
2201
+ ),
2202
+ policy_kind=policy_kind,
2203
+ )
2204
+
2205
+
2206
+ def _resolve_locked_builder(
2207
+ spec: _LockedBuilderSpec,
2208
+ *,
2209
+ expected_kind: str,
2210
+ ) -> LockedExternalArenaBuilder:
2211
+ if expected_kind not in _BUILDER_KINDS: # pragma: no cover - internal invariant
2212
+ raise ValueError("unknown external builder kind")
2213
+ factory_candidate = resolve_driver_entrypoint(spec.entrypoint)
2214
+ if not callable(factory_candidate):
2215
+ raise ArtifactError("external builder entrypoint must be a zero-argument factory")
2216
+ try:
2217
+ builder = factory_candidate()
2218
+ except TypeError as exc:
2219
+ raise ArtifactError("external builder entrypoint must be a zero-argument factory") from exc
2220
+ if not isinstance(builder, LockedExternalArenaBuilder):
2221
+ raise ArtifactError("external component does not implement the locked builder contract")
2222
+ if builder.builder_kind != expected_kind:
2223
+ raise ArtifactError(
2224
+ f"external builder kind is {builder.builder_kind!r}, expected {expected_kind!r}"
2225
+ )
2226
+ identity = _builder_fingerprint(builder, label=f"{expected_kind} builder")
2227
+ if content_hash(identity) != spec.fingerprint_sha256:
2228
+ raise ArtifactError(f"{expected_kind} builder fingerprint checksum mismatch")
2229
+ return builder
2230
+
2231
+
2232
+ def _builder_fingerprint(
2233
+ builder: LockedExternalArenaBuilder,
2234
+ *,
2235
+ label: str,
2236
+ ) -> Mapping[str, Any]:
2237
+ fingerprint = builder.fingerprint()
2238
+ if not isinstance(fingerprint, Mapping) or not fingerprint:
2239
+ raise ValidationError(f"{label} fingerprint must be a non-empty mapping")
2240
+ try:
2241
+ return freeze_mapping(fingerprint)
2242
+ except (TypeError, ValueError) as exc:
2243
+ raise ValidationError(f"{label} fingerprint must be canonical JSON") from exc
2244
+
2245
+
2246
+ def _builder_preflight(
2247
+ builder: LockedExternalArenaBuilder,
2248
+ parameters: Mapping[str, Any],
2249
+ ) -> tuple[str, ...]:
2250
+ blockers = builder.preflight(parameters)
2251
+ if isinstance(blockers, (str, bytes)) or not isinstance(blockers, Sequence):
2252
+ raise ValidationError("external component preflight must return a sequence of strings")
2253
+ normalized = tuple(blockers)
2254
+ if any(not isinstance(item, str) or not item.strip() for item in normalized):
2255
+ raise ValidationError("external component preflight blockers must be non-empty strings")
2256
+ return normalized
2257
+
2258
+
2259
+ def _compose_valid_success(
2260
+ raw_metrics: Mapping[str, float],
2261
+ *,
2262
+ benchmark_id: str,
2263
+ arm_id: str,
2264
+ trace_hash: str,
2265
+ bridge_lock: _BridgeIdentityLock,
2266
+ outcome_contract: _OutcomeContract,
2267
+ ) -> tuple[Mapping[str, float], Mapping[str, Any]]:
2268
+ """Create the primary valid outcome only inside the trusted host boundary."""
2269
+
2270
+ if _VALID_SUCCESS_METRIC in raw_metrics:
2271
+ raise ValidationError(f"external bridge arm {arm_id!r} must not precompute valid_success")
2272
+ required = {
2273
+ outcome_contract.official_success_metric,
2274
+ _CONTRACT_PRESERVED_METRIC,
2275
+ _PRIVILEGED_LEAKAGE_METRIC,
2276
+ }
2277
+ missing = sorted(required - set(raw_metrics))
2278
+ if missing:
2279
+ raise ValidationError(
2280
+ f"external bridge arm {arm_id!r} is missing valid-success components: "
2281
+ + ", ".join(missing)
2282
+ )
2283
+ official_success = _unit_interval_metric(
2284
+ raw_metrics[outcome_contract.official_success_metric],
2285
+ label=(
2286
+ f"external bridge arm {arm_id!r} metric {outcome_contract.official_success_metric!r}"
2287
+ ),
2288
+ )
2289
+ contract_preserved = _unit_interval_metric(
2290
+ raw_metrics[_CONTRACT_PRESERVED_METRIC],
2291
+ label=f"external bridge arm {arm_id!r} metric {_CONTRACT_PRESERVED_METRIC!r}",
2292
+ )
2293
+ privileged_leakage = _unit_interval_metric(
2294
+ raw_metrics[_PRIVILEGED_LEAKAGE_METRIC],
2295
+ label=f"external bridge arm {arm_id!r} metric {_PRIVILEGED_LEAKAGE_METRIC!r}",
2296
+ )
2297
+ valid_success = official_success * contract_preserved * (1.0 - privileged_leakage)
2298
+ receipt_body = {
2299
+ "schema_version": 1,
2300
+ "formula_id": outcome_contract.formula_id,
2301
+ "benchmark_id": benchmark_id,
2302
+ "arm_id": arm_id,
2303
+ "official_success_metric": outcome_contract.official_success_metric,
2304
+ "components": {
2305
+ "official_success": official_success,
2306
+ _CONTRACT_PRESERVED_METRIC: contract_preserved,
2307
+ _PRIVILEGED_LEAKAGE_METRIC: privileged_leakage,
2308
+ },
2309
+ _VALID_SUCCESS_METRIC: valid_success,
2310
+ "outcome_contract_sha256": content_hash(outcome_contract),
2311
+ "scorer_id": bridge_lock.scorer_id,
2312
+ "scorer_revision": bridge_lock.scorer_revision,
2313
+ "trace_hash": trace_hash,
2314
+ }
2315
+ composition_receipt = freeze_mapping(
2316
+ {
2317
+ **receipt_body,
2318
+ "composition_receipt_sha256": content_hash(receipt_body),
2319
+ }
2320
+ )
2321
+ metrics = dict(raw_metrics)
2322
+ metrics[_VALID_SUCCESS_METRIC] = valid_success
2323
+ return freeze_mapping(metrics), composition_receipt
2324
+
2325
+
2326
+ def _unit_interval_metric(value: Any, *, label: str) -> float:
2327
+ numeric = _finite_number(value, label=label)
2328
+ if not 0.0 <= numeric <= 1.0:
2329
+ raise ValidationError(f"{label} must be in [0, 1]")
2330
+ return numeric
2331
+
2332
+
2333
+ def _baseline_consistency_hash(record: AtomicPairedReplayRecord) -> str:
2334
+ baseline_receipts = tuple(arm for arm in record.receipt.arms if arm.role == "baseline")
2335
+ if len(baseline_receipts) != 1: # pragma: no cover - typed receipt invariant
2336
+ raise ValidationError("atomic pair lacks one baseline receipt")
2337
+ baseline_receipt = baseline_receipts[0]
2338
+ return content_hash(
2339
+ {
2340
+ "metrics": record.baseline.metrics,
2341
+ "trace_hash": record.baseline.trace_hash,
2342
+ "usage": record.baseline.usage,
2343
+ "receipt": {
2344
+ "benchmark_id": record.receipt.benchmark_id,
2345
+ "checkpoint_id": record.receipt.checkpoint_id,
2346
+ "source_revision": record.receipt.source_revision,
2347
+ "bridge_id": record.receipt.bridge_id,
2348
+ "bridge_revision": record.receipt.bridge_revision,
2349
+ "scorer_id": record.receipt.scorer_id,
2350
+ "scorer_revision": record.receipt.scorer_revision,
2351
+ "origin_state_hash": record.receipt.origin_state_hash,
2352
+ "replay_identity": record.receipt.replay_identity,
2353
+ "seed": record.receipt.seed,
2354
+ "locked_dependency_hashes": (record.receipt.locked_dependency_hashes),
2355
+ "baseline_arm": {
2356
+ "role": baseline_receipt.role,
2357
+ "arm_id": baseline_receipt.arm_id,
2358
+ "origin_state_hash": baseline_receipt.origin_state_hash,
2359
+ "replay_identity": baseline_receipt.replay_identity,
2360
+ "seed": baseline_receipt.seed,
2361
+ "locked_dependency_hashes": (baseline_receipt.locked_dependency_hashes),
2362
+ "completed": baseline_receipt.completed,
2363
+ },
2364
+ },
2365
+ }
2366
+ )
2367
+
2368
+
2369
+ def _validated_policy_decision(value: Mapping[str, Any]) -> Mapping[str, Any]:
2370
+ if not isinstance(value, Mapping):
2371
+ raise ValidationError("selector decision receipt must be an object")
2372
+ common = {
2373
+ "selected_family",
2374
+ "selected_release_id",
2375
+ "abstained",
2376
+ "predicted_gain",
2377
+ "confidence",
2378
+ }
2379
+ identity_fields = set(value) - common
2380
+ if identity_fields not in (
2381
+ {"selector_bundle_hash"},
2382
+ {"policy_manifest_sha256"},
2383
+ ):
2384
+ raise ValidationError("selector decision receipt fields are invalid")
2385
+ try:
2386
+ family = SemanticFamily(value["selected_family"])
2387
+ except (TypeError, ValueError) as exc:
2388
+ raise ValidationError("selector decision selected_family is invalid") from exc
2389
+ if family not in {*_CORE_FAMILIES, SemanticFamily.NO_OP}:
2390
+ raise ValidationError("selector decision selected_family is not executable")
2391
+ release_id = value["selected_release_id"]
2392
+ _require_text(release_id, label="selector decision selected_release_id")
2393
+ abstained = value["abstained"]
2394
+ if not isinstance(abstained, bool):
2395
+ raise ValidationError("selector decision abstained must be boolean")
2396
+ if abstained is not (family is SemanticFamily.NO_OP):
2397
+ raise ValidationError("selector decision abstention and selected_family are inconsistent")
2398
+ if family is SemanticFamily.NO_OP and release_id != "baseline.no_op":
2399
+ raise ValidationError("selector NO_OP decision must select baseline.no_op")
2400
+ predicted_gain = _finite_number(
2401
+ value["predicted_gain"],
2402
+ label="selector decision predicted_gain",
2403
+ )
2404
+ confidence = _finite_number(
2405
+ value["confidence"],
2406
+ label="selector decision confidence",
2407
+ )
2408
+ if not 0 <= confidence <= 1:
2409
+ raise ValidationError("selector decision confidence must be in [0, 1]")
2410
+ identity_field = next(iter(identity_fields))
2411
+ artifact_hash = value[identity_field]
2412
+ _require_sha256(
2413
+ artifact_hash,
2414
+ label=f"selector decision {identity_field}",
2415
+ )
2416
+ return freeze_mapping(
2417
+ {
2418
+ "selected_family": family.value,
2419
+ "selected_release_id": release_id,
2420
+ "abstained": abstained,
2421
+ "predicted_gain": predicted_gain,
2422
+ "confidence": confidence,
2423
+ identity_field: artifact_hash,
2424
+ }
2425
+ )
2426
+
2427
+
2428
+ def _read_manifest_file_binding(
2429
+ manifest: ExperimentManifest,
2430
+ *,
2431
+ environment_name: Any,
2432
+ checksum_environment_name: Any,
2433
+ label: str,
2434
+ ) -> tuple[bytes, str]:
2435
+ for value, field_label in (
2436
+ (environment_name, "file environment"),
2437
+ (checksum_environment_name, "checksum environment"),
2438
+ ):
2439
+ if not isinstance(value, str) or _ENVIRONMENT_NAME.fullmatch(value) is None:
2440
+ raise ValidationError(f"{label} {field_label} is invalid")
2441
+ matches = tuple(
2442
+ prerequisite
2443
+ for prerequisite in manifest.prerequisites
2444
+ if prerequisite.kind is PrerequisiteKind.FILE_ENV
2445
+ and prerequisite.name == environment_name
2446
+ and prerequisite.sha256_env == checksum_environment_name
2447
+ )
2448
+ if len(matches) != 1:
2449
+ raise ValidationError(
2450
+ f"{label} is not declared as one exact checksum-bound FILE_ENV prerequisite"
2451
+ )
2452
+ raw_path = os.environ.get(environment_name)
2453
+ expected_sha256 = os.environ.get(checksum_environment_name)
2454
+ if not raw_path or not expected_sha256:
2455
+ raise ArtifactError(f"{label} environment binding is missing")
2456
+ _require_sha256(expected_sha256, label=f"{label} expected SHA-256")
2457
+ source = Path(raw_path).expanduser()
2458
+ try:
2459
+ if not source.is_file():
2460
+ raise ArtifactError(f"{label} does not identify an existing file")
2461
+ if source.stat().st_size > _MAX_POLICY_ARTIFACT_BYTES:
2462
+ raise ArtifactError(f"{label} exceeds the size limit")
2463
+ raw = source.read_bytes()
2464
+ except OSError as exc:
2465
+ raise ArtifactError(f"{label} cannot be read") from exc
2466
+ observed_sha256 = hashlib.sha256(raw).hexdigest()
2467
+ if observed_sha256 != expected_sha256:
2468
+ raise ArtifactError(f"{label} checksum differs from its declared checksum")
2469
+ return raw, observed_sha256
2470
+
2471
+
2472
+ def _load_frozen_selector_bundle(
2473
+ raw: bytes,
2474
+ *,
2475
+ identity_environment_name: Any,
2476
+ manifest: ExperimentManifest,
2477
+ label: str,
2478
+ ) -> SelectorArtifactBundle:
2479
+ if (
2480
+ not isinstance(identity_environment_name, str)
2481
+ or _ENVIRONMENT_NAME.fullmatch(identity_environment_name) is None
2482
+ ):
2483
+ raise ValidationError(f"{label} identity environment is invalid")
2484
+ identity_prerequisites = tuple(
2485
+ prerequisite
2486
+ for prerequisite in manifest.prerequisites
2487
+ if prerequisite.kind is PrerequisiteKind.ENV
2488
+ and prerequisite.name == identity_environment_name
2489
+ )
2490
+ if len(identity_prerequisites) != 1:
2491
+ raise ValidationError(f"{label} identity is not declared as one exact ENV prerequisite")
2492
+ identity_raw = os.environ.get(identity_environment_name)
2493
+ if not identity_raw:
2494
+ raise ArtifactError(f"{label} identity environment is missing")
2495
+ identity = _strict_mapping(
2496
+ _strict_json(identity_raw.encode("utf-8")),
2497
+ path=f"{label} identity",
2498
+ keys={"selector_id", "revision", "sha256"},
2499
+ required={"selector_id", "revision", "sha256"},
2500
+ )
2501
+ try:
2502
+ bundle = SelectorArtifactBundle.from_artifact(_strict_json(raw))
2503
+ except (TypeError, ValueError) as exc:
2504
+ raise ArtifactError(f"{label} failed strict SelectorArtifactBundle parsing") from exc
2505
+ if bundle.stage is not SelectorArtifactStage.FROZEN:
2506
+ raise ArtifactError(f"{label} stage must be frozen")
2507
+ selector_id = _text(identity["selector_id"], f"{label} identity.selector_id")
2508
+ revision = _text(identity["revision"], f"{label} identity.revision")
2509
+ sha256 = _text(identity["sha256"], f"{label} identity.sha256")
2510
+ _require_sha256(sha256, label=f"{label} identity.sha256")
2511
+ if (
2512
+ selector_id != bundle.selector_card_id
2513
+ or revision != bundle.revision
2514
+ or sha256 != bundle.bundle_hash
2515
+ ):
2516
+ raise ArtifactError(f"{label} identity differs from its parsed frozen bundle")
2517
+ return bundle
2518
+
2519
+
2520
+ def _validate_selector_policy_provenance(
2521
+ bundle: SelectorArtifactBundle,
2522
+ *,
2523
+ policy_manifest: _SelectorPolicyManifest,
2524
+ label: str,
2525
+ ) -> None:
2526
+ expected = {
2527
+ "training_table_sha256": policy_manifest.training_table_sha256,
2528
+ "split_provenance_hash": policy_manifest.split_provenance_hash,
2529
+ "comparison_profile_hash": policy_manifest.comparison_profile_hash,
2530
+ "representative_by_family": dict(policy_manifest.representative_by_family),
2531
+ }
2532
+ if any(getattr(bundle, field) != value for field, value in expected.items()):
2533
+ raise ValidationError(
2534
+ f"{label} selector bundle differs from policy-manifest "
2535
+ "training/profile/representative provenance"
2536
+ )
2537
+ discovery = bundle.fit_lineage.get("discovery")
2538
+ if (
2539
+ not isinstance(discovery, Mapping)
2540
+ or discovery.get("revision") != policy_manifest.discovery_revision
2541
+ or discovery.get("sha256") != policy_manifest.discovery_sha256
2542
+ ):
2543
+ raise ValidationError(
2544
+ f"{label} selector bundle fit lineage differs from policy-manifest discovery"
2545
+ )
2546
+ if bundle.calibration_receipt.get("discovery_hash") != policy_manifest.discovery_sha256:
2547
+ raise ValidationError(
2548
+ f"{label} selector calibration receipt differs from policy-manifest discovery"
2549
+ )
2550
+
2551
+
2552
+ def _policy_representatives_from_mapping(
2553
+ value: Any,
2554
+ *,
2555
+ path: str,
2556
+ ) -> Mapping[SemanticFamily, str]:
2557
+ raw = _nonempty_object(value, path=path)
2558
+ representatives: dict[SemanticFamily, str] = {}
2559
+ for name, release_id in raw.items():
2560
+ try:
2561
+ family = SemanticFamily(_text(name, f"{path} family"))
2562
+ except ValueError as exc:
2563
+ raise ValidationError(f"{path} contains an unknown family") from exc
2564
+ representatives[family] = _text(
2565
+ release_id,
2566
+ f"{path}[{name!r}]",
2567
+ )
2568
+ return _validated_policy_representatives(representatives)
2569
+
2570
+
2571
+ def _validated_policy_representatives(
2572
+ value: Mapping[SemanticFamily, str],
2573
+ ) -> dict[SemanticFamily, str]:
2574
+ representatives = dict(value)
2575
+ if (
2576
+ not representatives
2577
+ or any(
2578
+ family not in _CORE_FAMILIES
2579
+ or not isinstance(release_id, str)
2580
+ or not release_id.strip()
2581
+ or release_id != release_id.strip()
2582
+ for family, release_id in representatives.items()
2583
+ )
2584
+ or len(set(representatives.values())) != len(representatives)
2585
+ ):
2586
+ raise ValidationError("policy manifest representatives require unique core-family releases")
2587
+ return representatives
2588
+
2589
+
2590
+ def _validate_external_gate_receipt(
2591
+ value: Mapping[str, Any],
2592
+ *,
2593
+ benchmark_id: str,
2594
+ split: str,
2595
+ source_revision: str,
2596
+ manifest_hash: str,
2597
+ driver_config_sha256: str,
2598
+ unit_inventory_sha256: str,
2599
+ bridge_lock: _BridgeIdentityLock,
2600
+ reset_mode: str,
2601
+ ) -> Mapping[str, Any]:
2602
+ path = f"external gate receipt for {benchmark_id}"
2603
+ data = _strict_mapping(
2604
+ value,
2605
+ path=path,
2606
+ keys={
2607
+ "schema_version",
2608
+ "artifact",
2609
+ "status",
2610
+ "benchmark_id",
2611
+ "split",
2612
+ "source_revision",
2613
+ "manifest_hash",
2614
+ "driver_config_sha256",
2615
+ "unit_inventory_sha256",
2616
+ "bridge",
2617
+ "scorer",
2618
+ "verification",
2619
+ "approval",
2620
+ },
2621
+ required={
2622
+ "schema_version",
2623
+ "artifact",
2624
+ "status",
2625
+ "benchmark_id",
2626
+ "split",
2627
+ "source_revision",
2628
+ "manifest_hash",
2629
+ "driver_config_sha256",
2630
+ "unit_inventory_sha256",
2631
+ "bridge",
2632
+ "scorer",
2633
+ "verification",
2634
+ "approval",
2635
+ },
2636
+ )
2637
+ if (
2638
+ data["schema_version"] != 1
2639
+ or data["artifact"] != "agentinputlab.external_arena_gate_receipt"
2640
+ or data["status"] != "approved"
2641
+ ):
2642
+ raise ValidationError(f"{path} is not an approved schema-v1 gate receipt")
2643
+ expected_identity = {
2644
+ "benchmark_id": benchmark_id,
2645
+ "split": split,
2646
+ "source_revision": source_revision,
2647
+ "manifest_hash": manifest_hash,
2648
+ "driver_config_sha256": driver_config_sha256,
2649
+ "unit_inventory_sha256": unit_inventory_sha256,
2650
+ }
2651
+ for name in ("manifest_hash", "driver_config_sha256", "unit_inventory_sha256"):
2652
+ _require_sha256(expected_identity[name], label=f"expected {name}")
2653
+ for name, expected in expected_identity.items():
2654
+ if _text(data[name], f"{path}.{name}") != expected:
2655
+ raise ValidationError(f"{path}.{name} differs from the frozen experiment")
2656
+ bridge = _strict_mapping(
2657
+ data["bridge"],
2658
+ path=f"{path}.bridge",
2659
+ keys={"bridge_id", "bridge_revision"},
2660
+ required={"bridge_id", "bridge_revision"},
2661
+ )
2662
+ scorer = _strict_mapping(
2663
+ data["scorer"],
2664
+ path=f"{path}.scorer",
2665
+ keys={"scorer_id", "scorer_revision"},
2666
+ required={"scorer_id", "scorer_revision"},
2667
+ )
2668
+ if (
2669
+ bridge.get("bridge_id") != bridge_lock.bridge_id
2670
+ or bridge.get("bridge_revision") != bridge_lock.bridge_revision
2671
+ ):
2672
+ raise ValidationError(f"{path}.bridge differs from the external driver lock")
2673
+ if (
2674
+ scorer.get("scorer_id") != bridge_lock.scorer_id
2675
+ or scorer.get("scorer_revision") != bridge_lock.scorer_revision
2676
+ ):
2677
+ raise ValidationError(f"{path}.scorer differs from the external driver lock")
2678
+ verification = _strict_mapping(
2679
+ data["verification"],
2680
+ path=f"{path}.verification",
2681
+ keys={
2682
+ "atomic_replay_contract",
2683
+ "reset_mode",
2684
+ "reset_verified",
2685
+ "reset_receipt_sha256",
2686
+ "no_op_parity_verified",
2687
+ "no_op_parity_receipt_sha256",
2688
+ },
2689
+ required={
2690
+ "atomic_replay_contract",
2691
+ "reset_mode",
2692
+ "reset_verified",
2693
+ "reset_receipt_sha256",
2694
+ "no_op_parity_verified",
2695
+ "no_op_parity_receipt_sha256",
2696
+ },
2697
+ )
2698
+ if (
2699
+ verification["atomic_replay_contract"] != ATOMIC_EXTERNAL_PAIR_CONTRACT
2700
+ or verification["reset_mode"] != reset_mode
2701
+ or verification["reset_verified"] is not True
2702
+ or verification["no_op_parity_verified"] is not True
2703
+ ):
2704
+ raise ValidationError(f"{path}.verification does not approve atomic reset and NO_OP parity")
2705
+ for name in ("reset_receipt_sha256", "no_op_parity_receipt_sha256"):
2706
+ _require_sha256(
2707
+ verification[name],
2708
+ label=f"{path}.verification.{name}",
2709
+ )
2710
+ approval = _strict_mapping(
2711
+ data["approval"],
2712
+ path=f"{path}.approval",
2713
+ keys={
2714
+ "approved",
2715
+ "approver_id",
2716
+ "approval_revision",
2717
+ "approved_at",
2718
+ "scope_sha256",
2719
+ },
2720
+ required={
2721
+ "approved",
2722
+ "approver_id",
2723
+ "approval_revision",
2724
+ "approved_at",
2725
+ "scope_sha256",
2726
+ },
2727
+ )
2728
+ if approval["approved"] is not True:
2729
+ raise ValidationError(f"{path}.approval.approved must be true")
2730
+ _require_text(approval["approver_id"], label=f"{path}.approval.approver_id")
2731
+ _require_text(approval["approved_at"], label=f"{path}.approval.approved_at")
2732
+ _require_sha256(
2733
+ approval["approval_revision"],
2734
+ label=f"{path}.approval.approval_revision",
2735
+ )
2736
+ _require_sha256(
2737
+ approval["scope_sha256"],
2738
+ label=f"{path}.approval.scope_sha256",
2739
+ )
2740
+ scope = {
2741
+ **expected_identity,
2742
+ "bridge": dict(bridge),
2743
+ "scorer": dict(scorer),
2744
+ "verification": dict(verification),
2745
+ }
2746
+ if approval["scope_sha256"] != content_hash(scope):
2747
+ raise ValidationError(f"{path}.approval.scope_sha256 differs from the approved scope")
2748
+ return freeze_mapping(data)
2749
+
2750
+
2751
+ def _validated_dependency_hashes(
2752
+ value: Mapping[str, str],
2753
+ *, benchmark_id: str = "",
2754
+ ) -> Mapping[str, str]:
2755
+ # The common user simulator and NL judge are part of Retail's scientific
2756
+ # identity. Other arenas retain their existing exact ten-slot contract.
2757
+ required = _REQUIRED_SHARED_DEPENDENCY_SLOTS
2758
+ if benchmark_id == "tau3.retail":
2759
+ required = required | _RETAIL_ENVIRONMENT_DEPENDENCY_SLOTS
2760
+ if not isinstance(value, Mapping):
2761
+ raise ValidationError(
2762
+ "external locked dependency hashes must be an object containing "
2763
+ "the exact shared dependency slots"
2764
+ )
2765
+ if set(value) != required:
2766
+ missing = sorted(required - set(value))
2767
+ extra = sorted(set(value) - required)
2768
+ raise ValidationError(
2769
+ "external locked dependency hashes must contain the exact shared "
2770
+ f"dependency slots; missing={missing}, extra={extra}"
2771
+ )
2772
+ normalized: dict[str, str] = {}
2773
+ for name, digest in value.items():
2774
+ _require_text(name, label="external dependency name")
2775
+ _require_sha256(digest, label=f"external dependency {name!r}")
2776
+ normalized[name] = digest
2777
+ return freeze_mapping(normalized)
2778
+
2779
+
2780
+ def _anonymized_group_hash(benchmark_id: str, raw_group_id: str) -> str:
2781
+ _require_text(benchmark_id, label="group benchmark_id")
2782
+ _require_text(raw_group_id, label="group_id")
2783
+ return "group_" + content_hash({"benchmark_id": benchmark_id, "group_id": raw_group_id})
2784
+
2785
+
2786
+ def external_dependency_not_applicable_hash(slot: str) -> str:
2787
+ """Return the canonical sentinel digest for an inapplicable shared slot."""
2788
+
2789
+ if slot not in _REQUIRED_SHARED_DEPENDENCY_SLOTS:
2790
+ raise ValidationError(f"unknown external dependency slot: {slot!r}")
2791
+ return _NOT_APPLICABLE_DEPENDENCY_HASHES[slot]
2792
+
2793
+
2794
+ def _target_key(target: ExperimentBenchmarkTarget) -> str:
2795
+ return f"{target.cell_id}:{target.split}"
2796
+
2797
+
2798
+ def _strict_json(raw: bytes) -> Mapping[str, Any]:
2799
+ try:
2800
+ parsed = json.loads(
2801
+ raw.decode("utf-8"),
2802
+ object_pairs_hook=_reject_duplicate_keys,
2803
+ parse_constant=lambda value: _reject_json_constant(value),
2804
+ )
2805
+ except (UnicodeError, json.JSONDecodeError) as exc:
2806
+ raise ArtifactError("external arena config is not strict UTF-8 JSON") from exc
2807
+ if not isinstance(parsed, Mapping):
2808
+ raise ValidationError("external arena config root must be an object")
2809
+ return parsed
2810
+
2811
+
2812
+ def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
2813
+ result: dict[str, Any] = {}
2814
+ for key, value in pairs:
2815
+ if key in result:
2816
+ raise ValidationError(f"duplicate JSON key: {key}")
2817
+ result[key] = value
2818
+ return result
2819
+
2820
+
2821
+ def _reject_json_constant(value: str) -> None:
2822
+ raise ValidationError(f"non-finite JSON constant is forbidden: {value}")
2823
+
2824
+
2825
+ def _strict_mapping(
2826
+ value: Any,
2827
+ *,
2828
+ path: str,
2829
+ keys: set[str],
2830
+ required: set[str],
2831
+ ) -> Mapping[str, Any]:
2832
+ if not isinstance(value, Mapping):
2833
+ raise ValidationError(f"{path} must be an object")
2834
+ if any(not isinstance(key, str) for key in value):
2835
+ raise ValidationError(f"{path} keys must be strings")
2836
+ unknown = set(value) - keys
2837
+ missing = required - set(value)
2838
+ if unknown:
2839
+ raise ValidationError(f"{path} has unknown fields: {', '.join(sorted(unknown))}")
2840
+ if missing:
2841
+ raise ValidationError(f"{path} is missing fields: {', '.join(sorted(missing))}")
2842
+ return value
2843
+
2844
+
2845
+ def _nonempty_object(value: Any, *, path: str) -> Mapping[str, Any]:
2846
+ if not isinstance(value, Mapping) or not value:
2847
+ raise ValidationError(f"{path} must be a non-empty object")
2848
+ if any(not isinstance(key, str) or not key.strip() for key in value):
2849
+ raise ValidationError(f"{path} keys must be non-empty strings")
2850
+ return value
2851
+
2852
+
2853
+ def _text(value: Any, path: str) -> str:
2854
+ if not isinstance(value, str) or not value.strip() or value != value.strip():
2855
+ raise ValidationError(f"{path} must be trimmed non-empty text")
2856
+ return value
2857
+
2858
+
2859
+ def _require_text(value: Any, *, label: str) -> None:
2860
+ if (
2861
+ not isinstance(value, str)
2862
+ or not value.strip()
2863
+ or value != value.strip()
2864
+ or any(not character.isprintable() for character in value)
2865
+ ):
2866
+ raise ValidationError(f"{label} must be trimmed non-empty printable text")
2867
+
2868
+
2869
+ def _require_sha256(value: Any, *, label: str) -> None:
2870
+ if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
2871
+ raise ValidationError(f"{label} must be a lowercase SHA-256")
2872
+
2873
+
2874
+ def _require_seed(value: Any) -> None:
2875
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
2876
+ raise ValidationError("external arena seed must be a non-negative integer")
2877
+
2878
+
2879
+ def _finite_number(value: Any, *, label: str) -> float:
2880
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
2881
+ raise ValidationError(f"{label} must be finite numeric")
2882
+ return float(value)
2883
+
2884
+
2885
+ def _safe_message(error: Exception) -> str:
2886
+ return f"{type(error).__name__}: {_safe_text(str(error))}"
2887
+
2888
+
2889
+ def _safe_text(value: str) -> str:
2890
+ result = value[:2048]
2891
+ for pattern in _SECRET_TEXT_PATTERNS:
2892
+ result = pattern.sub("<redacted>", result)
2893
+ return result
2894
+
2895
+
2896
+ __all__ = [
2897
+ "CONFIG_PATH_ENV",
2898
+ "CONFIG_SHA256_ENV",
2899
+ "ExternalArenaCheckpoint",
2900
+ "ExternalArenaExperimentDriver",
2901
+ "ExternalArmPayload",
2902
+ "ExternalArmPayloadBuildRequest",
2903
+ "ExternalBridgeBuildRequest",
2904
+ "ExternalCheckpointBuildRequest",
2905
+ "ExternalUnitPreflightRequest",
2906
+ "LockedExternalArenaBuilder",
2907
+ "factory",
2908
+ ]