agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,147 @@
1
+ """Materialize compact study blueprints against one content-bound arm catalog."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import re
8
+ from collections.abc import Mapping
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from agentinputlab.contracts.errors import ArtifactError, ValidationError
13
+ from agentinputlab.contracts.serialization import content_hash
14
+
15
+ from .schema import ExperimentManifest
16
+
17
+ _ENVIRONMENT_NAME = re.compile(r"^[A-Z_][A-Z0-9_]*$")
18
+ _GIT_OBJECT_ID_LENGTHS = frozenset({40, 64})
19
+
20
+
21
+ def load_experiment_blueprint(
22
+ blueprint_path: str | Path,
23
+ *,
24
+ arm_catalog_path: str | Path,
25
+ environment: Mapping[str, str] | None = None,
26
+ ) -> ExperimentManifest:
27
+ blueprint = _read_object(blueprint_path, label="experiment blueprint")
28
+ catalog = _read_object(arm_catalog_path, label="experiment arm catalog")
29
+ if set(blueprint) != {"schema_version", "arm_set", "manifest"}:
30
+ raise ValidationError("experiment blueprint fields are invalid")
31
+ if blueprint["schema_version"] != 1:
32
+ raise ValidationError("unsupported experiment blueprint schema")
33
+ arm_set_id = blueprint["arm_set"]
34
+ if not isinstance(arm_set_id, str) or not arm_set_id.strip():
35
+ raise ValidationError("experiment blueprint arm_set must be non-empty")
36
+ if set(catalog) != {"schema_version", "arm_sets"} or catalog["schema_version"] != 1:
37
+ raise ValidationError("unsupported experiment arm catalog schema")
38
+ arm_sets = catalog["arm_sets"]
39
+ if not isinstance(arm_sets, Mapping):
40
+ raise ValidationError("experiment arm_sets must be an object")
41
+ try:
42
+ arms = arm_sets[arm_set_id]
43
+ except KeyError as exc:
44
+ raise ValidationError(f"unknown experiment arm set: {arm_set_id}") from exc
45
+ manifest = blueprint["manifest"]
46
+ if not isinstance(manifest, Mapping):
47
+ raise ValidationError("experiment blueprint manifest must be an object")
48
+ if "arms" in manifest:
49
+ raise ValidationError("experiment blueprint cannot override catalog arms")
50
+ metadata = manifest.get("metadata", {})
51
+ if not isinstance(metadata, Mapping):
52
+ raise ValidationError("experiment blueprint metadata must be an object")
53
+ metadata = dict(metadata)
54
+ manifest_fields = dict(manifest)
55
+ benchmarks = manifest_fields.get("benchmarks")
56
+ env = os.environ if environment is None else environment
57
+ revision_environment = metadata.get("source_revision_placeholder_must_match_env")
58
+ if revision_environment is not None:
59
+ if not isinstance(revision_environment, str) or not revision_environment:
60
+ raise ValidationError(
61
+ "source_revision_placeholder_must_match_env must be a non-empty "
62
+ "environment-variable name"
63
+ )
64
+ revision = env.get(revision_environment)
65
+ if revision:
66
+ if not isinstance(benchmarks, list):
67
+ raise ValidationError("experiment blueprint benchmarks must be an array")
68
+ bound_benchmarks: list[Any] = []
69
+ for target in benchmarks:
70
+ if not isinstance(target, Mapping):
71
+ raise ValidationError("experiment blueprint benchmark targets must be objects")
72
+ bound_target = dict(target)
73
+ if bound_target.get("external") is True:
74
+ bound_target["source_revision"] = revision
75
+ bound_benchmarks.append(bound_target)
76
+ manifest_fields["benchmarks"] = bound_benchmarks
77
+
78
+ revision_bindings = metadata.get("checkout_expected_revision_from_env")
79
+ if revision_bindings is not None:
80
+ if not isinstance(revision_bindings, Mapping) or not revision_bindings:
81
+ raise ValidationError("checkout_expected_revision_from_env must be a non-empty object")
82
+ literal_expectations = metadata.get("checkout_expected_revisions", {})
83
+ if not isinstance(literal_expectations, Mapping):
84
+ raise ValidationError("checkout_expected_revisions must be an object")
85
+ materialized_expectations = dict(literal_expectations)
86
+ for checkout_environment, source_revision_environment in revision_bindings.items():
87
+ if (
88
+ not isinstance(checkout_environment, str)
89
+ or _ENVIRONMENT_NAME.fullmatch(checkout_environment) is None
90
+ or not isinstance(source_revision_environment, str)
91
+ or _ENVIRONMENT_NAME.fullmatch(source_revision_environment) is None
92
+ ):
93
+ raise ValidationError(
94
+ "checkout_expected_revision_from_env must map environment-variable "
95
+ "names to environment-variable names"
96
+ )
97
+ revision = env.get(source_revision_environment)
98
+ if not revision:
99
+ continue
100
+ _require_git_object_id(
101
+ revision,
102
+ label=f"environment {source_revision_environment}",
103
+ )
104
+ existing = materialized_expectations.get(checkout_environment)
105
+ if existing is not None and existing != revision:
106
+ raise ValidationError(
107
+ f"checkout revision for {checkout_environment} conflicts with "
108
+ "checkout_expected_revisions"
109
+ )
110
+ materialized_expectations[checkout_environment] = revision
111
+ if materialized_expectations:
112
+ metadata["checkout_expected_revisions"] = materialized_expectations
113
+
114
+ merged: dict[str, Any] = {
115
+ **manifest_fields,
116
+ "arms": arms,
117
+ "metadata": {
118
+ **metadata,
119
+ "blueprint_hash": content_hash(blueprint),
120
+ "arm_set_id": arm_set_id,
121
+ "arm_set_hash": content_hash({"arm_set_id": arm_set_id, "arms": arms}),
122
+ },
123
+ }
124
+ return ExperimentManifest.from_mapping(merged)
125
+
126
+
127
+ def _require_git_object_id(value: str, *, label: str) -> None:
128
+ if (
129
+ not isinstance(value, str)
130
+ or len(value) not in _GIT_OBJECT_ID_LENGTHS
131
+ or any(character not in "0123456789abcdef" for character in value)
132
+ ):
133
+ raise ValidationError(f"{label} must be a full lowercase Git object ID")
134
+
135
+
136
+ def _read_object(path: str | Path, *, label: str) -> Mapping[str, Any]:
137
+ source = Path(path).expanduser().resolve()
138
+ try:
139
+ value = json.loads(source.read_text(encoding="utf-8"))
140
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
141
+ raise ArtifactError(f"{label} is not valid UTF-8 JSON: {source}") from exc
142
+ if not isinstance(value, Mapping):
143
+ raise ValidationError(f"{label} must be an object")
144
+ return value
145
+
146
+
147
+ __all__ = ["load_experiment_blueprint"]
@@ -0,0 +1,45 @@
1
+ """Reviewed, checksum-lockable component builders for production experiments."""
2
+
3
+ from .executable_external_payload import (
4
+ EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
5
+ ExecutableReleaseArmPayloadBuilder,
6
+ executable_release_arm_payload_builder_factory,
7
+ )
8
+ from .external_process_arena import (
9
+ ATOMIC_PROCESS_BRIDGE_PROTOCOL,
10
+ ATOMIC_PROCESS_REQUEST_PROTOCOL,
11
+ ATOMIC_PROCESS_RESPONSE_PROTOCOL,
12
+ CHECKPOINT_INVENTORY_PROTOCOL,
13
+ RELEASE_PAYLOAD_PROTOCOL,
14
+ AtomicProcessBridgeBuilder,
15
+ FrozenCheckpointInventoryBuilder,
16
+ ReleaseArmPayloadBuilder,
17
+ atomic_process_bridge_builder_factory,
18
+ frozen_checkpoint_inventory_builder_factory,
19
+ release_arm_payload_builder_factory,
20
+ )
21
+ from .registry_intervention import (
22
+ REGISTRY_INTERVENTION_BUILDER_PROTOCOL,
23
+ RegistryInterventionBuilder,
24
+ registry_intervention_builder_factory,
25
+ )
26
+
27
+ __all__ = [
28
+ "ATOMIC_PROCESS_BRIDGE_PROTOCOL",
29
+ "ATOMIC_PROCESS_REQUEST_PROTOCOL",
30
+ "ATOMIC_PROCESS_RESPONSE_PROTOCOL",
31
+ "CHECKPOINT_INVENTORY_PROTOCOL",
32
+ "EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL",
33
+ "REGISTRY_INTERVENTION_BUILDER_PROTOCOL",
34
+ "RELEASE_PAYLOAD_PROTOCOL",
35
+ "AtomicProcessBridgeBuilder",
36
+ "ExecutableReleaseArmPayloadBuilder",
37
+ "FrozenCheckpointInventoryBuilder",
38
+ "RegistryInterventionBuilder",
39
+ "ReleaseArmPayloadBuilder",
40
+ "atomic_process_bridge_builder_factory",
41
+ "executable_release_arm_payload_builder_factory",
42
+ "frozen_checkpoint_inventory_builder_factory",
43
+ "registry_intervention_builder_factory",
44
+ "release_arm_payload_builder_factory",
45
+ ]
@@ -0,0 +1,423 @@
1
+ """Execute a registered intervention before entering an official external arena.
2
+
3
+ The external-arena transport deliberately keeps the protected benchmark
4
+ runtime in a separate process. A release name alone is not an intervention,
5
+ however: the selected algorithm must first run against the exact frozen,
6
+ optimizer-visible state. This builder performs that missing production step.
7
+
8
+ It reuses the same checksum-locked backend and registry-intervention builders
9
+ as the static production driver. The resulting payload contains the complete
10
+ ``InputPatch``, its materialized post-intervention state, and measured
11
+ intervention usage. The official arena worker can therefore inject the
12
+ materialized state without reimplementing or guessing any algorithm logic.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from collections.abc import Mapping, Sequence
18
+ from types import MappingProxyType
19
+ from typing import Any
20
+
21
+ from agentinputlab.contracts.enums import SemanticFamily
22
+ from agentinputlab.contracts.errors import (
23
+ ArmExecutionError,
24
+ ArtifactError,
25
+ BudgetExceededError,
26
+ IntentViolationError,
27
+ ValidationError,
28
+ )
29
+ from agentinputlab.contracts.lifecycle import Intervention, InterventionContext
30
+ from agentinputlab.contracts.serialization import content_hash, freeze_mapping, to_jsonable
31
+ from agentinputlab.experiments.drivers.static_production import (
32
+ LockedBuilderSpec,
33
+ StaticArmBuildRequest,
34
+ StaticBackendBuildRequest,
35
+ StaticRuntimeComponents,
36
+ static_runtime_product_sha256,
37
+ )
38
+ from agentinputlab.experiments.loader import resolve_driver_entrypoint
39
+ from agentinputlab.observability import RunMonitor
40
+ from agentinputlab.runner import InterventionRunner
41
+ from agentinputlab.selectors import NoOpIntervention
42
+
43
+ from .external_process_arena import (
44
+ ExternalArmPayload,
45
+ ExternalArmPayloadBuildRequest,
46
+ )
47
+
48
+ EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL = "agentinputlab.executable-release-payload.v1"
49
+
50
+ _SCHEMA_VERSION = 1
51
+ _FIELDS = frozenset(
52
+ {
53
+ "schema_version",
54
+ "protocol",
55
+ "allowed_release_ids",
56
+ "runtime_by_model_slot",
57
+ "intervention_builder",
58
+ }
59
+ )
60
+
61
+
62
+ class ExecutableReleaseArmPayloadBuilder:
63
+ """Run one unchanged registered release and serialize its materialized patch."""
64
+
65
+ builder_kind = "arm_payload"
66
+
67
+ def fingerprint(self) -> Mapping[str, Any]:
68
+ return freeze_mapping(
69
+ {
70
+ "builder_id": EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
71
+ "schema_version": _SCHEMA_VERSION,
72
+ "runtime_source": "checksum_locked_backend_builder",
73
+ "intervention_source": "checksum_locked_intervention_builder",
74
+ "execution": "agentinputlab.InterventionRunner",
75
+ "payload": "patch_plus_materialized_visible_state_plus_usage",
76
+ "protected_runtime_access": False,
77
+ }
78
+ )
79
+
80
+ def preflight(self, parameters: Mapping[str, Any]) -> Sequence[str]:
81
+ try:
82
+ parsed = _parse_parameters(parameters)
83
+ for model_slot, runtime_spec in parsed["runtime_by_model_slot"].items():
84
+ builder = _resolve_builder(runtime_spec, expected_kind="backend")
85
+ _require_clean_preflight(
86
+ builder,
87
+ runtime_spec.parameters,
88
+ label=f"runtime_by_model_slot.{model_slot}",
89
+ )
90
+ intervention_spec = parsed["intervention_builder"]
91
+ intervention_builder = _resolve_builder(
92
+ intervention_spec,
93
+ expected_kind="intervention",
94
+ )
95
+ _require_clean_preflight(
96
+ intervention_builder,
97
+ intervention_spec.parameters,
98
+ label="intervention_builder",
99
+ )
100
+ except (ArtifactError, ValidationError, TypeError, ValueError) as exc:
101
+ return (str(exc),)
102
+ return ()
103
+
104
+ def build(self, request: ExternalArmPayloadBuildRequest) -> ExternalArmPayload:
105
+ if not isinstance(request, ExternalArmPayloadBuildRequest):
106
+ raise TypeError(
107
+ "executable release payload builder requires ExternalArmPayloadBuildRequest"
108
+ )
109
+ parsed = _parse_parameters(request.builder_parameters)
110
+ if request.release_id not in parsed["allowed_release_ids"]:
111
+ raise ValidationError("release is outside the executable payload allowlist")
112
+ if request.arm_kind.value != "release":
113
+ raise ValidationError(
114
+ "executable release payload builder does not construct selector policy arms"
115
+ )
116
+
117
+ state = request.checkpoint.visible_state
118
+ if request.release_id == "baseline.no_op":
119
+ if request.family is not SemanticFamily.NO_OP:
120
+ raise ValidationError("baseline.no_op requires the no_op family")
121
+ runtime = None
122
+ intervention: Intervention = NoOpIntervention()
123
+ runtime_product_sha256 = content_hash(
124
+ {
125
+ "schema_version": 1,
126
+ "runtime": "not_used_by_no_op_intervention",
127
+ "model_slot": request.model_slot,
128
+ }
129
+ )
130
+ context = InterventionContext(
131
+ seed=request.seed,
132
+ run_config={
133
+ "benchmark_id": request.checkpoint.benchmark_id,
134
+ "checkpoint_id": request.checkpoint.checkpoint_id,
135
+ "model_slot": request.model_slot,
136
+ },
137
+ monitor=RunMonitor(),
138
+ )
139
+ else:
140
+ if request.family is SemanticFamily.NO_OP:
141
+ raise ValidationError("non-baseline release cannot use the no_op family")
142
+ runtime_spec = _runtime_spec(parsed, request.model_slot)
143
+ runtime_builder = _resolve_builder(runtime_spec, expected_kind="backend")
144
+ runtime = runtime_builder.build(
145
+ StaticBackendBuildRequest(
146
+ model_slot=request.model_slot,
147
+ seed=request.seed,
148
+ builder_parameters=runtime_spec.parameters,
149
+ )
150
+ )
151
+ if not isinstance(runtime, StaticRuntimeComponents):
152
+ raise ValidationError(
153
+ "external payload backend builder must return StaticRuntimeComponents"
154
+ )
155
+ runtime_product_sha256 = static_runtime_product_sha256(runtime)
156
+ if runtime_product_sha256 != runtime_spec.expected_product_sha256:
157
+ raise ArtifactError("external payload runtime product checksum mismatch")
158
+ _bind_benchmark_retriever(runtime, request.checkpoint.benchmark_id)
159
+
160
+ intervention_spec = parsed["intervention_builder"]
161
+ intervention_builder = _resolve_builder(
162
+ intervention_spec,
163
+ expected_kind="intervention",
164
+ )
165
+ intervention = intervention_builder.build(
166
+ StaticArmBuildRequest(
167
+ arm_id=request.arm_id,
168
+ release_id=request.release_id,
169
+ family=request.family,
170
+ profile_kind=request.profile_kind,
171
+ model_slot=request.model_slot,
172
+ seed=request.seed,
173
+ builder_parameters=intervention_spec.parameters,
174
+ arm_parameters=request.arm_parameters,
175
+ runtime=runtime,
176
+ )
177
+ )
178
+ if not isinstance(intervention, Intervention):
179
+ raise ValidationError(
180
+ "external payload intervention builder did not return an Intervention"
181
+ )
182
+ if (
183
+ intervention.release_id != request.release_id
184
+ or intervention.family is not request.family
185
+ ):
186
+ raise ValidationError(
187
+ "external payload intervention identity differs from the requested arm"
188
+ )
189
+ context = InterventionContext(
190
+ backend=runtime.backend,
191
+ dense_embedder=runtime.dense_embedder,
192
+ retriever=runtime.retriever,
193
+ vector_searcher=runtime.vector_searcher,
194
+ tools=runtime.tools,
195
+ external_runtime=runtime.external_runtime,
196
+ artifact_manager=runtime.artifact_manager,
197
+ seed=request.seed,
198
+ run_config={
199
+ **dict(runtime.run_config),
200
+ "benchmark_id": request.checkpoint.benchmark_id,
201
+ "checkpoint_id": request.checkpoint.checkpoint_id,
202
+ "model_slot": request.model_slot,
203
+ "runtime_product_sha256": runtime_product_sha256,
204
+ },
205
+ intent_contract=runtime.intent_contract,
206
+ monitor=RunMonitor(),
207
+ pricing_catalog=runtime.pricing_catalog,
208
+ strict_intent=runtime.strict_intent,
209
+ )
210
+
211
+ monitor = context.monitor
212
+ if not isinstance(monitor, RunMonitor): # pragma: no cover - construction invariant
213
+ raise TypeError("external payload context monitor must be a RunMonitor")
214
+ intervention_builder_sha256 = (
215
+ content_hash(parsed["intervention_builder"].binding_identity)
216
+ if request.release_id != "baseline.no_op"
217
+ else content_hash({"builder": "builtin.no_op"})
218
+ )
219
+ try:
220
+ record = InterventionRunner().run(intervention, state, context)
221
+ except (ArmExecutionError, BudgetExceededError, IntentViolationError):
222
+ snapshots = monitor.list_snapshots()
223
+ if len(snapshots) != 1 or snapshots[0].failure is None:
224
+ raise ValidationError(
225
+ "arm-local intervention failure has no unique monitored failure receipt"
226
+ ) from None
227
+ snapshot = snapshots[0]
228
+ execution = {
229
+ "status": "failed",
230
+ "input_state_hash": state.state_hash,
231
+ "output_state_hash": state.state_hash,
232
+ "patch": None,
233
+ "patch_hash": None,
234
+ # The official arena may still execute the unchanged state to
235
+ # preserve paired runtime/scorer evidence. Downstream
236
+ # normalization assigns the preregistered algorithm-failure
237
+ # utility; it never mistakes this continuation for success.
238
+ "materialized_state": to_jsonable(state),
239
+ "runtime_product_sha256": runtime_product_sha256,
240
+ "intervention_builder_sha256": intervention_builder_sha256,
241
+ "usage": to_jsonable(snapshot.usage),
242
+ "duration_ms": snapshot.duration_ms,
243
+ "intent_check": None,
244
+ "failure": {
245
+ "scope": "algorithm",
246
+ **to_jsonable(snapshot.failure),
247
+ "monitor_run_id": snapshot.run_id,
248
+ },
249
+ }
250
+ else:
251
+ materialized = record.patch.apply(state)
252
+ if materialized.state_hash != record.output_state_hash:
253
+ raise ValidationError("materialized external payload state hash is inconsistent")
254
+ execution = {
255
+ "status": record.status.value,
256
+ "input_state_hash": record.input_state_hash,
257
+ "output_state_hash": record.output_state_hash,
258
+ "patch": record.patch.as_dict(),
259
+ "patch_hash": record.patch.patch_hash,
260
+ "materialized_state": to_jsonable(materialized),
261
+ "runtime_product_sha256": runtime_product_sha256,
262
+ "intervention_builder_sha256": intervention_builder_sha256,
263
+ "usage": to_jsonable(record.usage),
264
+ "duration_ms": record.duration_ms,
265
+ "intent_check": to_jsonable(record.intent_check),
266
+ "failure": None,
267
+ }
268
+ payload = {
269
+ "schema_version": _SCHEMA_VERSION,
270
+ "protocol": EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
271
+ "arm_id": request.arm_id,
272
+ "release_id": request.release_id,
273
+ "family": request.family.value,
274
+ "profile_kind": request.profile_kind.value,
275
+ "model_slot": request.model_slot,
276
+ "seed": request.seed,
277
+ "checkpoint_identity_hash": request.checkpoint.identity_hash,
278
+ "visible_state_hash": state.state_hash,
279
+ "pre_action": request.pre_action,
280
+ "arm_parameters": request.arm_parameters,
281
+ "intervention_execution": execution,
282
+ }
283
+ return ExternalArmPayload(
284
+ arm_id=request.arm_id,
285
+ release_id=request.release_id,
286
+ family=request.family,
287
+ profile_kind=request.profile_kind,
288
+ payload=payload,
289
+ )
290
+
291
+
292
+ def executable_release_arm_payload_builder_factory() -> ExecutableReleaseArmPayloadBuilder:
293
+ """Return the zero-argument production payload builder."""
294
+
295
+ return ExecutableReleaseArmPayloadBuilder()
296
+
297
+
298
+ def _bind_benchmark_retriever(
299
+ runtime: StaticRuntimeComponents,
300
+ benchmark_id: str,
301
+ ) -> None:
302
+ """Bind the production retriever router to the audited checkpoint cell.
303
+
304
+ Static predictors perform this binding themselves. External-arena payloads
305
+ execute the intervention before a downstream predictor exists, so methods
306
+ such as Self-Ask and IRCoT need the same benchmark-validated route here.
307
+ The router fingerprint is intentionally stable across binding.
308
+ """
309
+
310
+ from agentinputlab.experiments.production_builders import ProductionRetrieverRouter
311
+
312
+ retriever = runtime.retriever
313
+ if isinstance(retriever, ProductionRetrieverRouter):
314
+ retriever.bind(benchmark_id)
315
+
316
+
317
+ def _parse_parameters(parameters: Mapping[str, Any]) -> Mapping[str, Any]:
318
+ if not isinstance(parameters, Mapping) or set(parameters) != _FIELDS:
319
+ raise ValidationError("executable release payload parameters are invalid")
320
+ if (
321
+ parameters["schema_version"] != _SCHEMA_VERSION
322
+ or parameters["protocol"] != EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL
323
+ ):
324
+ raise ValidationError("unsupported executable release payload protocol")
325
+ raw_ids = parameters["allowed_release_ids"]
326
+ if (
327
+ isinstance(raw_ids, (str, bytes))
328
+ or not isinstance(raw_ids, Sequence)
329
+ or not raw_ids
330
+ or len(raw_ids) != len(set(raw_ids))
331
+ or any(not isinstance(item, str) or not item.strip() for item in raw_ids)
332
+ ):
333
+ raise ValidationError(
334
+ "executable payload allowed_release_ids must be unique non-empty strings"
335
+ )
336
+ raw_runtimes = parameters["runtime_by_model_slot"]
337
+ if not isinstance(raw_runtimes, Mapping) or not raw_runtimes:
338
+ raise ValidationError("runtime_by_model_slot must be a non-empty object")
339
+ runtimes: dict[str, LockedBuilderSpec] = {}
340
+ for model_slot, value in raw_runtimes.items():
341
+ if not isinstance(model_slot, str) or not model_slot.strip():
342
+ raise ValidationError("runtime_by_model_slot keys must be non-empty strings")
343
+ spec = LockedBuilderSpec.from_mapping(
344
+ value,
345
+ path=f"runtime_by_model_slot.{model_slot}",
346
+ )
347
+ if spec.expected_product_sha256 is None:
348
+ raise ValidationError(
349
+ f"runtime_by_model_slot.{model_slot} requires expected_product_sha256"
350
+ )
351
+ runtimes[model_slot] = spec
352
+ intervention = LockedBuilderSpec.from_mapping(
353
+ parameters["intervention_builder"],
354
+ path="intervention_builder",
355
+ )
356
+ if intervention.expected_product_sha256 is not None:
357
+ raise ValidationError(
358
+ "executable payload intervention_builder must not declare a product checksum"
359
+ )
360
+ return MappingProxyType(
361
+ {
362
+ "schema_version": _SCHEMA_VERSION,
363
+ "protocol": EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
364
+ "allowed_release_ids": tuple(raw_ids),
365
+ "runtime_by_model_slot": MappingProxyType(runtimes),
366
+ "intervention_builder": intervention,
367
+ }
368
+ )
369
+
370
+
371
+ def _runtime_spec(
372
+ parsed: Mapping[str, Any],
373
+ model_slot: str,
374
+ ) -> LockedBuilderSpec:
375
+ try:
376
+ return parsed["runtime_by_model_slot"][model_slot]
377
+ except KeyError as exc:
378
+ raise ArtifactError(
379
+ f"executable payload has no runtime binding for model slot {model_slot!r}"
380
+ ) from exc
381
+
382
+
383
+ def _resolve_builder(spec: LockedBuilderSpec, *, expected_kind: str) -> Any:
384
+ factory = resolve_driver_entrypoint(spec.entrypoint)
385
+ if not callable(factory):
386
+ raise ArtifactError("executable payload builder entrypoint is not callable")
387
+ try:
388
+ builder = factory()
389
+ except TypeError as exc:
390
+ raise ArtifactError(
391
+ "executable payload builder entrypoint must be a zero-argument factory"
392
+ ) from exc
393
+ if getattr(builder, "builder_kind", None) != expected_kind:
394
+ raise ArtifactError(f"executable payload component kind must be {expected_kind!r}")
395
+ fingerprint = builder.fingerprint()
396
+ if not isinstance(fingerprint, Mapping) or not fingerprint:
397
+ raise ArtifactError("executable payload component fingerprint is invalid")
398
+ if content_hash(fingerprint) != spec.fingerprint_sha256:
399
+ raise ArtifactError("executable payload component fingerprint checksum mismatch")
400
+ return builder
401
+
402
+
403
+ def _require_clean_preflight(
404
+ builder: Any,
405
+ parameters: Mapping[str, Any],
406
+ *,
407
+ label: str,
408
+ ) -> None:
409
+ blockers = builder.preflight(parameters)
410
+ if isinstance(blockers, (str, bytes)) or not isinstance(blockers, Sequence):
411
+ raise ValidationError(f"{label} preflight must return a sequence")
412
+ messages = tuple(blockers)
413
+ if any(not isinstance(message, str) or not message.strip() for message in messages):
414
+ raise ValidationError(f"{label} preflight returned an invalid blocker")
415
+ if messages:
416
+ raise ArtifactError(f"{label}: " + "; ".join(messages))
417
+
418
+
419
+ __all__ = [
420
+ "EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL",
421
+ "ExecutableReleaseArmPayloadBuilder",
422
+ "executable_release_arm_payload_builder_factory",
423
+ ]