agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,819 @@
1
+ """Opt-in, allowlisted research-campaign orchestration for Studio.
2
+
3
+ The ordinary Studio run surface remains a tiny deterministic offline tool.
4
+ This module adds a separate operator-controlled path for the shipped experiment
5
+ blueprints. A browser can select only a profile ID from a catalog loaded at
6
+ server startup; it cannot submit paths, entrypoints, environment values,
7
+ commands, credentials, or arbitrary execution options.
8
+
9
+ Campaign preparation is deliberately the default action. It loads the locked
10
+ blueprint, runs the existing experiment preflight, and materializes an exact
11
+ plan without executing a cell. Execution requires a second request that
12
+ cross-binds the profile, preparation ID, plan hash, and a fixed confirmation
13
+ phrase. Immediately before execution the worker repeats file attestation,
14
+ preflight, and planning, then uses the existing guarded driver and experiment
15
+ executor.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import copy
21
+ import hashlib
22
+ import json
23
+ import logging
24
+ import re
25
+ import threading
26
+ import uuid
27
+ from collections.abc import Mapping
28
+ from concurrent.futures import ThreadPoolExecutor
29
+ from dataclasses import dataclass
30
+ from datetime import datetime, timezone
31
+ from pathlib import Path
32
+ from types import MappingProxyType
33
+ from typing import Any
34
+
35
+ from agentinputlab.contracts.errors import ArtifactError, ValidationError
36
+ from agentinputlab.contracts.serialization import content_hash
37
+ from agentinputlab.experiments import (
38
+ ExperimentArtifactIdentity,
39
+ ExperimentManifest,
40
+ ExperimentPlan,
41
+ PreflightReport,
42
+ activate_guarded_driver,
43
+ build_experiment_plan,
44
+ execute_experiment_plan,
45
+ load_experiment_blueprint,
46
+ preflight_experiment,
47
+ )
48
+
49
+ CAMPAIGN_CONFIRMATION = "execute_frozen_plan"
50
+ MAX_CAMPAIGN_CATALOG_BYTES = 2 * 1024 * 1024
51
+ MAX_CAMPAIGN_PROFILES = 32
52
+ MAX_CAMPAIGN_PREPARATIONS = 100
53
+ MAX_CAMPAIGN_EXECUTIONS = 100
54
+ MAX_PLAN_PREVIEW_CELLS = 100
55
+
56
+ _PROFILE_ID = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,95}$")
57
+ _ENTRYPOINT = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*:[A-Za-z_][A-Za-z0-9_.]*$")
58
+ _SHA256 = re.compile(r"^[0-9a-f]{64}$")
59
+ _LOGGER = logging.getLogger(__name__)
60
+ _EXTERNAL_ARENA_ENTRYPOINT = "agentinputlab.experiments.drivers.external_arena:factory"
61
+ _TYPED_GATE_TARGETS = frozenset(
62
+ {
63
+ "toolsandbox.common_support",
64
+ "appworld_ul.common_support",
65
+ "webarena.external",
66
+ }
67
+ )
68
+
69
+
70
+ class CampaignSecurityError(ValueError):
71
+ """A browser attempted to cross the fixed campaign request boundary."""
72
+
73
+
74
+ class CampaignValidationError(ValueError):
75
+ """A campaign request is well-formed but cannot be accepted."""
76
+
77
+
78
+ class CampaignQueueFullError(RuntimeError):
79
+ """The bounded in-memory campaign queue cannot accept more records."""
80
+
81
+
82
+ @dataclass(frozen=True, slots=True)
83
+ class _LockedCampaignFile:
84
+ path: Path
85
+ sha256: str
86
+ label: str
87
+
88
+ def __post_init__(self) -> None:
89
+ if not self.path.is_absolute():
90
+ raise ValidationError(f"{self.label} path must resolve to an absolute path")
91
+ if _SHA256.fullmatch(self.sha256) is None:
92
+ raise ValidationError(f"{self.label} sha256 must be lowercase SHA-256")
93
+ self.verify()
94
+
95
+ def verify(self) -> None:
96
+ try:
97
+ if not self.path.is_file():
98
+ raise ArtifactError(f"configured {self.label} is not an existing file")
99
+ observed = _sha256_file(self.path)
100
+ except OSError as exc:
101
+ raise ArtifactError(f"configured {self.label} cannot be read") from exc
102
+ if observed != self.sha256:
103
+ raise ArtifactError(f"configured {self.label} checksum mismatch")
104
+
105
+
106
+ @dataclass(frozen=True, slots=True)
107
+ class _CampaignProfile:
108
+ profile_id: str
109
+ label: str
110
+ description: str
111
+ blueprint: _LockedCampaignFile
112
+ arm_catalog: _LockedCampaignFile
113
+ driver_entrypoint: str
114
+ execution_enabled: bool
115
+
116
+ def __post_init__(self) -> None:
117
+ if _PROFILE_ID.fullmatch(self.profile_id) is None:
118
+ raise ValidationError("campaign profile id is invalid")
119
+ for value, label, maximum in (
120
+ (self.label, "label", 160),
121
+ (self.description, "description", 800),
122
+ ):
123
+ if (
124
+ not isinstance(value, str)
125
+ or not value.strip()
126
+ or value != value.strip()
127
+ or len(value) > maximum
128
+ ):
129
+ raise ValidationError(
130
+ f"campaign profile {label} must be trimmed non-empty text "
131
+ f"of at most {maximum} characters"
132
+ )
133
+ if (
134
+ not isinstance(self.driver_entrypoint, str)
135
+ or _ENTRYPOINT.fullmatch(self.driver_entrypoint) is None
136
+ ):
137
+ raise ValidationError("campaign profile driver entrypoint is invalid")
138
+ if not isinstance(self.execution_enabled, bool):
139
+ raise ValidationError("campaign profile execution_enabled must be boolean")
140
+
141
+ @property
142
+ def identity_hash(self) -> str:
143
+ return content_hash(
144
+ {
145
+ "profile_id": self.profile_id,
146
+ "blueprint_sha256": self.blueprint.sha256,
147
+ "arm_catalog_sha256": self.arm_catalog.sha256,
148
+ "driver_entrypoint": self.driver_entrypoint,
149
+ "execution_enabled": self.execution_enabled,
150
+ }
151
+ )
152
+
153
+ def verify_files(self) -> None:
154
+ self.blueprint.verify()
155
+ self.arm_catalog.verify()
156
+
157
+ def load_manifest(self) -> ExperimentManifest:
158
+ # Check both sides of loading so a file replacement cannot silently
159
+ # survive the startup lock or a preparation/execution boundary.
160
+ self.verify_files()
161
+ manifest = load_experiment_blueprint(
162
+ self.blueprint.path,
163
+ arm_catalog_path=self.arm_catalog.path,
164
+ )
165
+ self.verify_files()
166
+ return manifest
167
+
168
+ def public_payload(self, *, output_configured: bool) -> dict[str, Any]:
169
+ return {
170
+ "id": self.profile_id,
171
+ "label": self.label,
172
+ "description": self.description,
173
+ "profile_identity_sha256": self.identity_hash,
174
+ "blueprint_sha256": self.blueprint.sha256,
175
+ "arm_catalog_sha256": self.arm_catalog.sha256,
176
+ "execution_enabled": self.execution_enabled,
177
+ "execution_configured": self.execution_enabled and output_configured,
178
+ "default_action": "preflight_and_plan",
179
+ }
180
+
181
+
182
+ @dataclass(frozen=True, slots=True)
183
+ class _PreparedCampaign:
184
+ preparation_id: str
185
+ created_at: str
186
+ profile: _CampaignProfile
187
+ manifest: ExperimentManifest
188
+ report: PreflightReport
189
+ resolved_units: Mapping[str, tuple[str, ...]]
190
+ plan: ExperimentPlan | None
191
+ execution_blockers: tuple[str, ...]
192
+
193
+ @property
194
+ def can_execute(self) -> bool:
195
+ return self.report.ready and self.plan is not None
196
+
197
+
198
+ def load_campaign_profile_catalog(
199
+ path: str | Path,
200
+ ) -> tuple[Mapping[str, _CampaignProfile], str]:
201
+ """Load one strict operator-owned catalog and attest all referenced files."""
202
+
203
+ source = Path(path).expanduser().resolve()
204
+ try:
205
+ if not source.is_file():
206
+ raise ArtifactError("campaign profile catalog is not an existing file")
207
+ if source.stat().st_size > MAX_CAMPAIGN_CATALOG_BYTES:
208
+ raise ArtifactError("campaign profile catalog exceeds the size limit")
209
+ raw_bytes = source.read_bytes()
210
+ raw = json.loads(raw_bytes.decode("utf-8"))
211
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
212
+ raise ArtifactError("campaign profile catalog is not valid UTF-8 JSON") from exc
213
+ if not isinstance(raw, Mapping) or set(raw) != {"schema_version", "profiles"}:
214
+ raise ValidationError("campaign profile catalog fields are invalid")
215
+ if raw["schema_version"] != 1:
216
+ raise ValidationError("unsupported campaign profile catalog schema")
217
+ rows = raw["profiles"]
218
+ if not isinstance(rows, list) or not rows or len(rows) > MAX_CAMPAIGN_PROFILES:
219
+ raise ValidationError(
220
+ f"campaign profile catalog requires 1-{MAX_CAMPAIGN_PROFILES} profiles"
221
+ )
222
+ profiles: dict[str, _CampaignProfile] = {}
223
+ for index, value in enumerate(rows):
224
+ profile = _parse_profile(
225
+ value,
226
+ base=source.parent,
227
+ path=f"campaign_catalog.profiles[{index}]",
228
+ )
229
+ if profile.profile_id in profiles:
230
+ raise ValidationError("campaign profile IDs must be unique")
231
+ profiles[profile.profile_id] = profile
232
+ return MappingProxyType(profiles), hashlib.sha256(raw_bytes).hexdigest()
233
+
234
+
235
+ class CampaignManager:
236
+ """Prepare and explicitly queue allowlisted experiment campaigns."""
237
+
238
+ def __init__(
239
+ self,
240
+ *,
241
+ catalog_path: str | Path,
242
+ output_root: str | Path | None = None,
243
+ max_workers: int = 1,
244
+ max_preparations: int = MAX_CAMPAIGN_PREPARATIONS,
245
+ max_executions: int = MAX_CAMPAIGN_EXECUTIONS,
246
+ ) -> None:
247
+ if (
248
+ isinstance(max_workers, bool)
249
+ or not isinstance(max_workers, int)
250
+ or not 1 <= max_workers <= 4
251
+ ):
252
+ raise ValueError("campaign max_workers must be in [1, 4]")
253
+ for value, label in (
254
+ (max_preparations, "max_preparations"),
255
+ (max_executions, "max_executions"),
256
+ ):
257
+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:
258
+ raise ValueError(f"campaign {label} must be a positive integer")
259
+ profiles, catalog_sha256 = load_campaign_profile_catalog(catalog_path)
260
+ self._profiles = profiles
261
+ self._catalog_sha256 = catalog_sha256
262
+ self._output_root = (
263
+ Path(output_root).expanduser().resolve() if output_root is not None else None
264
+ )
265
+ self._max_preparations = max_preparations
266
+ self._max_executions = max_executions
267
+ self._preparations: dict[str, _PreparedCampaign] = {}
268
+ self._preparation_payloads: dict[str, dict[str, Any]] = {}
269
+ self._executions: dict[str, dict[str, Any]] = {}
270
+ self._submitted_preparations: set[str] = set()
271
+ self._lock = threading.RLock()
272
+ self._executor = ThreadPoolExecutor(
273
+ max_workers=max_workers,
274
+ thread_name_prefix="ailab-campaign",
275
+ )
276
+
277
+ @property
278
+ def configured(self) -> bool:
279
+ return True
280
+
281
+ def catalog_payload(self) -> dict[str, Any]:
282
+ output_configured = self._output_root is not None
283
+ return {
284
+ "schema_version": 1,
285
+ "configured": True,
286
+ "catalog_sha256": self._catalog_sha256,
287
+ "default_action": "preflight_and_plan",
288
+ "browser_request_schema": {
289
+ "preflight": ["profile_id"],
290
+ "execution": [
291
+ "profile_id",
292
+ "plan_id",
293
+ "plan_hash",
294
+ "confirmation",
295
+ ],
296
+ },
297
+ "profiles": [
298
+ profile.public_payload(output_configured=output_configured)
299
+ for profile in self._profiles.values()
300
+ ],
301
+ }
302
+
303
+ def prepare(self, payload: Any) -> dict[str, Any]:
304
+ profile_id = _profile_only_request(payload)
305
+ with self._lock:
306
+ if len(self._preparations) >= self._max_preparations:
307
+ raise CampaignQueueFullError("campaign preparation limit reached")
308
+ profile = self._profiles.get(profile_id)
309
+ if profile is None:
310
+ raise CampaignValidationError("campaign profile is not allowlisted")
311
+
312
+ manifest, report, units, plan = self._load_preflight_and_plan(profile)
313
+ preparation = _PreparedCampaign(
314
+ preparation_id=uuid.uuid4().hex,
315
+ created_at=_utc_now(),
316
+ profile=profile,
317
+ manifest=manifest,
318
+ report=report,
319
+ resolved_units=units,
320
+ plan=plan,
321
+ execution_blockers=_campaign_execution_blockers(
322
+ profile,
323
+ manifest,
324
+ report,
325
+ ),
326
+ )
327
+ public = self._preparation_payload(preparation)
328
+ with self._lock:
329
+ if len(self._preparations) >= self._max_preparations:
330
+ raise CampaignQueueFullError("campaign preparation limit reached")
331
+ self._preparations[preparation.preparation_id] = preparation
332
+ self._preparation_payloads[preparation.preparation_id] = public
333
+ return copy.deepcopy(public)
334
+
335
+ def get_preparation(self, preparation_id: str) -> dict[str, Any] | None:
336
+ if not isinstance(preparation_id, str) or not re.fullmatch(
337
+ r"[0-9a-f]{32}",
338
+ preparation_id,
339
+ ):
340
+ return None
341
+ with self._lock:
342
+ payload = self._preparation_payloads.get(preparation_id)
343
+ return copy.deepcopy(payload) if payload is not None else None
344
+
345
+ def submit_execution(self, payload: Any) -> dict[str, Any]:
346
+ request = _execution_request(payload)
347
+ with self._lock:
348
+ if len(self._executions) >= self._max_executions:
349
+ raise CampaignQueueFullError("campaign execution queue limit reached")
350
+ preparation = self._preparations.get(request["plan_id"])
351
+ if preparation is None:
352
+ raise CampaignValidationError("campaign plan does not exist")
353
+ if preparation.preparation_id in self._submitted_preparations:
354
+ raise CampaignValidationError(
355
+ "campaign plan was already submitted; prepare a new plan to run again"
356
+ )
357
+ if request["profile_id"] != preparation.profile.profile_id:
358
+ raise CampaignValidationError("campaign profile differs from the frozen plan")
359
+ if preparation.plan is None or not preparation.report.ready:
360
+ raise CampaignValidationError("campaign preflight is blocked")
361
+ if preparation.execution_blockers:
362
+ raise CampaignValidationError(
363
+ "campaign execution is blocked by an unverified driver safety contract"
364
+ )
365
+ if request["plan_hash"] != preparation.plan.plan_hash:
366
+ raise CampaignValidationError("campaign plan hash does not match")
367
+ if not preparation.profile.execution_enabled or self._output_root is None:
368
+ raise CampaignValidationError(
369
+ "campaign execution is not configured for this profile"
370
+ )
371
+ execution_id = uuid.uuid4().hex
372
+ record = {
373
+ "id": execution_id,
374
+ "profile_id": preparation.profile.profile_id,
375
+ "profile_identity_sha256": preparation.profile.identity_hash,
376
+ "plan_id": preparation.preparation_id,
377
+ "plan_hash": preparation.plan.plan_hash,
378
+ "manifest_hash": preparation.manifest.manifest_hash,
379
+ "guarded_driver_hash": preparation.report.guarded_driver_hash,
380
+ "status": "queued",
381
+ "queued_at": _utc_now(),
382
+ "started_at": None,
383
+ "finished_at": None,
384
+ "progress": {
385
+ "total_cells": len(preparation.plan.cells),
386
+ "completed_cells": 0,
387
+ "attempted_cells": 0,
388
+ "succeeded_cells": 0,
389
+ "failed_cells": 0,
390
+ "skipped_cells": 0,
391
+ },
392
+ "result_identity": None,
393
+ "summary": None,
394
+ "error": None,
395
+ }
396
+ self._executions[execution_id] = record
397
+ self._submitted_preparations.add(preparation.preparation_id)
398
+ try:
399
+ self._executor.submit(self._run_execution, execution_id, preparation)
400
+ except RuntimeError:
401
+ with self._lock:
402
+ self._executions.pop(execution_id, None)
403
+ self._submitted_preparations.discard(preparation.preparation_id)
404
+ raise CampaignQueueFullError("campaign executor is unavailable") from None
405
+ return copy.deepcopy(record)
406
+
407
+ def list_executions(self) -> list[dict[str, Any]]:
408
+ with self._lock:
409
+ records = sorted(
410
+ self._executions.values(),
411
+ key=lambda item: str(item["queued_at"]),
412
+ reverse=True,
413
+ )
414
+ return copy.deepcopy(records)
415
+
416
+ def get_execution(self, execution_id: str) -> dict[str, Any] | None:
417
+ if not isinstance(execution_id, str) or not re.fullmatch(
418
+ r"[0-9a-f]{32}",
419
+ execution_id,
420
+ ):
421
+ return None
422
+ with self._lock:
423
+ record = self._executions.get(execution_id)
424
+ return copy.deepcopy(record) if record is not None else None
425
+
426
+ def shutdown(self, *, wait: bool = True) -> None:
427
+ self._executor.shutdown(wait=wait, cancel_futures=not wait)
428
+
429
+ def _load_preflight_and_plan(
430
+ self,
431
+ profile: _CampaignProfile,
432
+ ) -> tuple[
433
+ ExperimentManifest,
434
+ PreflightReport,
435
+ Mapping[str, tuple[str, ...]],
436
+ ExperimentPlan | None,
437
+ ]:
438
+ manifest = profile.load_manifest()
439
+ report, units = preflight_experiment(
440
+ manifest,
441
+ driver_entrypoint=profile.driver_entrypoint,
442
+ resolve_units=True,
443
+ )
444
+ plan = build_experiment_plan(manifest, units) if report.ready else None
445
+ return manifest, report, units, plan
446
+
447
+ def _preparation_payload(self, preparation: _PreparedCampaign) -> dict[str, Any]:
448
+ manifest = preparation.manifest
449
+ report = preparation.report
450
+ plan = preparation.plan
451
+ output_configured = self._output_root is not None
452
+ plan_payload: dict[str, Any] | None = None
453
+ if plan is not None:
454
+ preview = tuple(plan.cells[:MAX_PLAN_PREVIEW_CELLS])
455
+ plan_payload = {
456
+ "id": preparation.preparation_id,
457
+ "plan_hash": plan.plan_hash,
458
+ "cell_count": len(plan.cells),
459
+ "preview_count": len(preview),
460
+ "preview_truncated": len(preview) != len(plan.cells),
461
+ "unit_inventory_hashes": dict(plan.unit_inventory_hashes),
462
+ "cells": [
463
+ {
464
+ "cell_id": cell.cell_id,
465
+ "benchmark_id": cell.benchmark_id,
466
+ "split": cell.split,
467
+ "unit_id": cell.unit_id,
468
+ "model_slot": cell.model_slot,
469
+ "seed": cell.seed,
470
+ "arm_ids": list(cell.arm_ids),
471
+ }
472
+ for cell in preview
473
+ ],
474
+ }
475
+ return {
476
+ "schema_version": 1,
477
+ "id": preparation.preparation_id,
478
+ "created_at": preparation.created_at,
479
+ "profile": preparation.profile.public_payload(output_configured=output_configured),
480
+ "manifest": {
481
+ "study_id": manifest.study_id,
482
+ "stage": manifest.stage.value,
483
+ "manifest_hash": manifest.manifest_hash,
484
+ "estimand_profile_kind": manifest.estimand_profile_kind.value,
485
+ "benchmarks": [
486
+ {
487
+ "cell_id": target.cell_id,
488
+ "split": target.split,
489
+ "external": target.external,
490
+ "source_revision": target.source_revision,
491
+ "arm_ids": list(
492
+ target.arm_ids or tuple(arm.arm_id for arm in manifest.arms)
493
+ ),
494
+ }
495
+ for target in manifest.benchmarks
496
+ ],
497
+ "arms": [
498
+ {
499
+ "arm_id": arm.arm_id,
500
+ "release_id": arm.release_id,
501
+ "family": arm.family.value,
502
+ "profile_kind": arm.profile_kind.value,
503
+ "arm_kind": arm.arm_kind.value,
504
+ }
505
+ for arm in manifest.arms
506
+ ],
507
+ "model_slots": list(manifest.model_slots),
508
+ "seeds": list(manifest.seeds),
509
+ },
510
+ "preflight": {
511
+ "ready": report.ready,
512
+ "blockers": list(report.blockers),
513
+ "warnings": list(report.warnings),
514
+ "resolved_unit_counts": dict(report.resolved_unit_counts),
515
+ "guarded_driver_hash": report.guarded_driver_hash,
516
+ },
517
+ "plan": plan_payload,
518
+ "execution": {
519
+ "requires_separate_confirmation": True,
520
+ "confirmation_phrase": CAMPAIGN_CONFIRMATION,
521
+ "profile_enabled": preparation.profile.execution_enabled,
522
+ "output_configured": output_configured,
523
+ "ready_for_confirmation": (
524
+ preparation.can_execute
525
+ and preparation.profile.execution_enabled
526
+ and output_configured
527
+ and not preparation.execution_blockers
528
+ ),
529
+ "blockers": list(preparation.execution_blockers),
530
+ },
531
+ }
532
+
533
+ def _run_execution(
534
+ self,
535
+ execution_id: str,
536
+ preparation: _PreparedCampaign,
537
+ ) -> None:
538
+ with self._lock:
539
+ record = self._executions[execution_id]
540
+ record["status"] = "running"
541
+ record["started_at"] = _utc_now()
542
+ try:
543
+ manifest, report, _units, plan = self._load_preflight_and_plan(preparation.profile)
544
+ execution_blockers = _campaign_execution_blockers(
545
+ preparation.profile,
546
+ manifest,
547
+ report,
548
+ )
549
+ if (
550
+ not report.ready
551
+ or plan is None
552
+ or execution_blockers
553
+ or manifest.manifest_hash != preparation.manifest.manifest_hash
554
+ or plan.plan_hash
555
+ != (preparation.plan.plan_hash if preparation.plan is not None else None)
556
+ or report.guarded_driver_hash != preparation.report.guarded_driver_hash
557
+ ):
558
+ raise ArtifactError("campaign bindings changed after the confirmed preflight")
559
+ if (
560
+ report.binding_receipt is None
561
+ or report.guarded_driver_hash is None
562
+ or not report.operator_driver_fingerprint
563
+ ):
564
+ raise ArtifactError("ready campaign preflight lacks guarded identity")
565
+ if self._output_root is None: # pragma: no cover - submit invariant
566
+ raise ArtifactError("campaign output root is not configured")
567
+ artifact_identity = ExperimentArtifactIdentity(
568
+ study_id=manifest.study_id,
569
+ manifest_hash=manifest.manifest_hash,
570
+ plan_hash=plan.plan_hash,
571
+ driver_hash=report.guarded_driver_hash,
572
+ )
573
+ with activate_guarded_driver(
574
+ manifest=manifest,
575
+ operator_entrypoint=preparation.profile.driver_entrypoint,
576
+ operator_fingerprint=report.operator_driver_fingerprint,
577
+ binding_receipt=report.binding_receipt,
578
+ parity_contract=report.parity_contract,
579
+ ) as guarded_entrypoint:
580
+ summary = execute_experiment_plan(
581
+ manifest,
582
+ plan,
583
+ driver_entrypoint=guarded_entrypoint,
584
+ output_root=str(self._output_root / execution_id),
585
+ resume=False,
586
+ fail_fast=False,
587
+ )
588
+ completed = summary.succeeded_cells + summary.failed_cells + summary.skipped_cells
589
+ with self._lock:
590
+ record = self._executions[execution_id]
591
+ record["status"] = (
592
+ "completed" if summary.failed_cells == 0 else "completed_with_failures"
593
+ )
594
+ record["finished_at"] = _utc_now()
595
+ record["progress"] = {
596
+ "total_cells": summary.planned_cells,
597
+ "completed_cells": completed,
598
+ "attempted_cells": summary.attempted_cells,
599
+ "succeeded_cells": summary.succeeded_cells,
600
+ "failed_cells": summary.failed_cells,
601
+ "skipped_cells": summary.skipped_cells,
602
+ }
603
+ record["result_identity"] = {
604
+ "artifact_identity_sha256": artifact_identity.identity_hash,
605
+ "manifest_sha256": manifest.manifest_hash,
606
+ "plan_sha256": plan.plan_hash,
607
+ "driver_sha256": report.guarded_driver_hash,
608
+ }
609
+ record["summary"] = {
610
+ "planned_cells": summary.planned_cells,
611
+ "attempted_cells": summary.attempted_cells,
612
+ "skipped_cells": summary.skipped_cells,
613
+ "succeeded_cells": summary.succeeded_cells,
614
+ "failed_cells": summary.failed_cells,
615
+ }
616
+ except Exception as exc:
617
+ _LOGGER.exception("Studio campaign execution %s failed", execution_id)
618
+ with self._lock:
619
+ record = self._executions[execution_id]
620
+ record["status"] = "failed"
621
+ record["finished_at"] = _utc_now()
622
+ record["error"] = {
623
+ "category": type(exc).__name__,
624
+ "message": (
625
+ "Campaign execution aborted. Inspect the server log and "
626
+ "content-addressed experiment artifacts for details."
627
+ ),
628
+ }
629
+
630
+
631
+ def _campaign_execution_blockers(
632
+ profile: _CampaignProfile,
633
+ manifest: ExperimentManifest,
634
+ report: PreflightReport,
635
+ ) -> tuple[str, ...]:
636
+ """Require fingerprinted P0 controls before external-arena execution.
637
+
638
+ External profiles remain available for useful preflight and exact planning.
639
+ They cannot be confirmed until the selected driver fingerprint explicitly
640
+ attests the safety contracts relevant to the manifest.
641
+ """
642
+
643
+ if profile.driver_entrypoint != _EXTERNAL_ARENA_ENTRYPOINT:
644
+ return ()
645
+ raw = report.operator_driver_fingerprint.get("studio_campaign_attestation")
646
+ attestation = raw if isinstance(raw, Mapping) else {}
647
+ blockers: list[str] = []
648
+ if (
649
+ attestation.get("schema_version") != 1
650
+ or attestation.get("shared_dependency_slots") is not True
651
+ ):
652
+ blockers.append("external driver has not attested required shared dependency slots")
653
+ if any(arm.arm_kind.value == "policy" for arm in manifest.arms) and (
654
+ attestation.get("selector_group_exclusion") is not True
655
+ ):
656
+ blockers.append("external driver has not attested discovery-group exclusion for policies")
657
+ if (
658
+ any(target.cell_id in _TYPED_GATE_TARGETS for target in manifest.benchmarks)
659
+ and attestation.get("typed_gate_receipts") is not True
660
+ ):
661
+ blockers.append("external driver has not attested strict typed gate/stack receipts")
662
+ return tuple(blockers)
663
+
664
+
665
+ def disabled_campaign_catalog_payload() -> dict[str, Any]:
666
+ return {
667
+ "schema_version": 1,
668
+ "configured": False,
669
+ "catalog_sha256": None,
670
+ "default_action": "preflight_and_plan",
671
+ "browser_request_schema": {
672
+ "preflight": ["profile_id"],
673
+ "execution": [
674
+ "profile_id",
675
+ "plan_id",
676
+ "plan_hash",
677
+ "confirmation",
678
+ ],
679
+ },
680
+ "profiles": [],
681
+ "reason": (
682
+ "Start Studio with a server-side campaign profile catalog to enable "
683
+ "research-campaign preflight."
684
+ ),
685
+ }
686
+
687
+
688
+ def _parse_profile(
689
+ value: Any,
690
+ *,
691
+ base: Path,
692
+ path: str,
693
+ ) -> _CampaignProfile:
694
+ if not isinstance(value, Mapping):
695
+ raise ValidationError(f"{path} must be an object")
696
+ expected = {
697
+ "id",
698
+ "label",
699
+ "description",
700
+ "blueprint",
701
+ "arm_catalog",
702
+ "driver_entrypoint",
703
+ "execution_enabled",
704
+ }
705
+ if set(value) != expected:
706
+ raise ValidationError(f"{path} fields are invalid")
707
+ return _CampaignProfile(
708
+ profile_id=_text(value["id"], f"{path}.id"),
709
+ label=_text(value["label"], f"{path}.label"),
710
+ description=_text(value["description"], f"{path}.description"),
711
+ blueprint=_locked_file(
712
+ value["blueprint"],
713
+ base=base,
714
+ path=f"{path}.blueprint",
715
+ label="campaign blueprint",
716
+ ),
717
+ arm_catalog=_locked_file(
718
+ value["arm_catalog"],
719
+ base=base,
720
+ path=f"{path}.arm_catalog",
721
+ label="campaign arm catalog",
722
+ ),
723
+ driver_entrypoint=_text(
724
+ value["driver_entrypoint"],
725
+ f"{path}.driver_entrypoint",
726
+ ),
727
+ execution_enabled=value["execution_enabled"],
728
+ )
729
+
730
+
731
+ def _locked_file(
732
+ value: Any,
733
+ *,
734
+ base: Path,
735
+ path: str,
736
+ label: str,
737
+ ) -> _LockedCampaignFile:
738
+ if not isinstance(value, Mapping) or set(value) != {"path", "sha256"}:
739
+ raise ValidationError(f"{path} fields are invalid")
740
+ raw_path = _text(value["path"], f"{path}.path")
741
+ source = Path(raw_path).expanduser()
742
+ if not source.is_absolute():
743
+ source = base / source
744
+ return _LockedCampaignFile(
745
+ path=source.resolve(),
746
+ sha256=_text(value["sha256"], f"{path}.sha256"),
747
+ label=label,
748
+ )
749
+
750
+
751
+ def _profile_only_request(payload: Any) -> str:
752
+ if not isinstance(payload, Mapping) or set(payload) != {"profile_id"}:
753
+ raise CampaignSecurityError("campaign preflight accepts only one allowlisted profile_id")
754
+ profile_id = payload["profile_id"]
755
+ if not isinstance(profile_id, str) or _PROFILE_ID.fullmatch(profile_id) is None:
756
+ raise CampaignValidationError("campaign profile_id is invalid")
757
+ return profile_id
758
+
759
+
760
+ def _execution_request(payload: Any) -> dict[str, str]:
761
+ expected = {"profile_id", "plan_id", "plan_hash", "confirmation"}
762
+ if not isinstance(payload, Mapping) or set(payload) != expected:
763
+ raise CampaignSecurityError(
764
+ "campaign execution accepts only frozen profile and plan identities"
765
+ )
766
+ profile_id = payload["profile_id"]
767
+ plan_id = payload["plan_id"]
768
+ plan_hash = payload["plan_hash"]
769
+ confirmation = payload["confirmation"]
770
+ if not isinstance(profile_id, str) or _PROFILE_ID.fullmatch(profile_id) is None:
771
+ raise CampaignValidationError("campaign profile_id is invalid")
772
+ if not isinstance(plan_id, str) or re.fullmatch(r"[0-9a-f]{32}", plan_id) is None:
773
+ raise CampaignValidationError("campaign plan_id is invalid")
774
+ if not isinstance(plan_hash, str) or _SHA256.fullmatch(plan_hash) is None:
775
+ raise CampaignValidationError("campaign plan_hash is invalid")
776
+ if confirmation != CAMPAIGN_CONFIRMATION:
777
+ raise CampaignValidationError("campaign execution requires the exact confirmation phrase")
778
+ return {
779
+ "profile_id": profile_id,
780
+ "plan_id": plan_id,
781
+ "plan_hash": plan_hash,
782
+ "confirmation": confirmation,
783
+ }
784
+
785
+
786
+ def _text(value: Any, path: str) -> str:
787
+ if not isinstance(value, str) or not value.strip() or value != value.strip():
788
+ raise ValidationError(f"{path} must be trimmed non-empty text")
789
+ return value
790
+
791
+
792
+ def _sha256_file(path: Path) -> str:
793
+ digest = hashlib.sha256()
794
+ with path.open("rb") as handle:
795
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
796
+ digest.update(block)
797
+ return digest.hexdigest()
798
+
799
+
800
+ def _utc_now() -> str:
801
+ return (
802
+ datetime.now(timezone.utc)
803
+ .isoformat(timespec="milliseconds")
804
+ .replace(
805
+ "+00:00",
806
+ "Z",
807
+ )
808
+ )
809
+
810
+
811
+ __all__ = [
812
+ "CAMPAIGN_CONFIRMATION",
813
+ "CampaignManager",
814
+ "CampaignQueueFullError",
815
+ "CampaignSecurityError",
816
+ "CampaignValidationError",
817
+ "disabled_campaign_catalog_payload",
818
+ "load_campaign_profile_catalog",
819
+ ]