agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1 @@
1
+ """Optional integrations outside the default runtime-core installation."""
@@ -0,0 +1 @@
1
+ """Post-v1 tool/schema and executable action-space intervention adapters."""
@@ -0,0 +1,76 @@
1
+ """Typed bridges to pinned upstream runtimes in isolated environments."""
2
+
3
+ from .ace import (
4
+ ACE_UPSTREAM_COMMIT,
5
+ ACEDiffOperation,
6
+ ACEPlaybook,
7
+ ACEWorkerRequest,
8
+ ACEWorkerResponse,
9
+ PlaybookBullet,
10
+ apply_playbook_diff,
11
+ )
12
+ from .base import JsonLineSubprocessWorker, WorkerClient
13
+ from .decomp_live import DecompLiveRuntime
14
+ from .decomp_planner import DecompPlannerLiveRuntime
15
+ from .ircot import (
16
+ IRCOT_UPSTREAM_COMMIT,
17
+ IRCoTControlledWorker,
18
+ IRCoTControls,
19
+ IRCoTWorkerRequest,
20
+ IRCoTWorkerResponse,
21
+ )
22
+ from .llmcompiler_live import LLMCompilerLiveWorker
23
+ from .method import (
24
+ MethodWorkerRequest,
25
+ MethodWorkerResponse,
26
+ call_worker,
27
+ dependency_fingerprint,
28
+ resolve_worker,
29
+ )
30
+ from .process import ProcessResourceObservation
31
+ from .supervisor import (
32
+ AcceleratorSampler,
33
+ DefaultProcessRunner,
34
+ DockerWorkerSupervisor,
35
+ LocalSubprocessSupervisor,
36
+ ProcessRunner,
37
+ SupervisedProcessResult,
38
+ SupervisedWorkerError,
39
+ WorkerSupervisor,
40
+ worker_supervisor_from_environment,
41
+ )
42
+
43
+ __all__ = [
44
+ "ACE_UPSTREAM_COMMIT",
45
+ "IRCOT_UPSTREAM_COMMIT",
46
+ "ACEDiffOperation",
47
+ "ACEPlaybook",
48
+ "ACEWorkerRequest",
49
+ "ACEWorkerResponse",
50
+ "AcceleratorSampler",
51
+ "DecompLiveRuntime",
52
+ "DecompPlannerLiveRuntime",
53
+ "DefaultProcessRunner",
54
+ "DockerWorkerSupervisor",
55
+ "IRCoTControlledWorker",
56
+ "IRCoTControls",
57
+ "IRCoTWorkerRequest",
58
+ "IRCoTWorkerResponse",
59
+ "JsonLineSubprocessWorker",
60
+ "LLMCompilerLiveWorker",
61
+ "LocalSubprocessSupervisor",
62
+ "MethodWorkerRequest",
63
+ "MethodWorkerResponse",
64
+ "PlaybookBullet",
65
+ "ProcessResourceObservation",
66
+ "ProcessRunner",
67
+ "SupervisedProcessResult",
68
+ "SupervisedWorkerError",
69
+ "WorkerClient",
70
+ "WorkerSupervisor",
71
+ "apply_playbook_diff",
72
+ "call_worker",
73
+ "dependency_fingerprint",
74
+ "resolve_worker",
75
+ "worker_supervisor_from_environment",
76
+ ]
@@ -0,0 +1,431 @@
1
+ """Schema and validators for the pinned ACE offline worker."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping, Sequence
6
+ from dataclasses import dataclass, field
7
+ from math import isfinite
8
+ from types import MappingProxyType
9
+ from typing import Any
10
+
11
+ from agentinputlab.contracts.errors import ExternalDependencyError, ValidationError
12
+ from agentinputlab.contracts.serialization import content_hash, to_jsonable
13
+
14
+ ACE_UPSTREAM_COMMIT = "bcb7cea0504afad6f55fec4845dd4864c9f9eee7"
15
+ _OPERATIONS = {"add", "edit", "delete", "merge"}
16
+
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class PlaybookBullet:
20
+ bullet_id: str
21
+ content: str
22
+ helpful: int = 0
23
+ harmful: int = 0
24
+ metadata: Mapping[str, Any] = field(default_factory=dict)
25
+
26
+ def __post_init__(self) -> None:
27
+ if not self.bullet_id.strip() or not self.content.strip():
28
+ raise ValidationError("ACE playbook bullets require ID and content")
29
+ if self.helpful < 0 or self.harmful < 0:
30
+ raise ValidationError("ACE helpful/harmful counters must be non-negative")
31
+ object.__setattr__(self, "metadata", _frozen_mapping(self.metadata))
32
+
33
+ @classmethod
34
+ def from_mapping(cls, value: Mapping[str, Any]) -> PlaybookBullet:
35
+ metadata = value.get("metadata", {})
36
+ if not isinstance(metadata, Mapping):
37
+ raise ValidationError("ACE bullet metadata must be a mapping")
38
+ try:
39
+ return cls(
40
+ bullet_id=str(value["bullet_id"]),
41
+ content=str(value["content"]),
42
+ helpful=int(value.get("helpful", 0)),
43
+ harmful=int(value.get("harmful", 0)),
44
+ metadata=dict(metadata),
45
+ )
46
+ except KeyError as exc:
47
+ raise ValidationError("ACE bullet lacks a required field") from exc
48
+
49
+
50
+ @dataclass(frozen=True, slots=True)
51
+ class ACEPlaybook:
52
+ bullets: tuple[PlaybookBullet, ...] = ()
53
+
54
+ def __post_init__(self) -> None:
55
+ object.__setattr__(self, "bullets", tuple(self.bullets))
56
+ ids = [bullet.bullet_id for bullet in self.bullets]
57
+ if len(ids) != len(set(ids)):
58
+ raise ValidationError("ACE playbook bullet IDs must be unique")
59
+
60
+ @property
61
+ def playbook_hash(self) -> str:
62
+ return content_hash(self)
63
+
64
+ @classmethod
65
+ def from_sequence(cls, value: Sequence[Mapping[str, Any]]) -> ACEPlaybook:
66
+ return cls(tuple(PlaybookBullet.from_mapping(item) for item in value))
67
+
68
+
69
+ @dataclass(frozen=True, slots=True)
70
+ class ACEDiffOperation:
71
+ operation: str
72
+ bullet_id: str
73
+ content: str = ""
74
+ source_ids: tuple[str, ...] = ()
75
+ helpful: int | None = None
76
+ harmful: int | None = None
77
+ metadata: Mapping[str, Any] = field(default_factory=dict)
78
+
79
+ def __post_init__(self) -> None:
80
+ if self.operation not in _OPERATIONS:
81
+ raise ValidationError(f"unsupported ACE curator operation: {self.operation}")
82
+ if not self.bullet_id.strip():
83
+ raise ValidationError("ACE curator operation requires bullet_id")
84
+ object.__setattr__(self, "source_ids", tuple(self.source_ids))
85
+ if len(self.source_ids) != len(set(self.source_ids)):
86
+ raise ValidationError("ACE merge source_ids must be unique")
87
+ for counter in (self.helpful, self.harmful):
88
+ if counter is not None and counter < 0:
89
+ raise ValidationError("ACE helpful/harmful counters must be non-negative")
90
+ object.__setattr__(self, "metadata", _frozen_mapping(self.metadata))
91
+
92
+ @classmethod
93
+ def from_mapping(cls, value: Mapping[str, Any]) -> ACEDiffOperation:
94
+ sources = value.get("source_ids", ())
95
+ if not isinstance(sources, Sequence) or isinstance(sources, (str, bytes)):
96
+ raise ValidationError("ACE source_ids must be a sequence")
97
+ metadata = value.get("metadata", {})
98
+ if not isinstance(metadata, Mapping):
99
+ raise ValidationError("ACE diff metadata must be a mapping")
100
+ try:
101
+ return cls(
102
+ operation=str(value["operation"]),
103
+ bullet_id=str(value["bullet_id"]),
104
+ content=str(value.get("content", "")),
105
+ source_ids=tuple(str(item) for item in sources),
106
+ helpful=_optional_counter(value, "helpful"),
107
+ harmful=_optional_counter(value, "harmful"),
108
+ metadata=dict(metadata),
109
+ )
110
+ except KeyError as exc:
111
+ raise ValidationError("ACE diff lacks a required field") from exc
112
+ except (TypeError, ValueError) as exc:
113
+ raise ValidationError("ACE diff counters are malformed") from exc
114
+
115
+
116
+ def apply_playbook_diff(
117
+ playbook: ACEPlaybook,
118
+ operations: Sequence[ACEDiffOperation],
119
+ ) -> ACEPlaybook:
120
+ """Replay curator operations exactly and fail on ambiguous diffs."""
121
+
122
+ bullets = {bullet.bullet_id: bullet for bullet in playbook.bullets}
123
+ order = [bullet.bullet_id for bullet in playbook.bullets]
124
+ for operation in operations:
125
+ if operation.operation == "add":
126
+ if operation.bullet_id in bullets:
127
+ raise ValidationError("ACE add targets an existing bullet")
128
+ bullet = _bullet_from_operation(operation)
129
+ bullets[bullet.bullet_id] = bullet
130
+ order.append(bullet.bullet_id)
131
+ elif operation.operation == "edit":
132
+ if operation.bullet_id not in bullets:
133
+ raise ValidationError("ACE edit targets a missing bullet")
134
+ old = bullets[operation.bullet_id]
135
+ if not operation.content.strip():
136
+ raise ValidationError("ACE edit requires non-empty content")
137
+ bullets[operation.bullet_id] = PlaybookBullet(
138
+ bullet_id=operation.bullet_id,
139
+ content=operation.content,
140
+ helpful=(old.helpful if operation.helpful is None else operation.helpful),
141
+ harmful=(old.harmful if operation.harmful is None else operation.harmful),
142
+ metadata={**dict(old.metadata), **dict(operation.metadata)},
143
+ )
144
+ elif operation.operation == "delete":
145
+ if operation.bullet_id not in bullets:
146
+ raise ValidationError("ACE delete targets a missing bullet")
147
+ del bullets[operation.bullet_id]
148
+ order.remove(operation.bullet_id)
149
+ elif operation.operation == "merge":
150
+ if len(operation.source_ids) < 2:
151
+ raise ValidationError("ACE merge requires at least two source bullets")
152
+ if not set(operation.source_ids) <= set(bullets):
153
+ raise ValidationError("ACE merge references a missing source bullet")
154
+ if operation.bullet_id in bullets and operation.bullet_id not in operation.source_ids:
155
+ raise ValidationError("ACE merge target conflicts with an unrelated bullet")
156
+ if not operation.content.strip():
157
+ raise ValidationError("ACE merge requires non-empty content")
158
+ insertion = min(order.index(source) for source in operation.source_ids)
159
+ helpful = (
160
+ sum(bullets[source].helpful for source in operation.source_ids)
161
+ if operation.helpful is None
162
+ else operation.helpful
163
+ )
164
+ harmful = (
165
+ sum(bullets[source].harmful for source in operation.source_ids)
166
+ if operation.harmful is None
167
+ else operation.harmful
168
+ )
169
+ for source in operation.source_ids:
170
+ del bullets[source]
171
+ order.remove(source)
172
+ bullet = PlaybookBullet(
173
+ bullet_id=operation.bullet_id,
174
+ content=operation.content,
175
+ helpful=helpful,
176
+ harmful=harmful,
177
+ metadata=dict(operation.metadata),
178
+ )
179
+ bullets[bullet.bullet_id] = bullet
180
+ order.insert(insertion, bullet.bullet_id)
181
+ return ACEPlaybook(tuple(bullets[bullet_id] for bullet_id in order))
182
+
183
+
184
+ def _bullet_from_operation(operation: ACEDiffOperation) -> PlaybookBullet:
185
+ return PlaybookBullet(
186
+ bullet_id=operation.bullet_id,
187
+ content=operation.content,
188
+ helpful=0 if operation.helpful is None else operation.helpful,
189
+ harmful=0 if operation.harmful is None else operation.harmful,
190
+ metadata=dict(operation.metadata),
191
+ )
192
+
193
+
194
+ @dataclass(frozen=True, slots=True)
195
+ class ACEWorkerIteration:
196
+ index: int
197
+ operations: tuple[ACEDiffOperation, ...]
198
+ candidate_playbook_hash: str
199
+ validation_score: float
200
+
201
+ @classmethod
202
+ def from_mapping(cls, value: Mapping[str, Any]) -> ACEWorkerIteration:
203
+ raw_operations = value.get("operations", ())
204
+ if not isinstance(raw_operations, Sequence) or isinstance(raw_operations, (str, bytes)):
205
+ raise ValidationError("ACE iteration operations must be a sequence")
206
+ try:
207
+ return cls(
208
+ index=int(value["index"]),
209
+ operations=tuple(
210
+ ACEDiffOperation.from_mapping(_mapping(item, "ACE operation"))
211
+ for item in raw_operations
212
+ ),
213
+ candidate_playbook_hash=str(value["candidate_playbook_hash"]),
214
+ validation_score=_finite_float(
215
+ value["validation_score"],
216
+ "ACE validation_score",
217
+ ),
218
+ )
219
+ except (KeyError, TypeError, ValueError) as exc:
220
+ raise ValidationError("ACE iteration fields are malformed") from exc
221
+
222
+
223
+ @dataclass(frozen=True, slots=True)
224
+ class ACEWorkerRequest:
225
+ request_id: str
226
+ train_examples: tuple[Mapping[str, Any], ...]
227
+ validation_examples: tuple[Mapping[str, Any], ...]
228
+ initial_playbook: ACEPlaybook
229
+ controls: Mapping[str, Any]
230
+ model_fingerprints: Mapping[str, Mapping[str, Any]]
231
+ split_audit_hash: str
232
+ upstream_commit: str = ACE_UPSTREAM_COMMIT
233
+ protocol_version: str = "agentinputlab.ace.offline.v1"
234
+
235
+ def __post_init__(self) -> None:
236
+ if not self.request_id or not self.split_audit_hash:
237
+ raise ValidationError("ACE worker request requires request and split-audit IDs")
238
+ if not self.train_examples or not self.validation_examples:
239
+ raise ValidationError("ACE offline worker requires train and validation examples")
240
+ if set(self.model_fingerprints) != {"target", "reflector", "curator"}:
241
+ raise ValidationError("ACE model_fingerprints must bind target, reflector, and curator")
242
+ if self.upstream_commit != ACE_UPSTREAM_COMMIT:
243
+ raise ValidationError("ACE request does not target the pinned upstream commit")
244
+ object.__setattr__(self, "controls", _frozen_mapping(self.controls))
245
+ object.__setattr__(
246
+ self,
247
+ "model_fingerprints",
248
+ MappingProxyType(
249
+ {
250
+ str(role): _frozen_mapping(fingerprint)
251
+ for role, fingerprint in self.model_fingerprints.items()
252
+ }
253
+ ),
254
+ )
255
+
256
+ def as_dict(self) -> dict[str, Any]:
257
+ return to_jsonable(self)
258
+
259
+
260
+ @dataclass(frozen=True, slots=True)
261
+ class ACEWorkerResponse:
262
+ request_id: str
263
+ final_playbook: ACEPlaybook
264
+ iterations: tuple[ACEWorkerIteration, ...]
265
+ selected_iteration: int
266
+ usage: Mapping[str, int | float]
267
+ split_audit_hash: str
268
+ train_ids: tuple[str, ...]
269
+ validation_ids: tuple[str, ...]
270
+ upstream_commit: str
271
+ protocol_version: str
272
+
273
+ @classmethod
274
+ def from_mapping(
275
+ cls,
276
+ value: Mapping[str, Any],
277
+ *,
278
+ request: ACEWorkerRequest,
279
+ ) -> ACEWorkerResponse:
280
+ if str(value.get("status", "")) != "ok":
281
+ code = str(value.get("failure_code", "WORKER_FAILURE"))
282
+ raise ExternalDependencyError(f"ACE worker failed with {code}")
283
+ if str(value.get("request_id", "")) != request.request_id:
284
+ raise ValidationError("ACE response request_id mismatch")
285
+ if str(value.get("upstream_commit", "")) != request.upstream_commit:
286
+ raise ValidationError("ACE worker upstream revision mismatch")
287
+ if str(value.get("protocol_version", "")) != request.protocol_version:
288
+ raise ValidationError("ACE worker protocol version mismatch")
289
+ if str(value.get("split_audit_hash", "")) != request.split_audit_hash:
290
+ raise ValidationError("ACE worker split-audit hash mismatch")
291
+ train_ids = _response_ids(value, "train_ids")
292
+ validation_ids = _response_ids(value, "validation_ids")
293
+ expected_train_ids = _request_example_ids(request.train_examples, "train")
294
+ expected_validation_ids = _request_example_ids(
295
+ request.validation_examples,
296
+ "validation",
297
+ )
298
+ if train_ids != expected_train_ids:
299
+ raise ValidationError("ACE worker train IDs differ from the audited split")
300
+ if validation_ids != expected_validation_ids:
301
+ raise ValidationError("ACE worker validation IDs differ from the audited split")
302
+ raw_iterations = value.get("iterations", ())
303
+ if not isinstance(raw_iterations, Sequence) or isinstance(raw_iterations, (str, bytes)):
304
+ raise ValidationError("ACE response iterations must be a sequence")
305
+ iterations = tuple(
306
+ ACEWorkerIteration.from_mapping(_mapping(item, "ACE iteration"))
307
+ for item in raw_iterations
308
+ )
309
+ if not iterations:
310
+ raise ValidationError("ACE worker returned no optimization iterations")
311
+ if len(iterations) > int(request.controls.get("max_rounds", len(iterations))):
312
+ raise ValidationError("ACE worker exceeded max_rounds")
313
+ snapshots: dict[int, ACEPlaybook] = {}
314
+ current = request.initial_playbook
315
+ for expected, iteration in enumerate(iterations):
316
+ if iteration.index != expected:
317
+ raise ValidationError("ACE iteration indices must be contiguous and zero-based")
318
+ current = apply_playbook_diff(current, iteration.operations)
319
+ if current.playbook_hash != iteration.candidate_playbook_hash:
320
+ raise ValidationError("ACE candidate playbook hash does not match its diff")
321
+ snapshots[iteration.index] = current
322
+ try:
323
+ selected = int(value["selected_iteration"])
324
+ except (KeyError, TypeError, ValueError) as exc:
325
+ raise ValidationError("ACE selected_iteration is malformed") from exc
326
+ if selected not in snapshots:
327
+ raise ValidationError("ACE selected_iteration does not exist")
328
+ best_score = max(iteration.validation_score for iteration in iterations)
329
+ selected_score = iterations[selected].validation_score
330
+ if selected_score != best_score:
331
+ raise ValidationError("ACE selected iteration is not validation-best")
332
+ raw_playbook = value.get("final_playbook")
333
+ if not isinstance(raw_playbook, Sequence) or isinstance(raw_playbook, (str, bytes)):
334
+ raise ValidationError("ACE final_playbook must be a sequence")
335
+ final_playbook = ACEPlaybook.from_sequence(
336
+ tuple(_mapping(item, "ACE final bullet") for item in raw_playbook)
337
+ )
338
+ if final_playbook.playbook_hash != snapshots[selected].playbook_hash:
339
+ raise ValidationError("ACE final playbook differs from the selected snapshot")
340
+ usage = value.get("usage", {})
341
+ if not isinstance(usage, Mapping):
342
+ raise ValidationError("ACE usage must be a mapping")
343
+ numeric_usage: dict[str, int | float] = {}
344
+ for key, amount in usage.items():
345
+ if (
346
+ not isinstance(amount, (int, float))
347
+ or isinstance(amount, bool)
348
+ or not isfinite(float(amount))
349
+ or amount < 0
350
+ ):
351
+ raise ValidationError("ACE usage values must be non-negative numbers")
352
+ numeric_usage[str(key)] = amount
353
+ _validate_usage_limits(numeric_usage, request.controls)
354
+ return cls(
355
+ request_id=request.request_id,
356
+ final_playbook=final_playbook,
357
+ iterations=iterations,
358
+ selected_iteration=selected,
359
+ usage=numeric_usage,
360
+ split_audit_hash=request.split_audit_hash,
361
+ train_ids=train_ids,
362
+ validation_ids=validation_ids,
363
+ upstream_commit=request.upstream_commit,
364
+ protocol_version=request.protocol_version,
365
+ )
366
+
367
+
368
+ def _validate_usage_limits(
369
+ usage: Mapping[str, int | float],
370
+ controls: Mapping[str, Any],
371
+ ) -> None:
372
+ pairs = (
373
+ ("model_calls", "max_model_calls"),
374
+ ("tokens", "max_tokens"),
375
+ ("cost_usd", "max_cost_usd"),
376
+ )
377
+ for usage_key, control_key in pairs:
378
+ if (
379
+ usage_key in usage
380
+ and control_key in controls
381
+ and float(usage[usage_key]) > float(controls[control_key])
382
+ ):
383
+ raise ValidationError(f"ACE worker exceeded {control_key}")
384
+
385
+
386
+ def _mapping(value: Any, label: str) -> Mapping[str, Any]:
387
+ if not isinstance(value, Mapping):
388
+ raise ValidationError(f"{label} must be a mapping")
389
+ return value
390
+
391
+
392
+ def _frozen_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]:
393
+ return MappingProxyType(dict(value))
394
+
395
+
396
+ def _optional_counter(value: Mapping[str, Any], key: str) -> int | None:
397
+ raw = value.get(key)
398
+ return None if raw is None else int(raw)
399
+
400
+
401
+ def _finite_float(value: Any, label: str) -> float:
402
+ try:
403
+ result = float(value)
404
+ except (TypeError, ValueError) as exc:
405
+ raise ValidationError(f"{label} must be numeric") from exc
406
+ if not isfinite(result):
407
+ raise ValidationError(f"{label} must be finite")
408
+ return result
409
+
410
+
411
+ def _response_ids(value: Mapping[str, Any], key: str) -> tuple[str, ...]:
412
+ raw = value.get(key)
413
+ if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)):
414
+ raise ValidationError(f"ACE response {key} must be a sequence")
415
+ ids = tuple(str(item) for item in raw)
416
+ if any(not item for item in ids) or len(ids) != len(set(ids)):
417
+ raise ValidationError(f"ACE response {key} contains empty or duplicate IDs")
418
+ return ids
419
+
420
+
421
+ def _request_example_ids(
422
+ examples: Sequence[Mapping[str, Any]],
423
+ split: str,
424
+ ) -> tuple[str, ...]:
425
+ try:
426
+ ids = tuple(str(item["example_id"]) for item in examples)
427
+ except KeyError as exc:
428
+ raise ValidationError(f"ACE {split} request examples lack example_id") from exc
429
+ if any(not item for item in ids) or len(ids) != len(set(ids)):
430
+ raise ValidationError(f"ACE {split} request IDs are empty or duplicated")
431
+ return ids