agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,418 @@
1
+ """ACE offline recipe delegating optimization to the pinned upstream worker."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Mapping, Sequence
7
+ from dataclasses import asdict, dataclass, field, fields
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from agentinputlab.artifacts import load_pinned_json_mapping
12
+ from agentinputlab.contracts import (
13
+ ExternalDependencyError,
14
+ InterventionContext,
15
+ ValidationError,
16
+ )
17
+ from agentinputlab.contracts.serialization import (
18
+ content_hash,
19
+ deep_freeze_jsonable,
20
+ freeze_mapping,
21
+ to_jsonable,
22
+ )
23
+ from agentinputlab.integrations.external_workers.ace import (
24
+ ACE_UPSTREAM_COMMIT,
25
+ ACEPlaybook,
26
+ ACEWorkerRequest,
27
+ ACEWorkerResponse,
28
+ PlaybookBullet,
29
+ )
30
+ from agentinputlab.integrations.external_workers.base import WorkerClient
31
+
32
+
33
+ @dataclass(frozen=True, slots=True)
34
+ class ACEExample:
35
+ example_id: str
36
+ input: Any
37
+ expected_output: Any = None
38
+ split: str = ""
39
+ metadata: Mapping[str, Any] = field(default_factory=dict)
40
+
41
+ def __post_init__(self) -> None:
42
+ if not self.example_id.strip():
43
+ raise ValidationError("ACE examples require a stable example_id")
44
+ if self.split not in {"train", "validation", "test"}:
45
+ raise ValidationError("ACE example split must be train/validation/test")
46
+ try:
47
+ frozen_input = deep_freeze_jsonable(self.input)
48
+ frozen_expected_output = deep_freeze_jsonable(self.expected_output)
49
+ frozen_metadata = freeze_mapping(self.metadata)
50
+ content_hash(frozen_input)
51
+ content_hash(frozen_expected_output)
52
+ except (TypeError, ValueError) as exc:
53
+ raise ValidationError("ACE example values must be canonically serializable") from exc
54
+ object.__setattr__(self, "input", frozen_input)
55
+ object.__setattr__(self, "expected_output", frozen_expected_output)
56
+ object.__setattr__(self, "metadata", frozen_metadata)
57
+
58
+ @property
59
+ def input_hash(self) -> str:
60
+ return content_hash(self.input)
61
+
62
+ @classmethod
63
+ def from_value(cls, value: Any, *, split: str) -> ACEExample:
64
+ if isinstance(value, cls):
65
+ if value.split != split:
66
+ raise ValidationError(
67
+ f"ACE example {value.example_id!r} is labeled {value.split}, expected {split}"
68
+ )
69
+ return value
70
+ if not isinstance(value, Mapping):
71
+ raise ValidationError("ACE examples must be ACEExample or mappings")
72
+ metadata = value.get("metadata", {})
73
+ if not isinstance(metadata, Mapping):
74
+ raise ValidationError("ACE example metadata must be a mapping")
75
+ try:
76
+ return cls(
77
+ example_id=str(value["example_id"]),
78
+ input=value["input"],
79
+ expected_output=value.get("expected_output"),
80
+ split=split,
81
+ metadata=dict(metadata),
82
+ )
83
+ except KeyError as exc:
84
+ raise ValidationError("ACE example lacks example_id or input") from exc
85
+
86
+ def as_dict(self) -> dict[str, Any]:
87
+ return to_jsonable(self)
88
+
89
+
90
+ @dataclass(frozen=True, slots=True)
91
+ class SplitAudit:
92
+ train_ids: tuple[str, ...]
93
+ validation_ids: tuple[str, ...]
94
+ test_ids: tuple[str, ...]
95
+ train_input_hashes: tuple[str, ...]
96
+ validation_input_hashes: tuple[str, ...]
97
+ test_input_hashes: tuple[str, ...]
98
+
99
+ @property
100
+ def audit_hash(self) -> str:
101
+ return content_hash(self)
102
+
103
+
104
+ def validate_split_isolation(
105
+ train: Sequence[ACEExample],
106
+ validation: Sequence[ACEExample],
107
+ test: Sequence[ACEExample] = (),
108
+ ) -> SplitAudit:
109
+ """Fail on ID or exact-input overlap between any ACE data split."""
110
+
111
+ groups = {
112
+ "train": tuple(train),
113
+ "validation": tuple(validation),
114
+ "test": tuple(test),
115
+ }
116
+ for split, examples in groups.items():
117
+ if any(example.split != split for example in examples):
118
+ raise ValidationError(f"ACE {split} collection contains a mislabeled example")
119
+ ids = [example.example_id for example in examples]
120
+ hashes = [example.input_hash for example in examples]
121
+ if len(ids) != len(set(ids)):
122
+ raise ValidationError(f"ACE {split} split contains duplicate IDs")
123
+ if len(hashes) != len(set(hashes)):
124
+ raise ValidationError(f"ACE {split} split contains duplicate inputs")
125
+ names = tuple(groups)
126
+ for left_index, left in enumerate(names):
127
+ for right in names[left_index + 1 :]:
128
+ left_ids = {item.example_id for item in groups[left]}
129
+ right_ids = {item.example_id for item in groups[right]}
130
+ if left_ids & right_ids:
131
+ raise ValidationError(f"ACE {left}/{right} example IDs overlap")
132
+ left_hashes = {item.input_hash for item in groups[left]}
133
+ right_hashes = {item.input_hash for item in groups[right]}
134
+ if left_hashes & right_hashes:
135
+ raise ValidationError(f"ACE {left}/{right} inputs overlap")
136
+ return SplitAudit(
137
+ train_ids=tuple(item.example_id for item in groups["train"]),
138
+ validation_ids=tuple(item.example_id for item in groups["validation"]),
139
+ test_ids=tuple(item.example_id for item in groups["test"]),
140
+ train_input_hashes=tuple(item.input_hash for item in groups["train"]),
141
+ validation_input_hashes=tuple(item.input_hash for item in groups["validation"]),
142
+ test_input_hashes=tuple(item.input_hash for item in groups["test"]),
143
+ )
144
+
145
+
146
+ @dataclass(frozen=True, slots=True)
147
+ class ACEControls:
148
+ max_rounds: int = 4
149
+ max_model_calls: int = 100
150
+ max_tokens: int = 200_000
151
+ max_cost_usd: float = 100.0
152
+ seed: int = 0
153
+
154
+ def __post_init__(self) -> None:
155
+ for name in ("max_rounds", "max_model_calls", "max_tokens"):
156
+ if type(getattr(self, name)) is not int or getattr(self, name) < 1:
157
+ raise ValidationError(f"ACE {name} must be a positive integer")
158
+ if type(self.seed) is not int or self.seed < 0:
159
+ raise ValidationError("ACE seed must be a non-negative integer")
160
+ if (
161
+ isinstance(self.max_cost_usd, bool)
162
+ or not isinstance(self.max_cost_usd, (int, float))
163
+ or (isinstance(self.max_cost_usd, float) and not math.isfinite(self.max_cost_usd))
164
+ or self.max_cost_usd < 0
165
+ ):
166
+ raise ValidationError("ACE max_cost_usd must be finite and non-negative")
167
+
168
+
169
+ @dataclass(frozen=True, slots=True)
170
+ class ACEContextArtifact:
171
+ artifact_id: str
172
+ release_id: str
173
+ playbook: ACEPlaybook
174
+ selected_iteration: int
175
+ split_audit_hash: str
176
+ upstream_commit: str
177
+ worker_fingerprint: Mapping[str, Any]
178
+ model_fingerprints: Mapping[str, Mapping[str, Any]]
179
+ controls: Mapping[str, Any]
180
+ usage: Mapping[str, int | float]
181
+ iteration_hashes: tuple[str, ...]
182
+
183
+ def __post_init__(self) -> None:
184
+ if self.release_id != "recipe.ace.offline":
185
+ raise ValidationError("ACE artifact release identity mismatch")
186
+ if self.upstream_commit != ACE_UPSTREAM_COMMIT:
187
+ raise ValidationError("ACE artifact upstream revision mismatch")
188
+ if not isinstance(self.playbook, ACEPlaybook):
189
+ raise ValidationError("ACE artifact playbook must be an ACEPlaybook")
190
+ if type(self.selected_iteration) is not int or self.selected_iteration < 0:
191
+ raise ValidationError("ACE selected_iteration must be a non-negative integer")
192
+ if any(
193
+ not isinstance(getattr(self, name), Mapping)
194
+ for name in (
195
+ "worker_fingerprint",
196
+ "model_fingerprints",
197
+ "controls",
198
+ "usage",
199
+ )
200
+ ) or any(not isinstance(value, Mapping) for value in self.model_fingerprints.values()):
201
+ raise ValidationError("ACE artifact fingerprints, controls, and usage must be mappings")
202
+ if not isinstance(self.iteration_hashes, (list, tuple)) or any(
203
+ not isinstance(value, str) for value in self.iteration_hashes
204
+ ):
205
+ raise ValidationError("ACE iteration_hashes must be an array of strings")
206
+ object.__setattr__(
207
+ self,
208
+ "playbook",
209
+ ACEPlaybook(
210
+ tuple(
211
+ PlaybookBullet(
212
+ b.bullet_id,
213
+ b.content,
214
+ b.helpful,
215
+ b.harmful,
216
+ metadata=freeze_mapping(b.metadata),
217
+ )
218
+ for b in self.playbook.bullets
219
+ )
220
+ ),
221
+ )
222
+ for name in ("worker_fingerprint", "model_fingerprints", "controls", "usage"):
223
+ object.__setattr__(self, name, freeze_mapping(getattr(self, name)))
224
+ object.__setattr__(self, "iteration_hashes", tuple(self.iteration_hashes))
225
+ if self.artifact_id != content_hash(self.identity_payload()):
226
+ raise ValidationError("ACE artifact content identity mismatch")
227
+
228
+ def identity_payload(self) -> dict[str, Any]:
229
+ """Original fit identity; a pinned file checksum separately protects usage."""
230
+ return {
231
+ "release_id": self.release_id,
232
+ "playbook_hash": self.playbook.playbook_hash,
233
+ "selected_iteration": self.selected_iteration,
234
+ "split_audit_hash": self.split_audit_hash,
235
+ "upstream_commit": self.upstream_commit,
236
+ "worker_fingerprint": self.worker_fingerprint,
237
+ "model_fingerprints": self.model_fingerprints,
238
+ "controls": self.controls,
239
+ "iteration_hashes": self.iteration_hashes,
240
+ }
241
+
242
+ @classmethod
243
+ def from_mapping(cls, value: Mapping[str, Any]) -> ACEContextArtifact:
244
+ if not isinstance(value, Mapping) or set(value) != {f.name for f in fields(cls)}:
245
+ raise ValidationError("ACE artifact fields are invalid")
246
+ book = value["playbook"]
247
+ if (
248
+ not isinstance(book, Mapping)
249
+ or set(book) != {"bullets"}
250
+ or not isinstance(book["bullets"], (list, tuple))
251
+ ):
252
+ raise ValidationError("ACE artifact playbook schema is invalid")
253
+ bullets = []
254
+ for item in book["bullets"]:
255
+ if (
256
+ not isinstance(item, Mapping)
257
+ or set(item) != {f.name for f in fields(PlaybookBullet)}
258
+ or any(
259
+ not isinstance(item.get(k), str) or not item[k].strip()
260
+ for k in ("bullet_id", "content")
261
+ )
262
+ or any(type(item.get(k)) is not int or item[k] < 0 for k in ("helpful", "harmful"))
263
+ or not isinstance(item.get("metadata"), Mapping)
264
+ ):
265
+ raise ValidationError("ACE artifact bullet schema is invalid")
266
+ bullets.append(PlaybookBullet(**dict(item)))
267
+ malformed = False
268
+ try:
269
+ return cls(**{**dict(value), "playbook": ACEPlaybook(tuple(bullets))})
270
+ except ValidationError:
271
+ raise
272
+ except (AttributeError, TypeError, ValueError, RecursionError):
273
+ malformed = True
274
+ if malformed:
275
+ raise ValidationError("ACE artifact values are malformed")
276
+
277
+ @classmethod
278
+ def from_json(
279
+ cls, path: str | Path, *, expected_sha256: str, max_bytes: int = 4 * 1024 * 1024
280
+ ) -> ACEContextArtifact:
281
+ return cls.from_mapping(
282
+ load_pinned_json_mapping(path, expected_sha256=expected_sha256, max_bytes=max_bytes)
283
+ )
284
+
285
+ def as_dict(self) -> dict[str, Any]:
286
+ return to_jsonable(self)
287
+
288
+
289
+ @dataclass(slots=True)
290
+ class ACEOfflineRecipe:
291
+ """Run train/validation-only ACE and return an eval-only playbook artifact."""
292
+
293
+ client: WorkerClient
294
+ initial_playbook: ACEPlaybook = field(default_factory=ACEPlaybook)
295
+ controls: ACEControls = field(default_factory=ACEControls)
296
+ method_id: str = "ace"
297
+ release_id: str = "recipe.ace.offline"
298
+
299
+ def __post_init__(self) -> None:
300
+ if not isinstance(self.client, WorkerClient):
301
+ raise ExternalDependencyError("ACE offline recipe requires a WorkerClient")
302
+
303
+ def fit(
304
+ self,
305
+ train_examples: tuple[Any, ...],
306
+ *,
307
+ validation_examples: tuple[Any, ...] = (),
308
+ context: InterventionContext,
309
+ ) -> ACEContextArtifact:
310
+ train = tuple(ACEExample.from_value(item, split="train") for item in train_examples)
311
+ validation = tuple(
312
+ ACEExample.from_value(item, split="validation") for item in validation_examples
313
+ )
314
+ if not train or not validation:
315
+ raise ValidationError("ACE offline recipe requires non-empty train and validation")
316
+ if self.controls.seed != context.seed:
317
+ raise ValidationError(
318
+ "ACE controls.seed must equal InterventionContext.seed; "
319
+ "the recipe does not silently override either seed"
320
+ )
321
+ audit = validate_split_isolation(train, validation)
322
+ fingerprints = _model_fingerprints(context)
323
+ controls = asdict(self.controls)
324
+ serialized_train = tuple(item.as_dict() for item in train)
325
+ serialized_validation = tuple(item.as_dict() for item in validation)
326
+ request_base = {
327
+ "train_ids": audit.train_ids,
328
+ "validation_ids": audit.validation_ids,
329
+ "train_examples": serialized_train,
330
+ "validation_examples": serialized_validation,
331
+ "initial_playbook_hash": self.initial_playbook.playbook_hash,
332
+ "controls": controls,
333
+ "model_fingerprints": fingerprints,
334
+ "split_audit_hash": audit.audit_hash,
335
+ "upstream_commit": ACE_UPSTREAM_COMMIT,
336
+ }
337
+ request = ACEWorkerRequest(
338
+ request_id=content_hash(request_base),
339
+ train_examples=serialized_train,
340
+ validation_examples=serialized_validation,
341
+ initial_playbook=self.initial_playbook,
342
+ controls=controls,
343
+ model_fingerprints=fingerprints,
344
+ split_audit_hash=audit.audit_hash,
345
+ )
346
+ raw = self.client.call(request.as_dict())
347
+ response = ACEWorkerResponse.from_mapping(raw, request=request)
348
+ iteration_hashes = tuple(
349
+ content_hash(
350
+ {
351
+ "index": iteration.index,
352
+ "operations": iteration.operations,
353
+ "candidate_playbook_hash": iteration.candidate_playbook_hash,
354
+ "validation_score": iteration.validation_score,
355
+ }
356
+ )
357
+ for iteration in response.iterations
358
+ )
359
+ artifact_base = {
360
+ "release_id": self.release_id,
361
+ "playbook_hash": response.final_playbook.playbook_hash,
362
+ "selected_iteration": response.selected_iteration,
363
+ "split_audit_hash": audit.audit_hash,
364
+ "upstream_commit": response.upstream_commit,
365
+ "worker_fingerprint": self.client.fingerprint(),
366
+ "model_fingerprints": fingerprints,
367
+ "controls": controls,
368
+ "iteration_hashes": iteration_hashes,
369
+ }
370
+ return ACEContextArtifact(
371
+ artifact_id=content_hash(artifact_base),
372
+ release_id=self.release_id,
373
+ playbook=response.final_playbook,
374
+ selected_iteration=response.selected_iteration,
375
+ split_audit_hash=audit.audit_hash,
376
+ upstream_commit=response.upstream_commit,
377
+ worker_fingerprint=dict(self.client.fingerprint()),
378
+ model_fingerprints=fingerprints,
379
+ controls=controls,
380
+ usage=response.usage,
381
+ iteration_hashes=iteration_hashes,
382
+ )
383
+
384
+
385
+ def _model_fingerprints(
386
+ context: InterventionContext,
387
+ ) -> Mapping[str, Mapping[str, Any]]:
388
+ configured = context.run_config.get("ace_model_fingerprints")
389
+ if configured is not None:
390
+ if not isinstance(configured, Mapping):
391
+ raise ValidationError("ace_model_fingerprints must be a mapping")
392
+ fingerprints: dict[str, Mapping[str, Any]] = {}
393
+ for role in ("target", "reflector", "curator"):
394
+ value = configured.get(role)
395
+ if not isinstance(value, Mapping):
396
+ raise ValidationError(f"ACE is missing the {role} model fingerprint")
397
+ fingerprints[role] = dict(value)
398
+ return fingerprints
399
+ if context.backend is None:
400
+ raise ExternalDependencyError(
401
+ "ACE requires explicit target/reflector/curator fingerprints or a backend"
402
+ )
403
+ try:
404
+ shared = dict(context.backend.fingerprint())
405
+ except AttributeError as exc:
406
+ raise ExternalDependencyError("ACE backend does not expose a fingerprint") from exc
407
+ return {role: dict(shared) for role in ("target", "reflector", "curator")}
408
+
409
+
410
+ def playbook_from_strings(values: Sequence[str]) -> ACEPlaybook:
411
+ """Convenience constructor with stable, position-derived bullet IDs."""
412
+
413
+ return ACEPlaybook(
414
+ tuple(
415
+ PlaybookBullet(bullet_id=f"bullet-{index:04d}", content=value)
416
+ for index, value in enumerate(values)
417
+ )
418
+ )
@@ -0,0 +1,273 @@
1
+ """BFCL Python-simple AST feedback for ACE's native text generator.
2
+
3
+ The explicit task adapter asks ACE for function-call JSON in ``final_answer``;
4
+ it does not replace ACE's generator with a function-calling agent. Evaluation
5
+ uses the caller's source-verified BFCL adapter, never text exact match. Training
6
+ targets are opaque IDs, not the benchmark's hidden reference arguments.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import math
13
+ from collections.abc import Mapping, Sequence
14
+
15
+ from agentinputlab.benchmarks.base import BenchmarkExample
16
+ from agentinputlab.contracts import ValidationError
17
+ from agentinputlab.contracts.serialization import canonical_json, content_hash, to_jsonable
18
+
19
+ from .ace import ACEExample
20
+
21
+
22
+ def _unique_object(pairs):
23
+ value = {}
24
+ for key, item in pairs:
25
+ if key in value:
26
+ raise ValueError("duplicate function-call JSON key")
27
+ value[key] = item
28
+ return value
29
+
30
+
31
+ def parse_bfcl_ace_answer(text: str) -> list[dict]:
32
+ """Decode exactly one FC-JSON sequence; no fences, repair or code execution."""
33
+ if not isinstance(text, str):
34
+ raise ValueError("ACE BFCL final_answer must be a JSON string")
35
+ value = json.loads(
36
+ text,
37
+ object_pairs_hook=_unique_object,
38
+ parse_constant=lambda x: (_ for _ in ()).throw(ValueError("nonfinite JSON")),
39
+ )
40
+ canonical_json(value) # Reject exponent overflow as well as explicit NaN/Infinity.
41
+ if not isinstance(value, list) or not value:
42
+ raise ValueError("ACE BFCL answer must be a nonempty function-call sequence")
43
+ for call in value:
44
+ if not isinstance(call, dict) or len(call) != 1:
45
+ raise ValueError("each BFCL call must contain one function name")
46
+ name, arguments = next(iter(call.items()))
47
+ if not isinstance(name, str) or not name.strip() or not isinstance(arguments, dict):
48
+ raise ValueError("BFCL function name and argument object required")
49
+ return value
50
+
51
+
52
+ class ACEBFCLFeedback:
53
+ """Bind disjoint caller-declared groups to the native Python AST checker.
54
+
55
+ This is a task-specific text-to-FC adapter, not a claim of native BFCL model
56
+ handler prompt parity or a standard leaderboard training split.
57
+ """
58
+
59
+ processor_id = "agentinputlab.ace.BFCL_AST_feedback.v1"
60
+
61
+ def __init__(
62
+ self,
63
+ *,
64
+ adapter,
65
+ train: Sequence[BenchmarkExample],
66
+ validation: Sequence[BenchmarkExample],
67
+ partition_label: str = "caller_declared_compatibility_partitions",
68
+ ):
69
+ self.train, self.validation = tuple(train), tuple(validation)
70
+ self._initialize(
71
+ adapter=adapter,
72
+ groups=(("train", self.train), ("validation", self.validation)),
73
+ partition_label=partition_label,
74
+ )
75
+
76
+ def _initialize(self, *, adapter, groups, partition_label):
77
+ if getattr(adapter, "benchmark_id", None) != "bfcl.v3.simple_python":
78
+ raise ValidationError("ACE BFCL requires the Python-simple adapter")
79
+ if (
80
+ not callable(getattr(adapter, "verify_integrity", None))
81
+ or not callable(getattr(adapter, "score", None))
82
+ or not callable(getattr(adapter, "function_tools", None))
83
+ ):
84
+ raise ValidationError("ACE BFCL requires integrity-checked native scoring")
85
+ if not isinstance(partition_label, str) or not partition_label.strip():
86
+ raise ValidationError("ACE BFCL partition label must be explicit")
87
+ self.adapter = adapter
88
+ self._groups = tuple(groups)
89
+ cases = tuple(e for _, group in self._groups for e in group)
90
+ if any(not group for _, group in self._groups) or any(
91
+ not isinstance(e, BenchmarkExample) for e in cases
92
+ ):
93
+ raise ValidationError("ACE BFCL requires nonempty typed fit groups")
94
+ if len({e.example_id for e in cases}) != len(cases):
95
+ raise ValidationError("ACE BFCL fit IDs must be disjoint")
96
+ for e in cases:
97
+ if not isinstance(e.visible_metadata.get("function"), Mapping):
98
+ raise ValidationError("ACE BFCL example lacks its visible function catalog")
99
+ if len({content_hash(self._visible(e)) for e in cases}) != len(cases):
100
+ raise ValidationError("ACE BFCL fit inputs must be disjoint")
101
+ self.partition_label = partition_label
102
+ self._examples = {e.example_id: e for e in cases}
103
+ self._binding = self._native_binding()
104
+ self.events = []
105
+
106
+ def _visible(self, example):
107
+ state = example.visible_state()
108
+ return dict(
109
+ query=state.query,
110
+ instruction=state.instruction,
111
+ messages=state.messages,
112
+ context=state.context,
113
+ tool_names=state.tool_names,
114
+ function=example.visible_metadata.get("function"),
115
+ native_function_tools=self.adapter.function_tools(example),
116
+ )
117
+
118
+ def _native_binding(self):
119
+ self.adapter.verify_integrity()
120
+ return to_jsonable(
121
+ dict(
122
+ prepared_files=dict(self.adapter.prepared.file_sha256),
123
+ evaluation_provenance=self.adapter.evaluation_provenance,
124
+ )
125
+ )
126
+
127
+ def fit_examples(self, split):
128
+ if split not in {"train", "validation"}:
129
+ raise ValidationError("ACE BFCL exposes only train and validation groups")
130
+ cases = self.train if split == "train" else self.validation
131
+ return self._examples_for(cases, split)
132
+
133
+ def _examples_for(self, cases, split):
134
+ return tuple(
135
+ ACEExample(
136
+ example_id=e.example_id,
137
+ input=dict(
138
+ question=(
139
+ "Solve the supplied function-calling task. Preserve its task "
140
+ "instruction, user query and function schema. Put exactly a JSON "
141
+ "array of function calls in your final_answer string, using "
142
+ '[{"function_name": {"argument_name": "value"}}]. '
143
+ "Use the names in native_function_tools and their argument types, not this "
144
+ "illustrative name. Do not include code fences or commentary.\n"
145
+ + canonical_json(self._visible(e))
146
+ )
147
+ ),
148
+ expected_output=e.example_id,
149
+ split=split,
150
+ metadata=dict(
151
+ task_processor=self.processor_id,
152
+ source_state_hash=e.visible_state().state_hash,
153
+ partition_label=self.partition_label,
154
+ evaluator_target_is_opaque_id=True,
155
+ ),
156
+ )
157
+ for e in cases
158
+ )
159
+
160
+ def validate_examples(self, train, validation):
161
+ for supplied, split in ((train, "train"), (validation, "validation")):
162
+ if tuple(to_jsonable(e) for e in supplied) != tuple(
163
+ e.as_dict() for e in self.fit_examples(split)
164
+ ):
165
+ raise ValidationError("ACE BFCL examples differ from frozen fit groups")
166
+
167
+ def fingerprint(self):
168
+ return dict(
169
+ processor_id=self.processor_id,
170
+ native_binding=self._binding,
171
+ metric="ast_accuracy",
172
+ partition_label=self.partition_label,
173
+ generator_output_adapter="native_ACE_text_final_answer_to_BFCL_FC_JSON",
174
+ native_BFCL_handler_prompt_parity_claimed=False,
175
+ training_success_criterion="native_ast_accuracy_strictly_greater_than_zero",
176
+ validation_selection="mean_native_ast_accuracy",
177
+ inputs={
178
+ split: [
179
+ dict(
180
+ case_id=e.example_id,
181
+ visible_hash=content_hash(self._visible(e)),
182
+ evaluator_gold_sha256=content_hash(e.evaluator_gold),
183
+ )
184
+ for e in cases
185
+ ]
186
+ for split, cases in self._groups
187
+ },
188
+ automatic_retry=False,
189
+ )
190
+
191
+ def score_answer(self, predicted, target):
192
+ if target not in self._examples:
193
+ raise ValidationError("ACE BFCL target is outside frozen fit groups")
194
+ if self._native_binding() != self._binding:
195
+ raise ValidationError("ACE BFCL source/scorer fingerprint changed")
196
+ try:
197
+ calls = parse_bfcl_ace_answer(predicted)
198
+ except (ValueError, TypeError):
199
+ metrics, status = {"ast_accuracy": 0.0}, "invalid_FC_JSON_training_feedback"
200
+ else:
201
+ # The pinned OpenAI FC decoder calls json.loads on each argument
202
+ # string. Preserve argument types while crossing that exact seam.
203
+ native_calls = [
204
+ {name: canonical_json(arguments)}
205
+ for call in calls
206
+ for name, arguments in call.items()
207
+ ]
208
+ metrics = dict(self.adapter.score(self._examples[target], native_calls))
209
+ status = "native_AST_scored"
210
+ value = metrics.get("ast_accuracy")
211
+ if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1:
212
+ raise ValidationError("ACE BFCL native ast_accuracy is absent or invalid")
213
+ self.events.append(
214
+ dict(
215
+ target=target,
216
+ prediction_sha256=content_hash(predicted),
217
+ metrics=metrics,
218
+ status=status,
219
+ )
220
+ )
221
+ return float(value)
222
+
223
+ def answer_is_correct(self, predicted, ground_truth):
224
+ return self.score_answer(predicted, ground_truth) > 0
225
+
226
+ def evaluate_accuracy(self, predictions, ground_truths):
227
+ if len(predictions) != len(ground_truths):
228
+ raise ValidationError("ACE BFCL predictions and targets differ in length")
229
+ return (
230
+ sum(self.score_answer(p, t) for p, t in zip(predictions, ground_truths, strict=True))
231
+ / len(predictions)
232
+ if predictions
233
+ else 0.0
234
+ )
235
+
236
+
237
+ class ACEOnlineBFCLFeedback(ACEBFCLFeedback):
238
+ """AST feedback for an explicitly transductive native ACE online stream.
239
+
240
+ Stream order is frozen before generation. This object is not an offline
241
+ training split and its adapted artifact is not independent held-out evidence.
242
+ """
243
+
244
+ processor_id = "agentinputlab.ace.BFCL_AST_online_feedback.v1"
245
+
246
+ def __init__(self, *, adapter, stream: Sequence[BenchmarkExample]):
247
+ self.stream = tuple(stream)
248
+ self._initialize(
249
+ adapter=adapter,
250
+ groups=(("stream", self.stream),),
251
+ partition_label="explicit_native_BFCL_feedback_transductive_online_stream",
252
+ )
253
+
254
+ def stream_examples(self):
255
+ return self._examples_for(self.stream, "test")
256
+
257
+ def validate_online_examples(self, examples):
258
+ supplied = tuple(ACEExample.from_value(row, split="test").as_dict() for row in examples)
259
+ if supplied != tuple(row.as_dict() for row in self.stream_examples()):
260
+ raise ValidationError("ACE online BFCL inputs/order differ from frozen stream")
261
+
262
+ def fingerprint(self):
263
+ return dict(
264
+ super().fingerprint(),
265
+ transductive_feedback_not_offline_or_heldout=True,
266
+ stream_order_preserved=True,
267
+ )
268
+
269
+ def fit_examples(self, split):
270
+ raise ValidationError("online BFCL feedback cannot be used as offline fit groups")
271
+
272
+ def validate_examples(self, train, validation):
273
+ raise ValidationError("online BFCL feedback cannot validate offline fit groups")