agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,382 @@
1
+ """Portable STATE-Bench tool-plan feedback for native ACE fitting/online APIs.
2
+
3
+ This adapter scores a noninteractive plan on original training tasks. It does
4
+ not replace the benchmark's closed-loop heldout agent or its native judge.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Mapping, Sequence
10
+ from pathlib import Path
11
+ from urllib.parse import urlsplit
12
+
13
+ from agentinputlab.api import _state
14
+ from agentinputlab.contracts import ValidationError
15
+ from agentinputlab.contracts.serialization import canonical_json, to_jsonable
16
+
17
+ from .ace import ACEExample
18
+ from .ace_native_plan import (
19
+ PLAN_INSTRUCTION,
20
+ NativePlanFeedback,
21
+ _save,
22
+ _sha,
23
+ native_process_scope,
24
+ source_file_pins,
25
+ verify_native_checkout,
26
+ )
27
+
28
+ SCHEMA = "agentinputlab.ace.STATE.native_feedback_input.v1"
29
+
30
+
31
+ def _provider():
32
+ from workers.official_arenas.providers.statebench import StateBenchProvider
33
+
34
+ return StateBenchProvider()
35
+
36
+
37
+ def _native_inputs(checkout, configuration, rows):
38
+ from state_bench.domain import get_domain_config
39
+ from state_bench.paths import domain_tasks_dir
40
+ from state_bench.protocol import load_default_protocol, load_split_task_ids
41
+ from state_bench.schemas import TaskDefinition
42
+
43
+ protocol = load_default_protocol()
44
+ if protocol.validate_prompt_hashes():
45
+ raise ValidationError("STATE protocol prompt checksums differ")
46
+ domains, tasks, files = (
47
+ {},
48
+ {},
49
+ [
50
+ checkout / "state_bench" / n
51
+ for n in (
52
+ "client.py",
53
+ "domain.py",
54
+ "env_loader.py",
55
+ "orchestrator.py",
56
+ "protocol.py",
57
+ "scoring.py",
58
+ )
59
+ ],
60
+ )
61
+ for row in rows:
62
+ protected = row["protected_checkpoint"]["protected"]
63
+ name, task_id = protected["domain"], protected["task_id"]
64
+ if name not in configuration["domains"] or task_id not in load_split_task_ids(
65
+ name, "train", protocol.split_version
66
+ ):
67
+ raise ValidationError("STATE feedback must use original official training tasks")
68
+ path = domain_tasks_dir(name) / (task_id + ".json")
69
+ task = TaskDefinition.load(path)
70
+ domain = get_domain_config(name)
71
+ state = _state(row["state"])
72
+ if (
73
+ row["case_id"] != name + ":" + task_id
74
+ or state.task_id != task_id
75
+ or state.query != task.opening_message
76
+ or protected["user_id"] != task.user_id
77
+ or protected["protocol_id"] != protocol.protocol_id
78
+ or protected["locked_dependency_hashes"] != configuration["locked_dependency_hashes"]
79
+ ):
80
+ raise ValidationError("STATE fitting checkpoint, task or dependency binding differs")
81
+ instruction = domain.agent_system_prompt.format(now=task.now, user_id=task.user_id)
82
+ names = {t.get("name") or t.get("function", {}).get("name") for t in domain.tool_schemas}
83
+ if state.instruction != instruction or not names or names != set(state.tool_names):
84
+ raise ValidationError("STATE visible policy or native tool catalog differs")
85
+ domains[row["case_id"]], tasks[row["case_id"]] = domain, task
86
+ files.append(path)
87
+ files.extend(p for p in domain.prompts_dir.rglob("*") if p.is_file())
88
+ return domains, tasks, source_file_pins(checkout, files)
89
+
90
+
91
+ def build_ace_statebench_feedback_input(
92
+ *,
93
+ checkout,
94
+ captures: Sequence[Mapping],
95
+ native_configuration: Mapping,
96
+ train_ids: Sequence[str],
97
+ validation_ids: Sequence[str],
98
+ heldout_ids: Sequence[str],
99
+ target_model: Mapping,
100
+ environment_model: Mapping,
101
+ output_path,
102
+ ):
103
+ """Freeze existing official captures without a model call or generated labels.
104
+
105
+ IDs are native checkpoint IDs (``domain:task_id``). The split is checked
106
+ against the pinned protocol, not inferred from the caller's labels.
107
+ """
108
+ provider = _provider()
109
+ root = verify_native_checkout(checkout, revision=provider.source_revision)
110
+ configuration = provider._validate_config(to_jsonable(native_configuration))
111
+ if configuration["split"] != "train":
112
+ raise ValidationError("STATE ACE fitting requires the official train split")
113
+ ids = [*train_ids, *validation_ids]
114
+ if (
115
+ not train_ids
116
+ or not validation_ids
117
+ or len(ids) != len(set(ids))
118
+ or set(ids) & set(heldout_ids)
119
+ or any(not isinstance(x, str) or not x for x in ids)
120
+ ):
121
+ raise ValidationError("STATE ACE groups must be disjoint nonempty fitting IDs")
122
+ selected = {r["checkpoint_id"]: r for r in captures if r["checkpoint_id"] in ids}
123
+ if len(selected) != len(ids) or len([r for r in captures if r["checkpoint_id"] in ids]) != len(
124
+ ids
125
+ ):
126
+ raise ValidationError("STATE capture IDs are missing or duplicated")
127
+ rows = []
128
+ for role, group in (("train", train_ids), ("validation", validation_ids)):
129
+ for key in group:
130
+ capture = to_jsonable(selected[key])
131
+ state = _state(capture["visible_state"])
132
+ rows.append(
133
+ dict(
134
+ case_id=key,
135
+ role=role,
136
+ official_split="train",
137
+ state=to_jsonable(state),
138
+ state_hash=state.state_hash,
139
+ protected_checkpoint=capture,
140
+ )
141
+ )
142
+ with native_process_scope(root, package="state_bench"):
143
+ _, _, pins = _native_inputs(root, configuration, rows)
144
+ if (
145
+ not isinstance(target_model, Mapping)
146
+ or not isinstance(environment_model, Mapping)
147
+ or target_model.get("model_id") != configuration.get("agent_model")
148
+ or environment_model.get("model_id") != configuration.get("evaluator_model")
149
+ ):
150
+ raise ValidationError("STATE target/judge model fingerprints must match the deployment")
151
+ value = dict(
152
+ schema=SCHEMA,
153
+ benchmark=provider.benchmark_id,
154
+ native_source_revision=provider.source_revision,
155
+ source_files=pins,
156
+ native_configuration=configuration,
157
+ fitting_rows=rows,
158
+ heldout_ids=list(heldout_ids),
159
+ source_official_split="train",
160
+ target_model=to_jsonable(target_model),
161
+ environment_model=to_jsonable(environment_model),
162
+ builder_model_calls=0,
163
+ ToolTalk_not_used=True,
164
+ )
165
+ path = Path(output_path).expanduser().absolute()
166
+ if path.resolve().is_relative_to(root):
167
+ raise ValidationError("STATE feedback input must be outside the clean checkout")
168
+ if path.exists() or path.is_symlink():
169
+ raise ValidationError("STATE feedback input must be a new file")
170
+ _save(path, value)
171
+ return dict(path=str(path), sha256=_sha(path))
172
+
173
+
174
+ class ACEStateBenchFeedback(NativePlanFeedback):
175
+ """Hash-pinned, split-isolated native STATE tools and judge feedback."""
176
+
177
+ processor_id = "agentinputlab.ace.STATE.native_tool_plan_feedback.v1"
178
+ metric = "task_completion_pass"
179
+
180
+ def __init__(
181
+ self, *, checkout, source_input, output_root, environment_base_url=None, mode="offline"
182
+ ):
183
+ self._initialize(
184
+ source_input=source_input, checkout=checkout, output_root=output_root, mode=mode
185
+ )
186
+ self.provider = _provider()
187
+ if (
188
+ self.source.get("schema") != SCHEMA
189
+ or self.source.get("benchmark") != self.provider.benchmark_id
190
+ ):
191
+ raise ValidationError("STATE ACE feedback input schema or benchmark differs")
192
+ self.checkout = verify_native_checkout(checkout, revision=self.provider.source_revision)
193
+ if self.source["native_source_revision"] != self.provider.source_revision:
194
+ raise ValidationError("STATE ACE feedback source revision differs")
195
+ self._verify_source()
196
+ self.configuration = self.provider._validate_config(self.source["native_configuration"])
197
+ if (
198
+ self.configuration["split"] != "train"
199
+ or self.source["source_official_split"] != "train"
200
+ ):
201
+ raise ValidationError("STATE ACE feedback excludes official test inputs")
202
+ with native_process_scope(self.checkout, package="state_bench"):
203
+ self.domains, self.tasks, actual = _native_inputs(
204
+ self.checkout, self.configuration, list(self.rows.values())
205
+ )
206
+ if actual != self.source["source_files"]:
207
+ raise ValidationError("STATE task, source or prompt file binding differs")
208
+ if (
209
+ self.source["target_model"].get("model_id") != self.configuration["agent_model"]
210
+ or self.source["environment_model"].get("model_id")
211
+ != self.configuration["evaluator_model"]
212
+ ):
213
+ raise ValidationError("STATE feedback target/judge deployment identity differs")
214
+ self.environment_base_url = environment_base_url or self.configuration["evaluator_base_url"]
215
+ url = (
216
+ urlsplit(self.environment_base_url)
217
+ if isinstance(self.environment_base_url, str)
218
+ else None
219
+ )
220
+ if (
221
+ url is None
222
+ or url.scheme not in {"http", "https"}
223
+ or not url.hostname
224
+ or url.username is not None
225
+ or url.password is not None
226
+ or url.query
227
+ or url.fragment
228
+ ):
229
+ raise ValidationError("STATE feedback requires an explicit judge HTTP deployment")
230
+ self.tool_names = {k: set(s.tool_names) for k, s in self.states.items()}
231
+ self._create_output()
232
+
233
+ def fingerprint(self):
234
+ self._verify_source()
235
+ return dict(
236
+ processor_id=self.processor_id,
237
+ mode=self.mode,
238
+ source_input=self.source_input,
239
+ source_files=self.source["source_files"],
240
+ source_revision=self.provider.source_revision,
241
+ scorer_id=self.provider.scorer_id,
242
+ scorer_revision=self.provider.scorer_revision,
243
+ target_model=self.source["target_model"],
244
+ environment_model=self.source["environment_model"],
245
+ environment_base_url=self.environment_base_url,
246
+ provider_source_sha256=_sha(
247
+ __import__(self.provider.__module__, fromlist=["__file__"]).__file__
248
+ ),
249
+ task_adapter_source_sha256=_sha(__file__),
250
+ feedback_metric=self.metric,
251
+ source_official_split="train",
252
+ groups={
253
+ role: [
254
+ dict(case_id=k, state_hash=r["state_hash"])
255
+ for k, r in self.rows.items()
256
+ if r["role"] == role
257
+ ]
258
+ for role in ("train", "validation")
259
+ },
260
+ noninteractive_text_plan_NOT_native_closed_loop_agent=True,
261
+ heldout_feedback_NOT_used=True,
262
+ transductive_NOT_independent_heldout=self.mode == "online",
263
+ native_user_simulator_calls=0,
264
+ judge_costs_separate=True,
265
+ automatic_retry=False,
266
+ )
267
+
268
+ def _example(self, key, split):
269
+ state = self.states[key]
270
+ visible = dict(
271
+ query=state.query,
272
+ instruction=state.instruction,
273
+ messages=state.messages,
274
+ context=state.context,
275
+ tool_names=state.tool_names,
276
+ native_function_tools=to_jsonable(self.domains[key].tool_schemas),
277
+ )
278
+ return ACEExample(
279
+ example_id=key,
280
+ input=dict(question=PLAN_INSTRUCTION + canonical_json(visible)),
281
+ expected_output=key,
282
+ split=split,
283
+ metadata=dict(
284
+ source_state_hash=state.state_hash,
285
+ task_processor=self.processor_id,
286
+ evaluator_target_is_opaque_case_id=True,
287
+ ),
288
+ )
289
+
290
+ def _native_score(self, key, plan, folder):
291
+ with native_process_scope(self.checkout, package="state_bench"):
292
+ from state_bench.env_loader import load_task_environment
293
+ from state_bench.protocol import load_default_protocol
294
+ from state_bench.schemas import StateDiff
295
+ from state_bench.scoring import (
296
+ TaskRequirementsJudge,
297
+ combine_task_completion,
298
+ evaluate_state_requirements,
299
+ )
300
+
301
+ import workers.official_arenas.providers.statebench as module
302
+
303
+ task, domain = self.tasks[key], self.domains[key]
304
+ data, _ = load_task_environment(domain, task)
305
+ env = domain.environment_class(data.deep_copy(), now=task.now)
306
+ before = env.get_full_snapshot()
307
+ calls = []
308
+ conversation = [dict(role="user", content=self.states[key].query)]
309
+ for action in plan["actions"]:
310
+ result = env.tool_handlers[action["name"]](action["arguments"])
311
+ call = dict(
312
+ name=action["name"], arguments=action["arguments"], result=to_jsonable(result)
313
+ )
314
+ calls.append(call)
315
+ conversation.append(dict(role="assistant", content="", tool_calls=[call]))
316
+ conversation.append(
317
+ dict(role="tool", name=action["name"], content=canonical_json(result))
318
+ )
319
+ conversation.append(dict(role="assistant", content=plan["response"]))
320
+ diff = StateDiff.compute(before, env.get_full_snapshot())
321
+ state_result = evaluate_state_requirements(task, diff)
322
+ _save(
323
+ folder / "before-native-judge.json",
324
+ dict(
325
+ case_id=key,
326
+ source_state_hash=self.states[key].state_hash,
327
+ plan=plan,
328
+ native_tool_calls=calls,
329
+ conversation=conversation,
330
+ state_diff=diff.to_dict(),
331
+ state_requirements_score=state_result.score,
332
+ evaluated_agent_model_calls=0,
333
+ noninteractive_plan_execution_NOT_native_agent_rollout=True,
334
+ ),
335
+ )
336
+ cfg = self.configuration
337
+ model = cfg["evaluator_model"]
338
+ client = module._openai_compatible_client(
339
+ model=model,
340
+ base_url=self.environment_base_url,
341
+ api_key_var=cfg["evaluator_api_key_var"],
342
+ )
343
+ module._install_statebench_judge_json_compat(client)
344
+ protocol = load_default_protocol()
345
+ effort = module.normalize_qwen38_reasoning_effort(
346
+ model, cfg["score_reasoning_effort"] or protocol.judge_reasoning_effort
347
+ )
348
+ judge = TaskRequirementsJudge(
349
+ client=client,
350
+ prompts_dir=domain.prompts_dir,
351
+ system_prompt=domain.judge_system_prompt,
352
+ reasoning_effort=effort,
353
+ )
354
+ try:
355
+ task_result = judge.evaluate(task, conversation, calls, diff)
356
+ finally:
357
+ client._client.close()
358
+ if task_result is None:
359
+ raise ValidationError("original native STATE task judge did not complete")
360
+ score = combine_task_completion(state_result, task_result)
361
+ if score is None:
362
+ raise ValidationError("original native STATE completion is absent")
363
+ value = dict(
364
+ status="native_STATE_tool_plan_scored",
365
+ case_id=key,
366
+ native_metrics=dict(
367
+ task_completion_pass=float(score),
368
+ state_requirements_met=float(state_result.score),
369
+ task_requirements_met=float(task_result.score),
370
+ ),
371
+ evaluator_agent_model_calls=0,
372
+ native_user_simulator_calls=0,
373
+ task_judge_model=model,
374
+ original_judge_controls=True,
375
+ )
376
+ _save(folder / "native-score.json", value)
377
+ return value
378
+
379
+
380
+ def create_statebench_feedback(**configuration):
381
+ """Trusted startup factory for ``python -m workers.ace.driver``."""
382
+ return ACEStateBenchFeedback(**configuration)