agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,960 @@
1
+ """Production provider for the pinned Microsoft STATE-Bench runtime."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import hashlib
7
+ import os
8
+ import subprocess
9
+ from collections.abc import Mapping, Sequence
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from ..protocol import (
14
+ OfficialArenaError,
15
+ ProviderArmResult,
16
+ content_hash,
17
+ intervention_usage,
18
+ materialized_state,
19
+ require_materialized_budget,
20
+ resolve_shared_budget_config,
21
+ validate_dependency_hashes,
22
+ visible_shared_budget,
23
+ )
24
+ from ._message_compat import (
25
+ fold_responses_system_messages_into_instructions,
26
+ normalize_qwen38_reasoning_effort,
27
+ requires_qwen38_initial_system_message,
28
+ )
29
+ from ._live_history import ResponsesHistorySession, validate_history_transform
30
+
31
+
32
+ class _SharedBudgetStop(RuntimeError):
33
+ """Internal, scored hard stop after a STATE-Bench resource cap is reached."""
34
+
35
+
36
+ class _EvaluatedAgentOutputInvalid(RuntimeError):
37
+ """A recognized malformed model generation from the evaluated agent."""
38
+
39
+ def __init__(self, error: Exception) -> None:
40
+ super().__init__(f"{type(error).__name__}: evaluated agent output is invalid")
41
+ self.error = error
42
+
43
+
44
+ def _model_output_invalid_result(
45
+ *,
46
+ replay_record: Mapping[str, Any],
47
+ role: str,
48
+ seed: int,
49
+ work_dir: Path,
50
+ error: Exception,
51
+ measured_usage: Mapping[str, int | float],
52
+ ) -> ProviderArmResult:
53
+ """Materialize malformed generated tool arguments as an official zero score."""
54
+
55
+ protected = replay_record["protected"]
56
+ trace = {
57
+ "provider": "state_bench",
58
+ "source_revision": StateBenchProvider.source_revision,
59
+ "protocol_id": protected["protocol_id"],
60
+ "task_id": protected["task_id"],
61
+ "domain": protected["domain"],
62
+ "role": role,
63
+ "seed": seed,
64
+ "termination_reason": "generated_tool_arguments_invalid",
65
+ "generation_controls": {
66
+ "hard_stops": 1,
67
+ "failure_type": type(error).__name__,
68
+ "failed_generation_scope": "evaluated_agent",
69
+ },
70
+ "score_provenance": "paper_model_output_invalid_zero_action",
71
+ }
72
+ work_dir.joinpath("model-output-invalid.json").write_text(
73
+ json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
74
+ encoding="utf-8",
75
+ )
76
+ usage = {
77
+ **measured_usage,
78
+ "cost_usd": 0.0,
79
+ "unpriced_model_calls": int(measured_usage.get("model_calls", 0)),
80
+ }
81
+ return ProviderArmResult(
82
+ metrics={
83
+ "task_completion_pass": 0.0,
84
+ "state_requirements_met": 0.0,
85
+ "task_requirements_met": 0.0,
86
+ "adapter_hard_stops": 1.0,
87
+ },
88
+ trace=trace,
89
+ usage=usage,
90
+ runtime_instance_id=content_hash(
91
+ {
92
+ "provider": "state_bench",
93
+ "checkpoint_id": replay_record["checkpoint_id"],
94
+ "role": role,
95
+ "seed": seed,
96
+ "termination_reason": trace["termination_reason"],
97
+ "work_dir": str(work_dir),
98
+ }
99
+ ),
100
+ )
101
+
102
+
103
+ class StateBenchProvider:
104
+ benchmark_id = "state_bench.common_support"
105
+ source_revision = "4efcbf2d4fe60df04878859b692d9391f3d5b33a"
106
+ scorer_id = "state_bench.official_v0.8.1"
107
+ scorer_revision = "00f5c68a83e55535797435f1fed59e9d334878ef3431f9906d703778dbcf09e8"
108
+
109
+ _CONFIG_FIELDS = frozenset(
110
+ {
111
+ "domains",
112
+ "split",
113
+ "limit",
114
+ "agent_provider",
115
+ "agent_api_key_var",
116
+ "score_reasoning_effort",
117
+ "locked_dependency_hashes",
118
+ }
119
+ )
120
+ _OPENAI_COMPATIBLE_FIELDS = frozenset(
121
+ {
122
+ "agent_model",
123
+ "agent_base_url",
124
+ "evaluator_model",
125
+ "evaluator_base_url",
126
+ "evaluator_api_key_var",
127
+ }
128
+ )
129
+ _OPTIONAL_CONFIG_FIELDS = frozenset({"budget_override"})
130
+
131
+ def __init__(self, *, history_transform=None):
132
+ validate_history_transform(history_transform)
133
+ self.history_transform = history_transform
134
+
135
+ def doctor(self, config: Mapping[str, Any]) -> tuple[str, ...]:
136
+ try:
137
+ self._validate_config(config)
138
+ head = subprocess.check_output(
139
+ ["git", "rev-parse", "HEAD"],
140
+ text=True,
141
+ stderr=subprocess.DEVNULL,
142
+ ).strip()
143
+ if head != self.source_revision:
144
+ raise OfficialArenaError("STATE-Bench checkout revision differs")
145
+ from state_bench.client import (
146
+ build_llm_client,
147
+ build_locked_judge_client,
148
+ build_user_sim_client,
149
+ )
150
+ from state_bench.domain import get_domain_config
151
+ from state_bench.orchestrator import run_task
152
+ from state_bench.scoring import combine_task_completion
153
+
154
+ del (
155
+ build_llm_client,
156
+ build_locked_judge_client,
157
+ build_user_sim_client,
158
+ get_domain_config,
159
+ run_task,
160
+ combine_task_completion,
161
+ )
162
+ except Exception as exc:
163
+ return (f"{type(exc).__name__}: {exc}",)
164
+ return ()
165
+
166
+ def capture(
167
+ self,
168
+ config: Mapping[str, Any],
169
+ *,
170
+ output_root: Path,
171
+ ) -> tuple[Mapping[str, Any], ...]:
172
+ del output_root
173
+ parsed = self._validate_config(config)
174
+ from state_bench.domain import get_domain_config
175
+ from state_bench.paths import domain_tasks_dir
176
+ from state_bench.protocol import load_default_protocol, load_split_task_ids
177
+ from state_bench.schemas import TaskDefinition
178
+
179
+ protocol = load_default_protocol()
180
+ errors = protocol.validate_prompt_hashes()
181
+ if errors:
182
+ raise OfficialArenaError("STATE-Bench protocol prompt hashes differ")
183
+
184
+ rows: list[Mapping[str, Any]] = []
185
+ for domain_name in parsed["domains"]:
186
+ domain = get_domain_config(domain_name)
187
+ task_ids = load_split_task_ids(
188
+ domain_name,
189
+ parsed["split"],
190
+ protocol.split_version,
191
+ )
192
+ for task_id in task_ids:
193
+ task = TaskDefinition.load(domain_tasks_dir(domain_name) / f"{task_id}.json")
194
+ user_id = task.user_id
195
+ if not user_id:
196
+ continue
197
+ instruction = domain.agent_system_prompt.format(
198
+ now=task.now,
199
+ user_id=user_id,
200
+ )
201
+ tool_names = tuple(
202
+ str(tool.get("name"))
203
+ for tool in domain.tool_schemas
204
+ if isinstance(tool, Mapping) and isinstance(tool.get("name"), str)
205
+ )
206
+ if not tool_names:
207
+ raise OfficialArenaError("STATE-Bench domain exposes no tool schemas")
208
+ visible_state = {
209
+ "query": task.opening_message,
210
+ "task_id": task.task_id,
211
+ "instruction": instruction,
212
+ "context": [
213
+ {
214
+ "text": instruction,
215
+ "block_id": "official_policy",
216
+ "source": "state_bench.domain.agent_system_prompt",
217
+ "score": None,
218
+ "metadata": {"domain": domain_name},
219
+ },
220
+ {
221
+ "text": json.dumps(
222
+ domain.tool_schemas,
223
+ ensure_ascii=False,
224
+ sort_keys=True,
225
+ ),
226
+ "block_id": "official_tool_catalog",
227
+ "source": "state_bench.domain.tool_schemas",
228
+ "score": None,
229
+ "metadata": {"domain": domain_name},
230
+ },
231
+ ],
232
+ "messages": [
233
+ {
234
+ "role": "user",
235
+ "content": task.opening_message,
236
+ "name": None,
237
+ }
238
+ ],
239
+ "tool_names": list(tool_names),
240
+ "metadata": {
241
+ "benchmark_id": self.benchmark_id,
242
+ "domain": domain_name,
243
+ "checkpoint_phase": "turn_zero",
244
+ "clarification_roundtrip_available": True,
245
+ "state_restore_available": True,
246
+ },
247
+ "budget": visible_shared_budget(
248
+ parsed,
249
+ {
250
+ "max_model_calls": domain.max_agent_turns * 9,
251
+ "max_retrieval_calls": 32,
252
+ "max_tool_calls": domain.max_agent_turns * 8,
253
+ "max_output_tokens": 65536,
254
+ "max_context_tokens": None,
255
+ "max_cost_usd": None,
256
+ },
257
+ ),
258
+ }
259
+ rows.append(
260
+ {
261
+ "checkpoint_id": f"{domain_name}:{task.task_id}",
262
+ "group_key": f"{domain_name}:{task.task_id}",
263
+ "visible_state": visible_state,
264
+ "eligible_families": [
265
+ "reformulate",
266
+ "decompose",
267
+ "acquire",
268
+ "clarify",
269
+ "curate",
270
+ ],
271
+ "protected": {
272
+ "domain": domain_name,
273
+ "task_id": task.task_id,
274
+ "user_id": user_id,
275
+ "protocol_id": protocol.protocol_id,
276
+ "locked_dependency_hashes": parsed["locked_dependency_hashes"],
277
+ },
278
+ }
279
+ )
280
+ if len(rows) >= parsed["limit"]:
281
+ return tuple(rows)
282
+ if not rows:
283
+ raise OfficialArenaError("STATE-Bench capture found no protocol tasks")
284
+ return tuple(rows)
285
+
286
+ def run_arm(
287
+ self, config: Mapping[str, Any], *, replay_record: Mapping[str, Any],
288
+ arm_payload: Mapping[str, Any], seed: int, role: str, work_dir: Path,
289
+ ) -> ProviderArmResult:
290
+ context = {}
291
+ try:
292
+ result = self._run_arm(config, replay_record=replay_record, arm_payload=arm_payload,
293
+ seed=seed, role=role, work_dir=work_dir, _history_context=context)
294
+ finally:
295
+ agent = context.get("agent")
296
+ session = getattr(agent, "_history_session", None)
297
+ if session is not None:
298
+ session.write_receipt(work_dir / "live-history.json",
299
+ target_usage=agent.agentinputlab_usage())
300
+ if session is None:
301
+ return result
302
+ receipt_hash = hashlib.sha256((work_dir / "live-history.json").read_bytes()).hexdigest()
303
+ return ProviderArmResult(metrics=result.metrics,
304
+ trace={**result.trace, "live_history_transform": session.hook.fingerprint(),
305
+ "live_history_receipt": {"path": "live-history.json", "sha256": receipt_hash},
306
+ "history_usage": dict(session.usage)},
307
+ usage=session.combined_usage(result.usage),
308
+ runtime_instance_id=content_hash({"native_runtime": result.runtime_instance_id,
309
+ "live_history_receipt_sha256": receipt_hash}))
310
+
311
+ def _run_arm(
312
+ self,
313
+ config: Mapping[str, Any],
314
+ *,
315
+ replay_record: Mapping[str, Any],
316
+ arm_payload: Mapping[str, Any],
317
+ seed: int,
318
+ role: str,
319
+ work_dir: Path,
320
+ _history_context: dict,
321
+ ) -> ProviderArmResult:
322
+ parsed = self._validate_config(config)
323
+ state = materialized_state(arm_payload)
324
+ budget = require_materialized_budget(state, parsed)
325
+ optimizer_usage = intervention_usage(arm_payload)
326
+ remaining_model_calls = max(
327
+ 0,
328
+ int(budget["max_model_calls"]) - int(optimizer_usage.get("model_calls", 0)),
329
+ )
330
+ remaining_output_tokens = max(
331
+ 0,
332
+ int(budget["max_output_tokens"]) - int(optimizer_usage.get("output_tokens", 0)),
333
+ )
334
+ context_cap = budget.get("max_context_tokens")
335
+ remaining_input_tokens = (
336
+ 2**63 - 1
337
+ if context_cap is None
338
+ else max(
339
+ 0,
340
+ int(context_cap) - int(optimizer_usage.get("input_tokens", 0)),
341
+ )
342
+ )
343
+ remaining_tool_calls = max(
344
+ 0,
345
+ int(budget["max_tool_calls"]) - int(optimizer_usage.get("tool_calls", 0)),
346
+ )
347
+ override = parsed.get("budget_override")
348
+ per_call_output_cap = (
349
+ int(override["max_output_tokens_per_call"])
350
+ if isinstance(override, Mapping)
351
+ else min(512, remaining_output_tokens)
352
+ )
353
+ protected = replay_record["protected"]
354
+ domain_name = protected["domain"]
355
+ task_id = protected["task_id"]
356
+ user_id = protected["user_id"]
357
+
358
+ from state_bench.agents.base import AgentRuntimeContext
359
+ from state_bench.client import (
360
+ build_llm_client,
361
+ build_locked_judge_client,
362
+ build_user_sim_client,
363
+ )
364
+ from state_bench.domain import get_domain_config
365
+ from state_bench.env_loader import load_task_environment
366
+ from state_bench.orchestrator import run_task
367
+ from state_bench.paths import domain_tasks_dir
368
+ from state_bench.protocol import load_default_protocol
369
+ from state_bench.schemas import TaskDefinition
370
+ from state_bench.scoring import (
371
+ TaskRequirementsJudge,
372
+ combine_task_completion,
373
+ evaluate_state_requirements,
374
+ )
375
+
376
+ domain = get_domain_config(domain_name)
377
+ task = TaskDefinition.load(domain_tasks_dir(domain_name) / f"{task_id}.json")
378
+ env_data, _ = load_task_environment(domain, task)
379
+ env_copy = env_data.deep_copy()
380
+ env = domain.environment_class(env_copy, now=task.now)
381
+ agent_system_prompt = domain.agent_system_prompt.format(
382
+ now=task.now,
383
+ user_id=user_id,
384
+ )
385
+ if self._OPENAI_COMPATIBLE_FIELDS.issubset(parsed):
386
+ agent_client = _openai_compatible_client(
387
+ model=parsed["agent_model"],
388
+ base_url=parsed["agent_base_url"],
389
+ api_key_var=parsed["agent_api_key_var"],
390
+ )
391
+ simulator_client = _openai_compatible_client(
392
+ model=parsed["evaluator_model"],
393
+ base_url=parsed["evaluator_base_url"],
394
+ api_key_var=parsed["evaluator_api_key_var"],
395
+ )
396
+ judge_client = _openai_compatible_client(
397
+ model=parsed["evaluator_model"],
398
+ base_url=parsed["evaluator_base_url"],
399
+ api_key_var=parsed["evaluator_api_key_var"],
400
+ )
401
+ _install_statebench_judge_json_compat(judge_client)
402
+ else:
403
+ agent_client = build_llm_client(
404
+ provider=parsed["agent_provider"],
405
+ api_key_var=parsed["agent_api_key_var"],
406
+ )
407
+ simulator_client = build_user_sim_client()
408
+ judge_client = build_locked_judge_client()
409
+ runtime_context = AgentRuntimeContext(
410
+ task_id=task.task_id,
411
+ user_id=user_id,
412
+ domain=domain.name,
413
+ now=task.now,
414
+ task_summary=task.task_summary,
415
+ state_requirements=task.state_requirements,
416
+ task_requirements=task.task_requirements,
417
+ )
418
+ agent = _intervened_agent_class()(
419
+ agent_client,
420
+ agent_system_prompt,
421
+ domain.tool_schemas,
422
+ env.tool_handlers,
423
+ runtime_context=runtime_context,
424
+ materialized=state,
425
+ max_api_calls=remaining_model_calls,
426
+ max_output_tokens=remaining_output_tokens,
427
+ max_input_tokens=remaining_input_tokens,
428
+ max_output_tokens_per_call=per_call_output_cap,
429
+ max_tool_calls=remaining_tool_calls,
430
+ history_transform=self.history_transform,
431
+ )
432
+ _history_context["agent"] = agent
433
+ try:
434
+ trajectory = run_task(
435
+ task,
436
+ env_data,
437
+ user_id,
438
+ client=None,
439
+ domain=domain,
440
+ agent=agent,
441
+ env=env,
442
+ trajectory_metadata={
443
+ "agentinputlab": {
444
+ "release_id": arm_payload["release_id"],
445
+ "role": role,
446
+ "seed": seed,
447
+ "output_state_hash": arm_payload["intervention_execution"][
448
+ "output_state_hash"
449
+ ],
450
+ }
451
+ },
452
+ simulator_client=simulator_client,
453
+ )
454
+ except _EvaluatedAgentOutputInvalid as failure:
455
+ return _model_output_invalid_result(
456
+ replay_record=replay_record,
457
+ role=role,
458
+ seed=seed,
459
+ work_dir=work_dir,
460
+ error=failure.error,
461
+ measured_usage=agent.agentinputlab_usage(),
462
+ )
463
+
464
+ tool_calls = [
465
+ call
466
+ for message in trajectory.conversation
467
+ for call in (message.get("tool_calls") or [])
468
+ ]
469
+ state_result = evaluate_state_requirements(task, trajectory.state_diff)
470
+ protocol = load_default_protocol()
471
+ judge_reasoning_effort = normalize_qwen38_reasoning_effort(
472
+ parsed.get("evaluator_model"),
473
+ parsed["score_reasoning_effort"] or protocol.judge_reasoning_effort,
474
+ )
475
+ judge = TaskRequirementsJudge(
476
+ client=judge_client,
477
+ prompts_dir=domain.prompts_dir,
478
+ system_prompt=domain.judge_system_prompt,
479
+ reasoning_effort=judge_reasoning_effort,
480
+ )
481
+ task_result = judge.evaluate(
482
+ task,
483
+ trajectory.conversation,
484
+ tool_calls,
485
+ trajectory.state_diff,
486
+ )
487
+ completion = combine_task_completion(state_result, task_result)
488
+ if completion is None:
489
+ raise OfficialArenaError("STATE-Bench official task judge did not complete")
490
+
491
+ trace = {
492
+ "provider": "state_bench",
493
+ "source_revision": self.source_revision,
494
+ "protocol_id": protected["protocol_id"],
495
+ "task_id": task_id,
496
+ "domain": domain_name,
497
+ "role": role,
498
+ "seed": seed,
499
+ "conversation": trajectory.conversation,
500
+ "state_diff": trajectory.state_diff.to_dict(),
501
+ "efficiency": (trajectory.efficiency.to_dict() if trajectory.efficiency else None),
502
+ "task_completion_pass": completion,
503
+ "adapter_compatibility_repairs": {
504
+ "qwen3_4b_judge_reasoning_json": int(
505
+ getattr(
506
+ judge_client,
507
+ "_agentinputlab_qwen3_4b_judge_json_repairs",
508
+ 0,
509
+ )
510
+ )
511
+ },
512
+ }
513
+ token_usage = trajectory.token_usage.to_dict()
514
+ audited_agent_usage = agent.agentinputlab_usage()
515
+ usage = {
516
+ # Conversation rows are not API-call receipts: a clarification or
517
+ # budget-stop message can be emitted without calling the model.
518
+ # The wrapper counters below are incremented exactly at the model
519
+ # and tool boundaries and are the only counters compatible with
520
+ # the shared remaining-budget calculation above.
521
+ "model_calls": audited_agent_usage["model_calls"],
522
+ "tool_calls": audited_agent_usage["tool_calls"],
523
+ "input_tokens": audited_agent_usage["input_tokens"],
524
+ "output_tokens": audited_agent_usage["output_tokens"],
525
+ "cost_usd": float(token_usage.get("total_cost_usd", 0.0) or 0.0),
526
+ }
527
+ work_dir.joinpath("trajectory.json").write_text(
528
+ json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
529
+ encoding="utf-8",
530
+ )
531
+ return ProviderArmResult(
532
+ metrics={
533
+ "task_completion_pass": float(completion),
534
+ "state_requirements_met": float(state_result.score),
535
+ "task_requirements_met": float(task_result.score),
536
+ },
537
+ trace=trace,
538
+ usage=usage,
539
+ runtime_instance_id=content_hash(
540
+ {
541
+ "provider": "state_bench",
542
+ "checkpoint_id": replay_record["checkpoint_id"],
543
+ "role": role,
544
+ "seed": seed,
545
+ "work_dir": str(work_dir),
546
+ }
547
+ ),
548
+ )
549
+
550
+ def _validate_config(self, config: Mapping[str, Any]) -> Mapping[str, Any]:
551
+ config = resolve_shared_budget_config(config)
552
+ fields = set(config) if isinstance(config, Mapping) else set()
553
+ valid_fields = {
554
+ self._CONFIG_FIELDS,
555
+ self._CONFIG_FIELDS | self._OPENAI_COMPATIBLE_FIELDS,
556
+ self._CONFIG_FIELDS | self._OPTIONAL_CONFIG_FIELDS,
557
+ self._CONFIG_FIELDS | self._OPENAI_COMPATIBLE_FIELDS | self._OPTIONAL_CONFIG_FIELDS,
558
+ }
559
+ if not isinstance(config, Mapping) or fields not in valid_fields:
560
+ raise OfficialArenaError("STATE-Bench provider config fields are invalid")
561
+ domains = config["domains"]
562
+ if (
563
+ isinstance(domains, (str, bytes))
564
+ or not isinstance(domains, Sequence)
565
+ or not domains
566
+ or len(domains) != len(set(domains))
567
+ or any(not isinstance(item, str) or not item.strip() for item in domains)
568
+ ):
569
+ raise OfficialArenaError("STATE-Bench domains must be unique strings")
570
+ split = config["split"]
571
+ if split not in {"train", "test"}:
572
+ raise OfficialArenaError("STATE-Bench split must be train or test")
573
+ limit = config["limit"]
574
+ if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
575
+ raise OfficialArenaError("STATE-Bench limit must be positive")
576
+ if config["agent_provider"] not in {"openai", "azure_openai"}:
577
+ raise OfficialArenaError("STATE-Bench agent_provider is unsupported")
578
+ key_fields = ["agent_api_key_var"]
579
+ if self._OPENAI_COMPATIBLE_FIELDS.issubset(config):
580
+ if config["agent_provider"] != "openai":
581
+ raise OfficialArenaError(
582
+ "STATE-Bench OpenAI-compatible runtime requires agent_provider=openai"
583
+ )
584
+ for name in (
585
+ "agent_model",
586
+ "agent_base_url",
587
+ "evaluator_model",
588
+ "evaluator_base_url",
589
+ ):
590
+ if not isinstance(config[name], str) or not config[name].strip():
591
+ raise OfficialArenaError(f"STATE-Bench {name} is invalid")
592
+ key_fields.append("evaluator_api_key_var")
593
+ for name in key_fields:
594
+ value = config[name]
595
+ if (
596
+ not isinstance(value, str)
597
+ or not value.strip()
598
+ or not value.replace("_", "").isalnum()
599
+ ):
600
+ raise OfficialArenaError(f"STATE-Bench {name} is invalid")
601
+ effort = config["score_reasoning_effort"]
602
+ if effort is not None and effort not in {"low", "medium", "high"}:
603
+ raise OfficialArenaError("STATE-Bench score_reasoning_effort is invalid")
604
+ validate_dependency_hashes(
605
+ config["locked_dependency_hashes"],
606
+ label="STATE-Bench",
607
+ )
608
+ return config
609
+
610
+
611
+ def _openai_compatible_client(
612
+ *,
613
+ model: str,
614
+ base_url: str,
615
+ api_key_var: str,
616
+ ) -> Any:
617
+ """Inject only the transport while retaining pinned STATE-Bench client logic."""
618
+ from openai import OpenAI
619
+ from state_bench.client import LLMClient
620
+
621
+ api_key = os.environ.get(api_key_var)
622
+ if not api_key:
623
+ raise OfficialArenaError(
624
+ f"STATE-Bench OpenAI-compatible environment {api_key_var} is missing"
625
+ )
626
+ client = LLMClient.__new__(LLMClient)
627
+ client.provider = "openai"
628
+ client.endpoint = base_url
629
+ client.deployment = model
630
+ client.deployments = [model]
631
+ client._chat_response_history = {}
632
+ client._client = OpenAI(api_key=api_key, base_url=base_url, max_retries=0)
633
+ _install_stateless_tool_history(client)
634
+ return client
635
+
636
+
637
+ class _ResponseTextOverride:
638
+ """Delegate a Responses object while exposing one validated text repair."""
639
+
640
+ def __init__(self, response: Any, output_text: str) -> None:
641
+ self._response = response
642
+ self.output_text = output_text
643
+
644
+ def __getattr__(self, name: str) -> Any:
645
+ return getattr(self._response, name)
646
+
647
+
648
+ def _response_field(value: Any, name: str) -> Any:
649
+ if isinstance(value, Mapping):
650
+ return value.get(name)
651
+ return getattr(value, name, None)
652
+
653
+
654
+ def _qwen3_4b_judge_reasoning_json(response: Any, *, model: str) -> str | None:
655
+ """Recover only a completed Qwen3-4B judge JSON emitted as reasoning text.
656
+
657
+ Some Qwen3-4B vLLM Responses generations place the JSON-object answer in
658
+ a ``reasoning_text`` block while leaving the SDK's ``output_text`` empty.
659
+ STATE-Bench then retries and ultimately drops an otherwise valid official
660
+ judge result. Keep the repair deliberately narrower than generic response
661
+ promotion: it applies only to this served alias, completed responses, one
662
+ reasoning block, and the exact top-level shape consumed by
663
+ ``TaskRequirementsJudge``.
664
+ """
665
+
666
+ alias = model.strip().casefold().replace("_", "-")
667
+ if alias != "qwen3-4b":
668
+ return None
669
+ if _response_field(response, "status") != "completed":
670
+ return None
671
+ if _response_field(response, "output_text") not in {None, ""}:
672
+ return None
673
+
674
+ candidates: list[str] = []
675
+ output = _response_field(response, "output")
676
+ if not isinstance(output, Sequence) or isinstance(output, (str, bytes)):
677
+ return None
678
+ for item in output:
679
+ if _response_field(item, "type") != "reasoning":
680
+ continue
681
+ content = _response_field(item, "content")
682
+ if not isinstance(content, Sequence) or isinstance(content, (str, bytes)):
683
+ continue
684
+ for block in content:
685
+ if _response_field(block, "type") != "reasoning_text":
686
+ continue
687
+ text = _response_field(block, "text")
688
+ if isinstance(text, str) and text.strip():
689
+ candidates.append(text)
690
+ if len(candidates) != 1:
691
+ return None
692
+ try:
693
+ parsed = json.loads(candidates[0])
694
+ except (TypeError, ValueError):
695
+ return None
696
+ if not isinstance(parsed, Mapping) or not isinstance(parsed.get("details"), list):
697
+ return None
698
+ return candidates[0]
699
+
700
+
701
+ def _install_statebench_judge_json_compat(client: Any) -> None:
702
+ """Install the narrow Qwen3-4B JSON repair on the judge client only."""
703
+
704
+ original = client.complete_json_response
705
+ client._agentinputlab_qwen3_4b_judge_json_repairs = 0
706
+
707
+ def complete_json_response(**call_kwargs: Any) -> Any:
708
+ response = original(**call_kwargs)
709
+ recovered = _qwen3_4b_judge_reasoning_json(
710
+ response,
711
+ model=str(getattr(client, "deployment", "")),
712
+ )
713
+ if recovered is None:
714
+ return response
715
+ client._agentinputlab_qwen3_4b_judge_json_repairs += 1
716
+ return _ResponseTextOverride(response, recovered)
717
+
718
+ client.complete_json_response = complete_json_response
719
+
720
+
721
+ def _install_stateless_tool_history(client: Any) -> None:
722
+ """Replay Responses tool history for stateless OpenAI-compatible servers.
723
+
724
+ STATE-Bench natively chains tool turns with ``previous_response_id``.
725
+ Self-hosted vLLM implements the Responses wire format but does not retain
726
+ provider-side response objects. Preserve the upstream turn loop while
727
+ replacing only that transport assumption with explicit input replay.
728
+ """
729
+
730
+ original = client.complete_with_tools
731
+ histories: dict[str, list[Any]] = {}
732
+
733
+ def replayable(items: list[Any]) -> list[Any]:
734
+ if str(getattr(client, "deployment", "")).casefold() != "gpt-oss-20b":
735
+ return items
736
+ # vLLM's Responses endpoint does not accept its own occasional
737
+ # GPT-OSS ``mcp_call`` output as a subsequent input item. STATE-Bench
738
+ # ignores that item (only function_call is executable), so omit only
739
+ # this inert parser artifact from stateless replay.
740
+ return [
741
+ item
742
+ for item in items
743
+ if (
744
+ item.get("type") if isinstance(item, Mapping) else getattr(item, "type", None)
745
+ )
746
+ != "mcp_call"
747
+ ]
748
+
749
+ def stateless_complete_with_tools(**call_kwargs: Any) -> Any:
750
+ previous = call_kwargs.get("previous_response_id")
751
+ incoming = list(call_kwargs.get("input") or ())
752
+ if previous is not None:
753
+ history = histories.get(str(previous))
754
+ if history is None:
755
+ raise OfficialArenaError(
756
+ "STATE-Bench stateless tool continuation has no local response history"
757
+ )
758
+ incoming = [*replayable(history), *incoming]
759
+ call_kwargs["input"] = incoming
760
+ call_kwargs["previous_response_id"] = None
761
+ else:
762
+ history_view = getattr(client, "_agentinputlab_responses_history", None)
763
+ if history_view is not None:
764
+ call_kwargs = history_view(dict(call_kwargs, input=incoming))
765
+ incoming = list(call_kwargs["input"])
766
+ if requires_qwen38_initial_system_message(client.deployment):
767
+ normalized_input, normalized_instructions = (
768
+ fold_responses_system_messages_into_instructions(
769
+ incoming,
770
+ call_kwargs.get("instructions"),
771
+ )
772
+ )
773
+ call_kwargs["input"] = normalized_input
774
+ call_kwargs["instructions"] = normalized_instructions
775
+ response = original(**call_kwargs)
776
+ response_id = getattr(response, "id", None)
777
+ output = getattr(response, "output", None)
778
+ if not isinstance(response_id, str) or not response_id or not isinstance(output, list):
779
+ raise OfficialArenaError("STATE-Bench Responses result has no replayable output")
780
+ histories[response_id] = [*incoming, *output]
781
+ return response
782
+
783
+ client.complete_with_tools = stateless_complete_with_tools
784
+ client._agentinputlab_stateless_responses = True
785
+
786
+
787
+ def _is_gpt_oss_harmony_parse_failure(error: Exception, *, model_name: str) -> bool:
788
+ """Recognize only GPT-OSS response-parser HTTP-500 failures."""
789
+
790
+ if model_name.casefold() != "gpt-oss-20b" or getattr(error, "status_code", None) != 500:
791
+ return False
792
+ body = getattr(error, "body", None)
793
+ message: Any = None
794
+ if isinstance(body, Mapping):
795
+ message = body.get("message")
796
+ nested = body.get("error")
797
+ if message is None and isinstance(nested, Mapping):
798
+ message = nested.get("message")
799
+ if not isinstance(message, str):
800
+ return False
801
+ normalized = message.casefold()
802
+ return normalized.startswith("unexpected tokens remaining in message header") or normalized.startswith(
803
+ "unexpected token "
804
+ )
805
+
806
+
807
+ def _intervened_agent_class() -> type[Any]:
808
+ from state_bench.agents.state_bench import StateBenchAgent
809
+
810
+ class IntervenedStateBenchAgent(StateBenchAgent):
811
+ def __init__(
812
+ self,
813
+ *args: Any,
814
+ materialized: Mapping[str, Any],
815
+ max_api_calls: int,
816
+ max_output_tokens: int,
817
+ max_input_tokens: int,
818
+ max_output_tokens_per_call: int,
819
+ max_tool_calls: int,
820
+ history_transform=None,
821
+ **kwargs: Any,
822
+ ):
823
+ super().__init__(*args, **kwargs)
824
+ self._materialized = materialized
825
+ self._first_turn = True
826
+ self._budget_stopped = False
827
+ self._budget_api_calls = 0
828
+ self._budget_input_tokens = 0
829
+ self._budget_output_tokens = 0
830
+ self._budget_tool_calls = 0
831
+ self._max_api_calls = max_api_calls
832
+ self._max_input_tokens = max_input_tokens
833
+ self._max_output_tokens = max_output_tokens
834
+ self._max_output_tokens_per_call = max_output_tokens_per_call
835
+ self._max_tool_calls = max_tool_calls
836
+ self._history_session = (ResponsesHistorySession(history_transform, materialized,
837
+ dict(max_model_calls=max_api_calls, max_output_tokens=max_output_tokens))
838
+ if history_transform is not None else None)
839
+ if self._history_session is not None:
840
+ if not hasattr(self.client, "_agentinputlab_stateless_responses"):
841
+ raise OfficialArenaError("STATE history requires the native stateless Responses transport")
842
+
843
+ def history_view(call_kwargs):
844
+ call_kwargs["input"], call_kwargs["instructions"] = self._history_session.responses(
845
+ call_kwargs["input"], instructions=call_kwargs.get("instructions"),
846
+ agent_usage=self.agentinputlab_usage())
847
+ remaining = self._history_session.remaining(self.agentinputlab_usage())
848
+ if (remaining["max_model_calls"] < 1 or remaining["max_output_tokens"] < 1
849
+ or self._budget_input_tokens + self._history_session.usage.get("input_tokens", 0)
850
+ >= self._max_input_tokens):
851
+ raise _SharedBudgetStop
852
+ call_kwargs["max_tokens"] = min(int(call_kwargs["max_tokens"]),
853
+ remaining["max_output_tokens"])
854
+ return call_kwargs
855
+
856
+ self.client._agentinputlab_responses_history = history_view
857
+ original = self.client.complete_with_tools
858
+
859
+ def budgeted_complete_with_tools(**call_kwargs: Any) -> Any:
860
+ remaining_output = self._max_output_tokens - self._budget_output_tokens
861
+ calls = self._budget_api_calls
862
+ if self._history_session is not None:
863
+ remaining = self._history_session.remaining(self.agentinputlab_usage())
864
+ remaining_output = remaining["max_output_tokens"]
865
+ calls += int(self._history_session.usage.get("model_calls", 0))
866
+ if calls >= self._max_api_calls or remaining_output < 1:
867
+ raise _SharedBudgetStop
868
+ requested = int(call_kwargs.get("max_tokens", 4096))
869
+ call_kwargs["max_tokens"] = min(
870
+ requested,
871
+ self._max_output_tokens_per_call,
872
+ remaining_output,
873
+ )
874
+ response = original(**call_kwargs)
875
+ self._budget_api_calls += 1
876
+ usage = getattr(response, "usage", None)
877
+ self._budget_input_tokens += int(getattr(usage, "input_tokens", 0) or 0)
878
+ self._budget_output_tokens += int(getattr(usage, "output_tokens", 0) or 0)
879
+ if (
880
+ self._budget_input_tokens + (self._history_session.usage.get("input_tokens", 0)
881
+ if self._history_session is not None else 0) > self._max_input_tokens
882
+ or self._budget_output_tokens + (self._history_session.usage.get("output_tokens", 0)
883
+ if self._history_session is not None else 0) > self._max_output_tokens
884
+ ):
885
+ raise _SharedBudgetStop
886
+ return response
887
+
888
+ self.client.complete_with_tools = budgeted_complete_with_tools
889
+ original_handlers = dict(self.tool_handlers)
890
+ for tool_name, handler in original_handlers.items():
891
+
892
+ def budgeted_handler(
893
+ arguments: Any,
894
+ *,
895
+ _handler: Any = handler,
896
+ ) -> Any:
897
+ if self._budget_tool_calls >= self._max_tool_calls:
898
+ raise _SharedBudgetStop
899
+ self._budget_tool_calls += 1
900
+ return _handler(arguments)
901
+
902
+ self.tool_handlers[tool_name] = budgeted_handler
903
+
904
+ def prepare_conversation(self, conversation: list[Any]) -> list[Any]:
905
+ prepared = [dict(item) if isinstance(item, Mapping) else item for item in conversation]
906
+ if prepared and isinstance(prepared[0], Mapping):
907
+ prepared[0] = {**prepared[0], "content": self._materialized["query"]}
908
+ additions = [self._materialized.get("instruction", "")]
909
+ additions.extend(
910
+ item.get("text", "")
911
+ for item in self._materialized.get("context", [])
912
+ if isinstance(item, Mapping)
913
+ )
914
+ injected = "\n\n".join(item for item in additions if item)
915
+ return self.inject_system_message(prepared, injected)
916
+
917
+ def act(self, conversation: list[Any]) -> tuple[str, list[dict[str, Any]], list[Any]]:
918
+ metadata = self._materialized.get("metadata", {})
919
+ clarification = (
920
+ metadata.get("clarification_request") if isinstance(metadata, Mapping) else None
921
+ )
922
+ if self._first_turn and isinstance(clarification, str) and clarification.strip():
923
+ self._first_turn = False
924
+ return clarification, [], [{"role": "assistant", "content": clarification}]
925
+ self._first_turn = False
926
+ if self._budget_stopped:
927
+ message = "I could not complete the task within the shared run budget."
928
+ return message, [], [{"role": "assistant", "content": message}]
929
+ try:
930
+ return super().act(conversation)
931
+ except _SharedBudgetStop:
932
+ self._budget_stopped = True
933
+ message = "I could not complete the task within the shared run budget."
934
+ return message, [], [{"role": "assistant", "content": message}]
935
+ except json.JSONDecodeError as error:
936
+ raise _EvaluatedAgentOutputInvalid(error) from error
937
+ except Exception as error:
938
+ if not _is_gpt_oss_harmony_parse_failure(
939
+ error,
940
+ model_name=self.client.deployment,
941
+ ):
942
+ raise
943
+ self._budget_api_calls += 1
944
+ raise _EvaluatedAgentOutputInvalid(error) from None
945
+
946
+ def agentinputlab_usage(self) -> Mapping[str, int]:
947
+ """Return counters collected at the actual model/tool boundaries."""
948
+
949
+ return {
950
+ "model_calls": self._budget_api_calls,
951
+ "tool_calls": self._budget_tool_calls,
952
+ "input_tokens": self._budget_input_tokens,
953
+ "output_tokens": self._budget_output_tokens,
954
+ }
955
+
956
+ return IntervenedStateBenchAgent
957
+
958
+
959
+ def factory() -> StateBenchProvider:
960
+ return StateBenchProvider()