agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,334 @@
1
+ """Shared, dependency-light contracts for official arena providers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import os
9
+ import re
10
+ from collections.abc import Mapping
11
+ from dataclasses import dataclass, field
12
+ from pathlib import Path
13
+ from typing import Any, Protocol, runtime_checkable
14
+
15
+ REPLAY_INVENTORY_PROTOCOL = "agentinputlab.official-arena-replay-inventory.v1"
16
+ EXECUTABLE_PAYLOAD_PROTOCOL = "agentinputlab.executable-release-payload.v1"
17
+ _SHA256 = re.compile(r"^[0-9a-f]{64}$")
18
+ SHARED_BUDGET_PATH_ENV = "AILAB_SHARED_BUDGET_OVERRIDE_PATH"
19
+ SHARED_BUDGET_SHA256_ENV = "AILAB_SHARED_BUDGET_OVERRIDE_SHA256"
20
+ _BUDGET_FIELDS = frozenset(
21
+ {
22
+ "profile_id",
23
+ "max_model_calls",
24
+ "max_retrieval_calls",
25
+ "max_tool_calls",
26
+ "max_output_tokens",
27
+ "max_context_tokens",
28
+ "max_output_tokens_per_call",
29
+ }
30
+ )
31
+ _STANDARD_SHARED_BUDGET = {
32
+ "max_model_calls": 8,
33
+ "max_retrieval_calls": 8,
34
+ "max_tool_calls": 8,
35
+ "max_output_tokens": 4096,
36
+ # ExecutionBudget.max_context_tokens is a cumulative input-token budget,
37
+ # not the per-request model context window. Eight calls may each carry a
38
+ # growing interactive history, so the frozen run-level cap is eight times
39
+ # the 32,768-token per-call context control.
40
+ "max_context_tokens": 262144,
41
+ "max_output_tokens_per_call": 512,
42
+ }
43
+
44
+
45
+ class OfficialArenaError(RuntimeError):
46
+ """Stable fail-closed error raised by an official runtime provider."""
47
+
48
+
49
+ @dataclass(frozen=True, slots=True)
50
+ class ProviderArmResult:
51
+ """One completed official episode plus scorer and usage evidence."""
52
+
53
+ metrics: Mapping[str, float]
54
+ trace: Mapping[str, Any]
55
+ usage: Mapping[str, int | float] = field(default_factory=dict)
56
+ runtime_instance_id: str = ""
57
+
58
+ def __post_init__(self) -> None:
59
+ if not isinstance(self.metrics, Mapping) or not self.metrics:
60
+ raise OfficialArenaError("official provider metrics must be non-empty")
61
+ for name, value in self.metrics.items():
62
+ if (
63
+ not isinstance(name, str)
64
+ or not name.strip()
65
+ or isinstance(value, bool)
66
+ or not isinstance(value, (int, float))
67
+ or not math.isfinite(float(value))
68
+ ):
69
+ raise OfficialArenaError("official provider metrics are invalid")
70
+ if not isinstance(self.trace, Mapping) or not self.trace:
71
+ raise OfficialArenaError("official provider trace must be non-empty")
72
+ if not isinstance(self.usage, Mapping):
73
+ raise OfficialArenaError("official provider usage must be an object")
74
+ if not isinstance(self.runtime_instance_id, str) or len(self.runtime_instance_id) != 64:
75
+ raise OfficialArenaError(
76
+ "official provider runtime_instance_id must be a SHA-256 digest"
77
+ )
78
+
79
+
80
+ @runtime_checkable
81
+ class OfficialArenaProvider(Protocol):
82
+ """Runtime-specific capture and continuation boundary."""
83
+
84
+ benchmark_id: str
85
+ source_revision: str
86
+ scorer_id: str
87
+ scorer_revision: str
88
+
89
+ def doctor(self, config: Mapping[str, Any]) -> tuple[str, ...]: ...
90
+
91
+ def capture(
92
+ self,
93
+ config: Mapping[str, Any],
94
+ *,
95
+ output_root: Path,
96
+ ) -> tuple[Mapping[str, Any], ...]: ...
97
+
98
+ def run_arm(
99
+ self,
100
+ config: Mapping[str, Any],
101
+ *,
102
+ replay_record: Mapping[str, Any],
103
+ arm_payload: Mapping[str, Any],
104
+ seed: int,
105
+ role: str,
106
+ work_dir: Path,
107
+ ) -> ProviderArmResult: ...
108
+
109
+
110
+ def canonical_json(value: Any) -> str:
111
+ return json.dumps(
112
+ value,
113
+ ensure_ascii=False,
114
+ sort_keys=True,
115
+ separators=(",", ":"),
116
+ allow_nan=False,
117
+ )
118
+
119
+
120
+ def content_hash(value: Any) -> str:
121
+ return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
122
+
123
+
124
+ def sha256_file(path: Path) -> str:
125
+ digest = hashlib.sha256()
126
+ with path.open("rb") as handle:
127
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
128
+ digest.update(block)
129
+ return digest.hexdigest()
130
+
131
+
132
+ def validate_shared_budget(value: Any) -> Mapping[str, int | str]:
133
+ """Validate one paper-wide resource budget shared by every arena arm."""
134
+
135
+ if not isinstance(value, Mapping) or set(value) != _BUDGET_FIELDS:
136
+ raise OfficialArenaError("shared budget override fields are invalid")
137
+ if not isinstance(value["profile_id"], str) or not value["profile_id"].strip():
138
+ raise OfficialArenaError("shared budget profile_id must be non-empty")
139
+ for name in _BUDGET_FIELDS - {"profile_id"}:
140
+ raw = value[name]
141
+ if isinstance(raw, bool) or not isinstance(raw, int) or raw < 1:
142
+ raise OfficialArenaError(f"shared budget {name} must be a positive integer")
143
+ if value["max_output_tokens_per_call"] > value["max_output_tokens"]:
144
+ raise OfficialArenaError(
145
+ "shared budget per-call output cap exceeds the cumulative output cap"
146
+ )
147
+ return value
148
+
149
+
150
+ def standard_shared_budget(multiplier: float = 1.0) -> dict[str, int | str]:
151
+ """Materialize the frozen paper control profile at one resource multiplier."""
152
+
153
+ if (
154
+ isinstance(multiplier, bool)
155
+ or not isinstance(multiplier, (int, float))
156
+ or not math.isfinite(float(multiplier))
157
+ or multiplier <= 0
158
+ ):
159
+ raise OfficialArenaError("shared budget multiplier must be finite and positive")
160
+ return {
161
+ "profile_id": f"paper-shared-budget-x{multiplier:g}",
162
+ **{
163
+ name: max(1, round(value * multiplier))
164
+ for name, value in _STANDARD_SHARED_BUDGET.items()
165
+ },
166
+ }
167
+
168
+
169
+ def resolve_shared_budget_config(config: Mapping[str, Any]) -> Mapping[str, Any]:
170
+ """Attach an authenticated process-level budget factor to a provider config.
171
+
172
+ The factor is read by every official provider before capture. Consequently
173
+ the exact same budget becomes part of the captured visible state, the
174
+ intervention input hash, and the fair-comparison dependency locks.
175
+ """
176
+
177
+ raw_path = os.environ.get(SHARED_BUDGET_PATH_ENV)
178
+ raw_sha256 = os.environ.get(SHARED_BUDGET_SHA256_ENV)
179
+ if not raw_path and not raw_sha256:
180
+ return config
181
+ if not raw_path or not raw_sha256 or _SHA256.fullmatch(raw_sha256) is None:
182
+ raise OfficialArenaError("shared budget environment binding is incomplete")
183
+ unresolved = Path(raw_path).expanduser()
184
+ path = unresolved.resolve()
185
+ if unresolved.is_symlink() or not path.is_file() or sha256_file(path) != raw_sha256:
186
+ raise OfficialArenaError("shared budget environment artifact differs")
187
+ try:
188
+ loaded = json.loads(path.read_text(encoding="utf-8"))
189
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
190
+ raise OfficialArenaError("shared budget artifact is invalid JSON") from exc
191
+ override = validate_shared_budget(loaded)
192
+ existing = config.get("budget_override")
193
+ if existing is not None and existing != override:
194
+ raise OfficialArenaError("explicit and environment budget overrides differ")
195
+ return {**config, "budget_override": dict(override)}
196
+
197
+
198
+ def visible_shared_budget(config: Mapping[str, Any], fallback: Mapping[str, Any]) -> dict[str, Any]:
199
+ """Return the exact budget exposed to intervention code at capture time."""
200
+
201
+ raw = config.get("budget_override")
202
+ if raw is None:
203
+ return dict(fallback)
204
+ override = validate_shared_budget(raw)
205
+ return {
206
+ "max_model_calls": override["max_model_calls"],
207
+ "max_retrieval_calls": override["max_retrieval_calls"],
208
+ "max_tool_calls": override["max_tool_calls"],
209
+ "max_output_tokens": override["max_output_tokens"],
210
+ "max_context_tokens": override["max_context_tokens"],
211
+ "max_cost_usd": None,
212
+ }
213
+
214
+
215
+ def require_materialized_budget(
216
+ state: Mapping[str, Any],
217
+ config: Mapping[str, Any],
218
+ ) -> Mapping[str, Any]:
219
+ """Fail closed if a materialized arm changed the shared resource budget."""
220
+
221
+ raw = config.get("budget_override")
222
+ budget = state.get("budget")
223
+ if not isinstance(budget, Mapping):
224
+ raise OfficialArenaError("materialized state has no execution budget")
225
+ if raw is None:
226
+ return budget
227
+ expected = visible_shared_budget(config, {})
228
+ if dict(budget) != expected:
229
+ raise OfficialArenaError("materialized arm changed the shared budget factor")
230
+ return budget
231
+
232
+
233
+ def validate_dependency_hashes(
234
+ value: Any,
235
+ *,
236
+ label: str,
237
+ ) -> Mapping[str, str]:
238
+ if (
239
+ not isinstance(value, Mapping)
240
+ or not value
241
+ or any(
242
+ not isinstance(name, str)
243
+ or not name
244
+ or not isinstance(digest, str)
245
+ or _SHA256.fullmatch(digest) is None
246
+ for name, digest in value.items()
247
+ )
248
+ ):
249
+ raise OfficialArenaError(f"{label} dependency locks are invalid")
250
+ return value
251
+
252
+
253
+ def materialized_state(arm_payload: Mapping[str, Any]) -> Mapping[str, Any]:
254
+ if (
255
+ not isinstance(arm_payload, Mapping)
256
+ or arm_payload.get("schema_version") != 1
257
+ or arm_payload.get("protocol") != EXECUTABLE_PAYLOAD_PROTOCOL
258
+ ):
259
+ raise OfficialArenaError(
260
+ "official arena requires an executable materialized release payload"
261
+ )
262
+ execution = arm_payload.get("intervention_execution")
263
+ if not isinstance(execution, Mapping):
264
+ raise OfficialArenaError("intervention_execution is missing")
265
+ state = execution.get("materialized_state")
266
+ if not isinstance(state, Mapping):
267
+ raise OfficialArenaError("materialized intervention state is missing")
268
+ if content_hash(state) != execution.get("output_state_hash"):
269
+ raise OfficialArenaError("materialized intervention state hash differs")
270
+ if execution.get("input_state_hash") != arm_payload.get("visible_state_hash"):
271
+ raise OfficialArenaError("intervention input state differs from the checkpoint")
272
+ return state
273
+
274
+
275
+ def intervention_usage(arm_payload: Mapping[str, Any]) -> Mapping[str, int | float]:
276
+ execution = arm_payload.get("intervention_execution")
277
+ if not isinstance(execution, Mapping) or not isinstance(execution.get("usage"), Mapping):
278
+ raise OfficialArenaError("intervention usage is missing")
279
+ return execution["usage"]
280
+
281
+
282
+ def merge_usage(*values: Mapping[str, Any]) -> dict[str, int | float]:
283
+ integer_fields = (
284
+ "model_calls",
285
+ "retrieval_calls",
286
+ "tool_calls",
287
+ "embedding_calls",
288
+ "unpriced_model_calls",
289
+ "input_tokens",
290
+ "output_tokens",
291
+ )
292
+ float_fields = (
293
+ "cpu_seconds",
294
+ "accelerator_seconds",
295
+ "cost_usd",
296
+ "compute_cost_usd",
297
+ )
298
+ result: dict[str, int | float] = {name: 0 for name in integer_fields}
299
+ result.update({name: 0.0 for name in float_fields})
300
+ for value in values:
301
+ if not isinstance(value, Mapping):
302
+ raise OfficialArenaError("usage contribution must be an object")
303
+ for name in integer_fields:
304
+ raw = value.get(name, 0)
305
+ if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0:
306
+ raise OfficialArenaError(f"usage field {name} must be non-negative integer")
307
+ result[name] += raw
308
+ for name in float_fields:
309
+ raw = value.get(name, 0.0)
310
+ if (
311
+ isinstance(raw, bool)
312
+ or not isinstance(raw, (int, float))
313
+ or not math.isfinite(float(raw))
314
+ or raw < 0
315
+ ):
316
+ raise OfficialArenaError(f"usage field {name} must be non-negative finite")
317
+ result[name] += float(raw)
318
+ return result
319
+
320
+
321
+ __all__ = [
322
+ "EXECUTABLE_PAYLOAD_PROTOCOL",
323
+ "REPLAY_INVENTORY_PROTOCOL",
324
+ "OfficialArenaError",
325
+ "OfficialArenaProvider",
326
+ "ProviderArmResult",
327
+ "canonical_json",
328
+ "content_hash",
329
+ "intervention_usage",
330
+ "materialized_state",
331
+ "merge_usage",
332
+ "sha256_file",
333
+ "validate_dependency_hashes",
334
+ ]
@@ -0,0 +1 @@
1
+ """Official arena provider implementations."""
@@ -0,0 +1,228 @@
1
+ """Optional native-message history view; no native state/tool object is rewritten."""
2
+ from __future__ import annotations
3
+
4
+ import copy
5
+ import json
6
+ from collections.abc import Mapping
7
+ from pathlib import Path
8
+
9
+ from ..protocol import OfficialArenaError
10
+
11
+
12
+ def validate_history_transform(hook):
13
+ if hook is not None and not all(callable(getattr(hook, name, None))
14
+ for name in ("transform", "fingerprint")):
15
+ raise OfficialArenaError("native history requires transform and fingerprint methods")
16
+
17
+
18
+ class NativeHistorySession:
19
+ """Separate helper usage from target usage, enforcing their shared caps."""
20
+
21
+ def __init__(self, hook, materialized, budget):
22
+ validate_history_transform(hook)
23
+ self.hook = hook
24
+ self.materialized = copy.deepcopy(dict(materialized))
25
+ self.budget = dict(budget)
26
+ self.usage = {}
27
+
28
+ def remaining(self, agent_usage):
29
+ fields = {"max_model_calls": "model_calls", "max_output_tokens": "output_tokens"}
30
+ return {cap: int(self.budget[cap]) - int(agent_usage.get(name, 0))
31
+ - int(self.usage.get(name, 0)) for cap, name in fields.items()}
32
+
33
+ def messages(self, native, *, encode, agent_usage):
34
+ """Map only system/first-user text; reuse retained native objects exactly."""
35
+ from agentinputlab.integrations.live_history import _balanced_tool_frames
36
+
37
+ raw = encode(native)
38
+ logical = self._transform(raw, agent_usage=agent_usage)
39
+ first_user = next((i for i, row in enumerate(raw) if row.get("role") == "user"), None)
40
+ if (first_user is None or first_user == 0 or raw[0].get("role") != "system"
41
+ or len(logical) <= first_user or logical[0].get("role") != "system"
42
+ or logical[first_user].get("role") != "user"):
43
+ raise OfficialArenaError("native history changed the system/first-user framing")
44
+ if list(logical[1:first_user]) != raw[1:first_user]:
45
+ raise OfficialArenaError("native history changed a pre-user native frame")
46
+ text_slots = {0, first_user}
47
+ result, cursor = [], 0
48
+ for i, row in enumerate(logical):
49
+ row = dict(row)
50
+ if i in text_slots:
51
+ original = raw[i]
52
+ if ({k: v for k, v in row.items() if k != "content"}
53
+ != {k: v for k, v in original.items() if k != "content"}
54
+ or not isinstance(row.get("content"), str)):
55
+ raise OfficialArenaError("native history changed non-text first-turn fields")
56
+ result.append(native[i].model_copy(update={"content": row["content"]}, deep=True))
57
+ cursor = i + 1
58
+ else:
59
+ index = next((j for j in range(cursor, len(raw)) if raw[j] == row), None)
60
+ if index is None:
61
+ raise OfficialArenaError("native history invented or reordered a retained frame")
62
+ result.append(copy.deepcopy(native[index]))
63
+ cursor = index + 1
64
+ if encode(result) != list(logical):
65
+ raise OfficialArenaError("native history message roundtrip changed wire fields")
66
+ if not _balanced_tool_frames(logical):
67
+ raise OfficialArenaError("native history orphaned tool frames")
68
+ return result
69
+
70
+ def _transform(self, raw, *, agent_usage):
71
+ """Shared measured helper budget; protocol-specific roundtrips stay separate."""
72
+ from agentinputlab.observability import external_usage_counters, normalize_external_usage
73
+
74
+ remaining = self.remaining(agent_usage)
75
+ answer = self.hook.transform(copy.deepcopy(raw), state=copy.deepcopy(self.materialized),
76
+ remaining_budget=remaining)
77
+ if not isinstance(answer, Mapping) or set(answer) != {"messages", "usage"}:
78
+ raise OfficialArenaError("native history response fields differ")
79
+ usage = answer["usage"]
80
+ if not isinstance(usage, Mapping) or not {"model_calls", "output_tokens"} <= usage.keys():
81
+ raise OfficialArenaError("native history requires actual call/output usage")
82
+ if usage["model_calls"] and not {"input_tokens", "retrieval_calls"} <= usage.keys():
83
+ raise OfficialArenaError("native history calls require complete actual usage")
84
+ usage = external_usage_counters(normalize_external_usage({
85
+ "model_calls": 0, "retrieval_calls": 0, "input_tokens": 0, "output_tokens": 0, **usage}))
86
+ for key, value in usage.items():
87
+ self.usage[key] = self.usage.get(key, 0) + value
88
+ if not usage["model_calls"] and (usage["input_tokens"] or usage["output_tokens"]):
89
+ raise OfficialArenaError("skipped native history cannot report model tokens")
90
+ if usage["tool_calls"] or usage["retrieval_calls"] or usage["embedding_calls"]:
91
+ raise OfficialArenaError("native history cannot execute tools or retrieval")
92
+ if usage["model_calls"] >= remaining["max_model_calls"] or usage["output_tokens"] >= remaining["max_output_tokens"]:
93
+ raise OfficialArenaError("native history failed to reserve downstream budget")
94
+ logical = answer["messages"]
95
+ if not isinstance(logical, (list, tuple)) or any(not isinstance(row, Mapping) for row in logical):
96
+ raise OfficialArenaError("native history messages are invalid")
97
+ return list(logical)
98
+
99
+ def combined_usage(self, target_usage):
100
+ result = dict(target_usage)
101
+ for key, value in self.usage.items():
102
+ result[key] = result.get(key, 0) + value
103
+ return result
104
+
105
+ def write_receipt(self, path: Path, *, target_usage):
106
+ value = dict(fingerprint=self.hook.fingerprint(), records=list(getattr(self.hook, "records", ())),
107
+ history_usage=dict(self.usage), target_usage=dict(target_usage),
108
+ agent_and_history_usage=self.combined_usage(target_usage),
109
+ original_shared_budget=self.budget,
110
+ native_environment_history_NOT_modified=True)
111
+ if hasattr(self, "responses_records"):
112
+ value["responses_view"] = dict(protocol="native.responses.user_boundary.v1",
113
+ records=list(self.responses_records))
114
+ if path.exists():
115
+ if json.loads(path.read_text(encoding="utf-8")) != value:
116
+ raise OfficialArenaError("native history receipt changed before finalization")
117
+ return
118
+ with path.open("x", encoding="utf-8") as handle:
119
+ json.dump(value, handle, sort_keys=True, indent=2, allow_nan=False)
120
+
121
+
122
+ class ResponsesHistorySession(NativeHistorySession):
123
+ """A reversible optimizer view, never a Chat-Completions wire conversion.
124
+
125
+ Real instructions and injected system items are immutable. Each complete
126
+ assistant/tool episode between actual user messages is one atomic text view
127
+ for ACON. Retained Items are deep-copied from the original native objects,
128
+ including reasoning, annotations, call IDs, arguments and tool results.
129
+ Invoke only at new user turns, not inside a function-call continuation.
130
+ """
131
+
132
+ def __init__(self, *args, **kwargs):
133
+ super().__init__(*args, **kwargs)
134
+ self.responses_records = []
135
+
136
+ def responses(self, native, *, instructions, agent_usage):
137
+ from agentinputlab.contracts.serialization import canonical_json, content_hash
138
+
139
+ if not isinstance(instructions, str) or not instructions:
140
+ raise OfficialArenaError("Responses history requires real text instructions")
141
+ if not isinstance(native, (list, tuple)):
142
+ raise OfficialArenaError("Responses history requires native input Items")
143
+ rows = []
144
+ for item in native:
145
+ row = dict(item) if isinstance(item, Mapping) else item.model_dump(mode="json")
146
+ if not isinstance(row, dict):
147
+ raise OfficialArenaError("Responses history Item is not serializable")
148
+ rows.append(row)
149
+ system_indices = [i for i, row in enumerate(rows) if row.get("role") in {"system", "developer"}]
150
+ policy = canonical_json(dict(instructions=instructions,
151
+ input_policy_items=[rows[i] for i in system_indices]))
152
+ raw = [dict(role="system", content=policy)]
153
+ groups = []
154
+ pending_group = []
155
+
156
+ def flush():
157
+ if not pending_group:
158
+ return
159
+ pending, seen = set(), set()
160
+ for i in pending_group:
161
+ row = rows[i]
162
+ if row.get("type") == "function_call":
163
+ call = row.get("call_id")
164
+ if not isinstance(call, str) or not call or call in seen:
165
+ raise OfficialArenaError("Responses history has duplicate or missing call_id")
166
+ pending.add(call)
167
+ seen.add(call)
168
+ elif row.get("type") == "function_call_output":
169
+ call = row.get("call_id")
170
+ if call not in pending:
171
+ raise OfficialArenaError("Responses history has an orphaned function result")
172
+ pending.remove(call)
173
+ if pending:
174
+ raise OfficialArenaError("Responses history has an incomplete function episode")
175
+ raw.append(dict(role="assistant", content=canonical_json([rows[i] for i in pending_group])))
176
+ groups.append(list(pending_group))
177
+ pending_group.clear()
178
+
179
+ for i, row in enumerate(rows):
180
+ if i in system_indices:
181
+ continue # Preserved on the native wire and included in the policy view.
182
+ if row.get("role") == "user":
183
+ flush()
184
+ content = row.get("content")
185
+ if isinstance(content, list) and len(content) == 1 and isinstance(content[0], Mapping):
186
+ if content[0].get("type") != "input_text":
187
+ raise OfficialArenaError("Responses history user Item is not plain text")
188
+ content = content[0].get("text")
189
+ if not isinstance(content, str):
190
+ raise OfficialArenaError("Responses history user Item is not plain text")
191
+ raw.append(dict(role="user", content=content))
192
+ groups.append([i])
193
+ else:
194
+ pending_group.append(i)
195
+ flush()
196
+ if len(raw) < 2 or raw[1]["role"] != "user" or raw[-1]["role"] != "user":
197
+ raise OfficialArenaError("Responses history requires an actual new user-turn boundary")
198
+ logical = self._transform(raw, agent_usage=agent_usage)
199
+ if (len(logical) < 2 or logical[0] != raw[0]
200
+ or set(logical[1]) != {"role", "content"}
201
+ or logical[1].get("role") != "user" or not isinstance(logical[1].get("content"), str)):
202
+ raise OfficialArenaError("Responses history changed immutable policy or first-user framing")
203
+ selected = set(system_indices + groups[0])
204
+ cursor = 2
205
+ for row in logical[2:]:
206
+ index = next((j for j in range(cursor, len(raw)) if raw[j] == row), None)
207
+ if index is None:
208
+ raise OfficialArenaError("Responses history invented or reordered native Items")
209
+ selected.update(groups[index - 1])
210
+ cursor = index + 1
211
+ # The current real user request must remain; no model-generated turn is inserted.
212
+ if not set(groups[-1]) <= selected:
213
+ raise OfficialArenaError("Responses history removed the current user turn")
214
+ result = [copy.deepcopy(item) for i, item in enumerate(native) if i in selected]
215
+ first_index = groups[0][0]
216
+ first_output = sorted(selected).index(first_index)
217
+ original = rows[first_index]
218
+ content = logical[1]["content"]
219
+ if isinstance(original["content"], list):
220
+ content = [{**original["content"][0], "text": content}]
221
+ if isinstance(result[first_output], Mapping):
222
+ result[first_output] = {**result[first_output], "content": content}
223
+ else:
224
+ result[first_output] = result[first_output].model_copy(update={"content": content}, deep=True)
225
+ self.responses_records.append(dict(native_input_hash=content_hash(rows),
226
+ optimizer_view_hash=content_hash(raw), retained_source_item_indices=sorted(selected),
227
+ policy_unchanged=True, native_items_NOT_chat_messages=True))
228
+ return result, instructions
@@ -0,0 +1,77 @@
1
+ """Narrow message-shape compatibility helpers for self-hosted model templates."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping, Sequence
6
+ from typing import Any
7
+
8
+
9
+ def requires_qwen38_initial_system_message(model: Any) -> bool:
10
+ """Return whether the served alias uses Qwen3.8's strict system placement."""
11
+
12
+ if not isinstance(model, str):
13
+ return False
14
+ alias = model.strip().casefold().replace("_", "-")
15
+ return alias == "qwen3.8" or alias.startswith("qwen3.8-")
16
+
17
+
18
+ def normalize_qwen38_reasoning_effort(model: Any, effort: Any) -> Any:
19
+ """Map STATE-Bench's strongest effort label to Qwen3.8's equivalent."""
20
+
21
+ if requires_qwen38_initial_system_message(model) and effort == "high":
22
+ return "xhigh"
23
+ return effort
24
+
25
+
26
+ def coalesce_chat_system_messages(messages: Sequence[Any]) -> list[Any]:
27
+ """Move chat system content into one leading message without reordering peers."""
28
+
29
+ copied = [dict(item) if isinstance(item, Mapping) else item for item in messages]
30
+ system_items: list[dict[str, Any]] = []
31
+ non_system_items: list[Any] = []
32
+ for item in copied:
33
+ if isinstance(item, Mapping) and item.get("role") == "system":
34
+ content = item.get("content")
35
+ if not isinstance(content, str):
36
+ raise TypeError("Qwen3.8 system message content must be text")
37
+ system_items.append(dict(item))
38
+ else:
39
+ non_system_items.append(item)
40
+ if not system_items:
41
+ return copied
42
+ merged = dict(system_items[0])
43
+ merged["content"] = "\n\n".join(item["content"] for item in system_items)
44
+ return [merged, *non_system_items]
45
+
46
+
47
+ def fold_responses_system_messages_into_instructions(
48
+ input_items: Sequence[Any],
49
+ instructions: Any,
50
+ ) -> tuple[list[Any], Any]:
51
+ """Fold Responses input system items into its single instructions field.
52
+
53
+ Qwen3.8's chat template accepts only the leading system slot. The Responses
54
+ transport renders ``instructions`` there, so later system-role input items
55
+ must be removed and appended to that field in their original order.
56
+ """
57
+
58
+ copied = [dict(item) if isinstance(item, Mapping) else item for item in input_items]
59
+ system_contents: list[str] = []
60
+ non_system_items: list[Any] = []
61
+ for item in copied:
62
+ if isinstance(item, Mapping) and item.get("role") == "system":
63
+ content = item.get("content")
64
+ if not isinstance(content, str):
65
+ raise TypeError("Qwen3.8 Responses system item content must be text")
66
+ system_contents.append(content)
67
+ else:
68
+ non_system_items.append(item)
69
+ if not system_contents:
70
+ return copied, instructions
71
+ if instructions is not None and not isinstance(instructions, str):
72
+ raise TypeError("Qwen3.8 Responses instructions must be text or None")
73
+ parts = []
74
+ if instructions:
75
+ parts.append(instructions)
76
+ parts.extend(system_contents)
77
+ return non_system_items, "\n\n".join(parts)