agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,1672 @@
1
+ """Production provider for tau3 Banking Knowledge at the pinned revision."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import copy
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import subprocess
10
+ from collections.abc import Mapping
11
+ from pathlib import Path
12
+ from typing import Any
13
+ from urllib.parse import urlparse
14
+
15
+ from ..protocol import (
16
+ OfficialArenaError,
17
+ ProviderArmResult,
18
+ content_hash,
19
+ intervention_usage,
20
+ materialized_state,
21
+ require_materialized_budget,
22
+ resolve_shared_budget_config,
23
+ validate_dependency_hashes,
24
+ visible_shared_budget,
25
+ )
26
+ from ._live_history import NativeHistorySession, validate_history_transform
27
+
28
+ _SHARED_BUDGET_STOP_MESSAGE = "I could not complete the task within the shared run budget."
29
+
30
+
31
+ def _is_replayable_initial_user_message(message: Any) -> bool:
32
+ """Return whether a captured user turn is safe to replay as a checkpoint.
33
+
34
+ tau3 user messages may contain a user-side tool call together with visible
35
+ text. Stopping capture on that message leaves a dangling call without the
36
+ following ToolMessage, and environment replay then fails before the agent
37
+ can run. Let the orchestrator finish that tool exchange and checkpoint the
38
+ next ordinary visible user turn instead.
39
+ """
40
+
41
+ content = getattr(message, "content", None)
42
+ if not content:
43
+ return False
44
+ is_tool_call = getattr(message, "is_tool_call", None)
45
+ return not (callable(is_tool_call) and is_tool_call())
46
+
47
+
48
+ def _remaining_agent_budget(
49
+ budget: Mapping[str, Any],
50
+ optimizer_usage: Mapping[str, int | float],
51
+ ) -> dict[str, Any]:
52
+ """Subtract optimizer consumption from every cumulative agent-side cap."""
53
+
54
+ remaining = dict(budget)
55
+ fields = {
56
+ "max_model_calls": "model_calls",
57
+ "max_retrieval_calls": "retrieval_calls",
58
+ "max_tool_calls": "tool_calls",
59
+ "max_output_tokens": "output_tokens",
60
+ "max_context_tokens": "input_tokens",
61
+ }
62
+ for budget_name, usage_name in fields.items():
63
+ cap = budget.get(budget_name)
64
+ if cap is None:
65
+ remaining[budget_name] = None
66
+ continue
67
+ remaining[budget_name] = max(
68
+ 0,
69
+ int(cap) - int(optimizer_usage.get(usage_name, 0)),
70
+ )
71
+ return remaining
72
+
73
+
74
+ def _apply_tool_call_budget(
75
+ response: Any,
76
+ *,
77
+ remaining_tool_calls: int,
78
+ ) -> tuple[Any, int, int]:
79
+ """Truncate generated tool calls before tau3 can execute beyond the cap.
80
+
81
+ The returned counters are the number of calls retained for execution and
82
+ the number discarded by the hard budget boundary. Generated text-only
83
+ answers remain legal when the tool budget is exhausted.
84
+ """
85
+
86
+ if isinstance(remaining_tool_calls, bool) or remaining_tool_calls < 0:
87
+ raise OfficialArenaError("tau3 remaining tool-call budget is invalid")
88
+ calls = getattr(response, "tool_calls", None)
89
+ if calls is None:
90
+ return response, 0, 0
91
+ if not isinstance(calls, (list, tuple)):
92
+ raise OfficialArenaError("tau3 generated tool_calls are invalid")
93
+ generated = len(calls)
94
+ kept = min(generated, remaining_tool_calls)
95
+ discarded = generated - kept
96
+ if discarded == 0:
97
+ return response, kept, 0
98
+ copier = getattr(response, "model_copy", None)
99
+ if not callable(copier):
100
+ raise OfficialArenaError("tau3 assistant response cannot be budget-truncated")
101
+ if kept:
102
+ response = copier(update={"tool_calls": list(calls[:kept])}, deep=True)
103
+ else:
104
+ response = copier(
105
+ update={
106
+ "tool_calls": None,
107
+ "content": _SHARED_BUDGET_STOP_MESSAGE,
108
+ },
109
+ deep=True,
110
+ )
111
+ return response, kept, discarded
112
+
113
+
114
+ def _budget_exhausted_result(
115
+ *,
116
+ replay_record: Mapping[str, Any],
117
+ role: str,
118
+ seed: int,
119
+ work_dir: Path,
120
+ optimizer_usage: Mapping[str, int | float],
121
+ remaining_budget: Mapping[str, int | float],
122
+ ) -> ProviderArmResult:
123
+ """Materialize a scored, auditable zero-action arm after optimizer exhaustion.
124
+
125
+ A shared resource cap is an experimental outcome, not an infrastructure
126
+ failure. In particular, the canonical eight-document HyDE intervention
127
+ consumes all eight x1 model calls before the downstream tau3 agent can
128
+ act. Preserve the paired baseline, report a hard stop, and let the common
129
+ observation normalizer mark the treatment contract invalid with valid
130
+ success zero.
131
+ """
132
+
133
+ trace = {
134
+ "provider": "tau3",
135
+ "source_revision": Tau3Provider.source_revision,
136
+ "task_id": replay_record["protected"]["task_id"],
137
+ "role": role,
138
+ "seed": seed,
139
+ "termination_reason": "shared_budget_exhausted_before_agent",
140
+ "messages": [],
141
+ "reward_info": None,
142
+ "generation_controls": {
143
+ "hard_stops": 1,
144
+ "optimizer_usage": dict(optimizer_usage),
145
+ "remaining_budget": dict(remaining_budget),
146
+ },
147
+ "score_provenance": "paper_shared_budget_zero_action",
148
+ }
149
+ work_dir.joinpath("budget-stop.json").write_text(
150
+ json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
151
+ encoding="utf-8",
152
+ )
153
+ return ProviderArmResult(
154
+ metrics={
155
+ "official_reward": 0.0,
156
+ "cost": 0.0,
157
+ "adapter_hard_stops": 1.0,
158
+ },
159
+ trace=trace,
160
+ usage={
161
+ "model_calls": 0,
162
+ "tool_calls": 0,
163
+ "input_tokens": 0,
164
+ "output_tokens": 0,
165
+ "cost_usd": 0.0,
166
+ "unpriced_model_calls": 0,
167
+ },
168
+ runtime_instance_id=content_hash(
169
+ {
170
+ "provider": "tau3",
171
+ "checkpoint_id": replay_record["checkpoint_id"],
172
+ "role": role,
173
+ "seed": seed,
174
+ "termination_reason": trace["termination_reason"],
175
+ "work_dir": str(work_dir),
176
+ }
177
+ ),
178
+ )
179
+
180
+
181
+ def _model_output_invalid_result(
182
+ *,
183
+ replay_record: Mapping[str, Any],
184
+ role: str,
185
+ seed: int,
186
+ work_dir: Path,
187
+ error: Exception,
188
+ messages: list[object],
189
+ termination_reason: str = "generated_tool_arguments_invalid",
190
+ failed_generation_scope: str = "evaluated_agent",
191
+ measured_usage: Mapping[str, int | float] | None = None,
192
+ ) -> ProviderArmResult:
193
+ """Score a recognized malformed generation as an arm outcome, not infra loss.
194
+
195
+ ``tau2`` raises before producing ``reward_info`` when either the evaluated
196
+ agent emits malformed tool arguments or the benchmark user simulator emits
197
+ an empty message. Both are real trajectory terminal states. The latter
198
+ must retain the evaluated agent's already-consumed usage while excluding
199
+ the benchmark user's failed generation from the paper-wide agent budget.
200
+ """
201
+
202
+ serialized_messages = []
203
+ for message in messages:
204
+ dump = getattr(message, "model_dump", None)
205
+ if callable(dump):
206
+ serialized_messages.append(dump(mode="json"))
207
+ trace = {
208
+ "provider": "tau3",
209
+ "source_revision": Tau3Provider.source_revision,
210
+ "task_id": replay_record["protected"]["task_id"],
211
+ "role": role,
212
+ "seed": seed,
213
+ "termination_reason": termination_reason,
214
+ "messages": serialized_messages,
215
+ "reward_info": None,
216
+ "generation_controls": {
217
+ "hard_stops": 1,
218
+ "failure_type": type(error).__name__,
219
+ "failed_generation_scope": failed_generation_scope,
220
+ "failed_generation_usage_unknown": True,
221
+ },
222
+ "score_provenance": "paper_model_output_invalid_zero_action",
223
+ }
224
+ work_dir.joinpath("model-output-invalid.json").write_text(
225
+ json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
226
+ encoding="utf-8",
227
+ )
228
+ usage = (
229
+ {
230
+ "model_calls": 1,
231
+ "tool_calls": 0,
232
+ "input_tokens": 0,
233
+ "output_tokens": 0,
234
+ "cost_usd": 0.0,
235
+ "unpriced_model_calls": 1,
236
+ }
237
+ if measured_usage is None
238
+ else dict(measured_usage)
239
+ )
240
+ return ProviderArmResult(
241
+ metrics={
242
+ "official_reward": 0.0,
243
+ "cost": 0.0,
244
+ "adapter_hard_stops": 1.0,
245
+ },
246
+ trace=trace,
247
+ usage=usage,
248
+ runtime_instance_id=content_hash(
249
+ {
250
+ "provider": "tau3",
251
+ "checkpoint_id": replay_record["checkpoint_id"],
252
+ "role": role,
253
+ "seed": seed,
254
+ "termination_reason": trace["termination_reason"],
255
+ "work_dir": str(work_dir),
256
+ }
257
+ ),
258
+ )
259
+
260
+
261
+ def _is_generated_tool_argument_validation(error: Exception) -> bool:
262
+ errors = getattr(error, "errors", None)
263
+ if not callable(errors):
264
+ return False
265
+ return any(
266
+ item.get("type") == "dict_type" and tuple(item.get("loc", ()))[-1:] == ("arguments",)
267
+ for item in errors()
268
+ )
269
+
270
+
271
+ def _is_generated_user_message_validation(error: Exception) -> bool:
272
+ """Recognize only tau2's explicit empty generated-user-message failure."""
273
+
274
+ return isinstance(error, ValueError) and str(error).startswith(
275
+ "UserMessage must have either content or tool_calls. Got UserMessage"
276
+ )
277
+
278
+
279
+ def _downstream_agent_usage(
280
+ messages: list[object],
281
+ *,
282
+ initial_history_length: int,
283
+ assistant_message_type: type,
284
+ measured_model_calls: int | None = None,
285
+ measured_input_tokens: int | None = None,
286
+ measured_output_tokens: int | None = None,
287
+ ) -> dict[str, int]:
288
+ """Measure only calls made by the evaluated agent after the replay checkpoint.
289
+
290
+ The tau3 user simulator is part of the benchmark environment. Its generated
291
+ ``UserMessage`` objects may also carry provider usage, but charging those
292
+ tokens to the evaluated arm can falsely exceed the paper-wide agent budget.
293
+ """
294
+
295
+ input_tokens = 0
296
+ output_tokens = 0
297
+ model_calls = 0
298
+ tool_calls = 0
299
+ for message in messages[initial_history_length:]:
300
+ if not isinstance(message, assistant_message_type):
301
+ continue
302
+ model_calls += 1
303
+ usage = getattr(message, "usage", None)
304
+ if isinstance(usage, Mapping):
305
+ input_tokens += int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0)
306
+ output_tokens += int(usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0)
307
+ calls = getattr(message, "tool_calls", None)
308
+ if calls:
309
+ tool_calls += len(calls)
310
+ measured = {
311
+ "model_calls": model_calls if measured_model_calls is None else measured_model_calls,
312
+ "tool_calls": tool_calls,
313
+ "input_tokens": input_tokens if measured_input_tokens is None else measured_input_tokens,
314
+ "output_tokens": (
315
+ output_tokens if measured_output_tokens is None else measured_output_tokens
316
+ ),
317
+ }
318
+ if any(
319
+ isinstance(value, bool) or not isinstance(value, int) or value < 0
320
+ for value in measured.values()
321
+ ):
322
+ raise OfficialArenaError("tau3 evaluated-agent usage counters are invalid")
323
+ return measured
324
+
325
+
326
+ def _orchestrator_step_cap(
327
+ *,
328
+ remaining_budget: Mapping[str, int | float],
329
+ materialized: Mapping[str, Any],
330
+ ) -> int:
331
+ """Translate the evaluated-agent call budget into half-duplex steps.
332
+
333
+ tau3 counts each USER/AGENT/ENV transition as an orchestrator step, whereas
334
+ the paper budget counts only generations made by the evaluated agent. A
335
+ normal agent turn therefore needs two orchestrator steps. A materialized
336
+ clarification contributes one deterministic, zero-generation agent turn
337
+ before the remaining model calls.
338
+ """
339
+
340
+ model_calls = int(remaining_budget["max_model_calls"])
341
+ metadata = materialized.get("metadata", {})
342
+ clarification = metadata.get("clarification_request") if isinstance(metadata, Mapping) else None
343
+ deterministic_agent_turns = int(isinstance(clarification, str) and bool(clarification.strip()))
344
+ return 2 * (model_calls + deterministic_agent_turns)
345
+
346
+
347
+ def _local_tau_model_ids(config: Mapping[str, Any]) -> frozenset[str]:
348
+ """Return model ids whose tau3 endpoint is explicitly loopback-local."""
349
+
350
+ local: set[str] = set()
351
+ for role in ("agent", "user"):
352
+ args = config.get(f"llm_args_{role}")
353
+ model = config.get(f"llm_{role}")
354
+ if not isinstance(args, Mapping) or not isinstance(model, str):
355
+ continue
356
+ raw_url = args.get("api_base") or args.get("base_url")
357
+ if not isinstance(raw_url, str):
358
+ continue
359
+ hostname = urlparse(raw_url).hostname
360
+ if hostname not in {"127.0.0.1", "localhost", "::1"}:
361
+ continue
362
+ local.add(model.rsplit("/", 1)[-1])
363
+ return frozenset(local)
364
+
365
+
366
+ def _install_local_tau_cost_accounting(config: Mapping[str, Any]) -> frozenset[str]:
367
+ """Keep local unpriced models out of LiteLLM's remote price lookup path."""
368
+
369
+ local_models = _local_tau_model_ids(config)
370
+ if not local_models:
371
+ return local_models
372
+ from tau2.utils import llm_utils
373
+
374
+ current = llm_utils.get_response_cost
375
+ registered = getattr(current, "_agentinputlab_local_models", None)
376
+ if isinstance(registered, set):
377
+ registered.update(local_models)
378
+ return local_models
379
+ original = current
380
+ registered = set(local_models)
381
+
382
+ def local_aware_cost(response: Any) -> float:
383
+ model = str(getattr(response, "model", "")).rsplit("/", 1)[-1]
384
+ if model in registered:
385
+ return 0.0
386
+ return float(original(response))
387
+
388
+ local_aware_cost._agentinputlab_local_models = registered # type: ignore[attr-defined]
389
+ llm_utils.get_response_cost = local_aware_cost
390
+ return local_models
391
+
392
+
393
+ def _local_tau_nl_evaluator_binding(config: Mapping[str, Any]) -> Mapping[str, Any] | None:
394
+ """Bind τ³'s NL-assertion judge to the declared local model endpoint.
395
+
396
+ The pinned τ³ scorer evaluates the final Banking Knowledge task with an
397
+ LLM-backed natural-language assertion. Upstream defaults that judge to a
398
+ closed GPT endpoint. A fully local paper profile must not silently leave
399
+ the declared runtime and contact that endpoint. For explicit loopback
400
+ profiles, use the same frozen model/load set and deterministic decoding
401
+ controls as the two simulated participants. External/API profiles retain
402
+ the upstream evaluator binding.
403
+ """
404
+
405
+ local_models = _local_tau_model_ids(config)
406
+ model = config.get("llm_agent")
407
+ raw_args = config.get("llm_args_agent")
408
+ if not isinstance(model, str) or not isinstance(raw_args, Mapping):
409
+ return None
410
+ if model.rsplit("/", 1)[-1] not in local_models:
411
+ return None
412
+ args = dict(raw_args)
413
+ args["temperature"] = 0.0
414
+ args["response_format"] = {"type": "json_object"}
415
+ return {"model": model, "arguments": args, "provenance": "declared_local_load_set"}
416
+
417
+
418
+ def _install_local_tau_nl_evaluator(config: Mapping[str, Any]) -> Mapping[str, Any] | None:
419
+ """Install the explicit local NL-assertion evaluator in the pinned scorer."""
420
+
421
+ binding = _local_tau_nl_evaluator_binding(config)
422
+ if binding is None:
423
+ return None
424
+ from tau2.evaluator import evaluator_nl_assertions
425
+
426
+ evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS = binding["model"]
427
+ evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS_ARGS = dict(binding["arguments"])
428
+ return binding
429
+
430
+
431
+ class Tau3Provider:
432
+ benchmark_id = "tau3.banking_knowledge"
433
+ source_revision = "cf71a8070269883e38a365ffa85f78f46844c1f4"
434
+ scorer_id = "tau3.official_v1.0.1"
435
+ scorer_revision = "32526f9c17ba3725cc8327d43783a2b5d2c48015dc72dbc6f7af4d25a95328a6"
436
+
437
+ _CONFIG_FIELDS = frozenset(
438
+ {
439
+ "task_set_name",
440
+ "task_split_name",
441
+ "limit",
442
+ "llm_agent",
443
+ "llm_user",
444
+ "llm_args_agent",
445
+ "llm_args_user",
446
+ "retrieval_config",
447
+ "retrieval_config_kwargs",
448
+ "max_steps",
449
+ "max_errors",
450
+ "timeout",
451
+ "locked_dependency_hashes",
452
+ }
453
+ )
454
+ _OPTIONAL_CONFIG_FIELDS = frozenset(
455
+ {
456
+ "retriever_override",
457
+ "budget_override",
458
+ "capture_max_attempts",
459
+ "capture_text_only_fallback",
460
+ "capture_task_ids",
461
+ }
462
+ )
463
+
464
+ def __init__(self, *, history_transform: Any = None) -> None:
465
+ validate_history_transform(history_transform)
466
+ self.history_transform = history_transform
467
+
468
+ @staticmethod
469
+ def _history_result(result, agent, work_dir):
470
+ session = getattr(agent, "_history_session", None)
471
+ if session is None:
472
+ return result
473
+ receipt = work_dir / "live-history.json"
474
+ session.write_receipt(receipt, target_usage=agent._history_target_usage())
475
+ metrics = dict(result.metrics)
476
+ if "cost" in metrics:
477
+ metrics["cost"] += float(session.usage.get("cost_usd", 0))
478
+ return ProviderArmResult(
479
+ metrics=metrics,
480
+ trace={**result.trace, "live_history_transform": session.hook.fingerprint(),
481
+ "live_history_receipt": {"path": receipt.name, "sha256": _file_sha256(receipt)},
482
+ "history_usage": dict(session.usage)},
483
+ usage=session.combined_usage(result.usage),
484
+ runtime_instance_id=content_hash({"native_runtime": result.runtime_instance_id,
485
+ "live_history_receipt_sha256": _file_sha256(receipt)}),
486
+ )
487
+
488
+ def doctor(self, config: Mapping[str, Any]) -> tuple[str, ...]:
489
+ try:
490
+ self._validate_config(config)
491
+ head = subprocess.check_output(
492
+ ["git", "rev-parse", "HEAD"],
493
+ text=True,
494
+ stderr=subprocess.DEVNULL,
495
+ ).strip()
496
+ if head != self.source_revision:
497
+ raise OfficialArenaError("tau3 checkout revision differs")
498
+ from tau2.agent.llm_agent import LLMAgent
499
+ from tau2.runner.build import build_orchestrator
500
+ from tau2.runner.helpers import get_tasks
501
+ from tau2.runner.simulation import run_simulation
502
+
503
+ del LLMAgent, build_orchestrator, get_tasks, run_simulation
504
+ except Exception as exc:
505
+ return (f"{type(exc).__name__}: {exc}",)
506
+ return ()
507
+
508
+ def capture(
509
+ self,
510
+ config: Mapping[str, Any],
511
+ *,
512
+ output_root: Path,
513
+ ) -> tuple[Mapping[str, Any], ...]:
514
+ del output_root
515
+ parsed = self._validate_config(config)
516
+ _install_local_tau_cost_accounting(parsed)
517
+ _install_local_tau_nl_evaluator(parsed)
518
+ _install_agentinputlab_retriever(parsed)
519
+ _install_tau_tool_argument_normalizer()
520
+ from tau2.data_model.message import AssistantMessage, UserMessage
521
+ from tau2.runner.build import build_orchestrator
522
+ from tau2.runner.helpers import get_tasks
523
+
524
+ run_config = self._run_config(parsed)
525
+ tasks = get_tasks(
526
+ parsed["task_set_name"],
527
+ task_split_name=parsed["task_split_name"],
528
+ num_tasks=parsed["limit"],
529
+ task_ids=parsed.get("capture_task_ids"),
530
+ )
531
+ rows: list[Mapping[str, Any]] = []
532
+ capture_attempts, text_only_fallback = _capture_attempt_policy(
533
+ parsed.get("capture_max_attempts", 1),
534
+ parsed.get("capture_text_only_fallback", False),
535
+ )
536
+ capture_policy = {
537
+ "policy_id": "tau3.single_capture.v1" if capture_attempts == 1 else "tau3.explicit_bounded_capture.v1",
538
+ "max_attempts": capture_attempts,
539
+ "text_only_fallback_allowed": text_only_fallback,
540
+ "usage_scope": "environment_preparation_not_evaluated_agent_budget",
541
+ "attempt_count_is_not_model_call_or_token_usage": True,
542
+ }
543
+ for index, task in enumerate(tasks):
544
+ orchestrator, message, capture_attempt, capture_seed = _capture_initial_visible_user(
545
+ build_orchestrator=build_orchestrator,
546
+ run_config=run_config,
547
+ task=task,
548
+ base_seed=index,
549
+ assistant_message_type=AssistantMessage,
550
+ user_message_type=UserMessage,
551
+ max_attempts=capture_attempts,
552
+ allow_text_only_fallback=text_only_fallback,
553
+ )
554
+ try:
555
+ initial_history = []
556
+ for item in orchestrator.get_messages():
557
+ normalized = item.model_copy(update={"timestamp": None, "turn_idx": None})
558
+ initial_history.append(normalized.model_dump(mode="json"))
559
+ tools = orchestrator.environment.get_tools()
560
+ tool_schemas = [tool.openai_schema for tool in tools]
561
+ tool_names = tuple(
562
+ getattr(tool, "name", None) or getattr(tool, "__name__", None) for tool in tools
563
+ )
564
+ if any(not isinstance(name, str) or not name for name in tool_names):
565
+ raise OfficialArenaError("tau3 tool inventory contains unnamed tools")
566
+ policy = orchestrator.environment.get_policy()
567
+ if (
568
+ not isinstance(tool_schemas, list)
569
+ or not tool_schemas
570
+ or any(not isinstance(item, Mapping) for item in tool_schemas)
571
+ ):
572
+ raise OfficialArenaError("tau3 tool schemas are unavailable")
573
+ visible_state = {
574
+ "query": message.content,
575
+ "task_id": task.id,
576
+ "instruction": policy,
577
+ "context": [
578
+ {
579
+ "text": policy,
580
+ "block_id": "official_policy",
581
+ "source": "tau3.environment.policy",
582
+ "score": None,
583
+ "metadata": {"domain": "banking_knowledge"},
584
+ },
585
+ {
586
+ "text": json.dumps(
587
+ tool_schemas,
588
+ default=str,
589
+ ensure_ascii=False,
590
+ sort_keys=True,
591
+ ),
592
+ "block_id": "official_tool_catalog",
593
+ "source": "tau3.environment.tools",
594
+ "score": None,
595
+ "metadata": {
596
+ "domain": "banking_knowledge",
597
+ "tool_defs_hash": content_hash(tool_schemas),
598
+ },
599
+ },
600
+ ],
601
+ "messages": [
602
+ {
603
+ "role": "user",
604
+ "content": message.content,
605
+ "name": None,
606
+ }
607
+ ],
608
+ "tool_names": list(tool_names),
609
+ "metadata": {
610
+ "benchmark_id": self.benchmark_id,
611
+ "checkpoint_phase": "after_initial_user_message",
612
+ "clarification_roundtrip_available": True,
613
+ "state_restore_available": True,
614
+ "retrieval_config": parsed["retrieval_config"],
615
+ "capture_attempt": capture_attempt,
616
+ "capture_seed": capture_seed,
617
+ "capture_policy": capture_policy,
618
+ },
619
+ "budget": visible_shared_budget(
620
+ parsed,
621
+ {
622
+ "max_model_calls": parsed["max_steps"],
623
+ "max_retrieval_calls": parsed["max_steps"],
624
+ "max_tool_calls": parsed["max_steps"],
625
+ "max_output_tokens": 65536,
626
+ "max_context_tokens": None,
627
+ "max_cost_usd": None,
628
+ },
629
+ ),
630
+ }
631
+ rows.append(
632
+ {
633
+ "checkpoint_id": str(task.id),
634
+ "group_key": str(task.id),
635
+ "visible_state": visible_state,
636
+ "eligible_families": [
637
+ "reformulate",
638
+ "decompose",
639
+ "acquire",
640
+ "clarify",
641
+ "curate",
642
+ ],
643
+ "protected": {
644
+ "task_id": str(task.id),
645
+ "initial_user_message": message.model_dump(mode="json"),
646
+ "initial_message_history": initial_history,
647
+ "capture_attempt": capture_attempt,
648
+ "capture_seed": capture_seed,
649
+ "capture_policy": capture_policy,
650
+ "locked_dependency_hashes": parsed["locked_dependency_hashes"],
651
+ },
652
+ }
653
+ )
654
+ finally:
655
+ orchestrator._cleanup()
656
+ if not rows:
657
+ raise OfficialArenaError("tau3 capture found no tasks")
658
+ return tuple(rows)
659
+
660
+ def run_arm(
661
+ self,
662
+ config: Mapping[str, Any],
663
+ *,
664
+ replay_record: Mapping[str, Any],
665
+ arm_payload: Mapping[str, Any],
666
+ seed: int,
667
+ role: str,
668
+ work_dir: Path,
669
+ ) -> ProviderArmResult:
670
+ if self.history_transform is not None and (work_dir / "live-history.json").exists():
671
+ raise OfficialArenaError("native history output must be a fresh episode")
672
+ parsed = self._validate_config(config)
673
+ local_unpriced_models = _install_local_tau_cost_accounting(parsed)
674
+ nl_evaluator_binding = _install_local_tau_nl_evaluator(parsed)
675
+ _install_agentinputlab_retriever(parsed)
676
+ _install_tau_tool_argument_normalizer()
677
+ state = materialized_state(arm_payload)
678
+ budget = require_materialized_budget(state, parsed)
679
+ optimizer_usage = intervention_usage(arm_payload)
680
+ remaining_budget = _remaining_agent_budget(budget, optimizer_usage)
681
+ if remaining_budget["max_model_calls"] < 1 or remaining_budget["max_output_tokens"] < 1:
682
+ return _budget_exhausted_result(
683
+ replay_record=replay_record,
684
+ role=role,
685
+ seed=seed,
686
+ work_dir=work_dir,
687
+ optimizer_usage=optimizer_usage,
688
+ remaining_budget=remaining_budget,
689
+ )
690
+ protected = replay_record["protected"]
691
+
692
+ from tau2.data_model.message import AssistantMessage, ToolMessage, UserMessage
693
+ from tau2.data_model.tasks import InitialState
694
+ from tau2.orchestrator.orchestrator import DEFAULT_FIRST_AGENT_MESSAGE
695
+ from tau2.runner.build import (
696
+ _build_env_kwargs,
697
+ build_environment,
698
+ build_user,
699
+ )
700
+ from tau2.runner.helpers import get_tasks
701
+ from tau2.runner.simulation import run_simulation
702
+
703
+ task_id = protected["task_id"]
704
+ tasks = get_tasks(
705
+ parsed["task_set_name"],
706
+ task_split_name=parsed["task_split_name"],
707
+ task_ids=[task_id],
708
+ )
709
+ if len(tasks) != 1:
710
+ raise OfficialArenaError("tau3 protected task identity is unavailable")
711
+ task = tasks[0]
712
+ raw_history = protected.get("initial_message_history")
713
+ if isinstance(raw_history, list) and raw_history:
714
+ message_types = {
715
+ "assistant": AssistantMessage,
716
+ "user": UserMessage,
717
+ "tool": ToolMessage,
718
+ }
719
+ try:
720
+ initial_history = [
721
+ message_types[str(item["role"])].model_validate(item) for item in raw_history
722
+ ]
723
+ except (KeyError, TypeError, ValueError) as exc:
724
+ raise OfficialArenaError(
725
+ "tau3 protected initial message history is invalid"
726
+ ) from exc
727
+ else:
728
+ initial_user = UserMessage.model_validate(protected["initial_user_message"])
729
+ initial_history = [copy.deepcopy(DEFAULT_FIRST_AGENT_MESSAGE), initial_user]
730
+ original_initial = task.initial_state
731
+ task = task.model_copy(
732
+ update={
733
+ "initial_state": InitialState(
734
+ initialization_data=(
735
+ original_initial.initialization_data
736
+ if original_initial is not None
737
+ else None
738
+ ),
739
+ initialization_actions=(
740
+ original_initial.initialization_actions
741
+ if original_initial is not None
742
+ else None
743
+ ),
744
+ message_history=copy.deepcopy(initial_history),
745
+ )
746
+ },
747
+ deep=True,
748
+ )
749
+ run_config = self._run_config(parsed, budget=remaining_budget)
750
+ env_kwargs = _build_env_kwargs(run_config, task)
751
+ environment = build_environment(
752
+ "banking_knowledge",
753
+ env_kwargs=env_kwargs,
754
+ )
755
+ agent = _intervened_tau_agent_class()(
756
+ tools=environment.get_tools(),
757
+ domain_policy=environment.get_policy(),
758
+ llm=parsed["llm_agent"],
759
+ llm_args=parsed["llm_args_agent"],
760
+ materialized=state,
761
+ budget=remaining_budget,
762
+ history_transform=self.history_transform,
763
+ )
764
+ user = build_user(
765
+ "user_simulator",
766
+ environment,
767
+ task,
768
+ llm=parsed["llm_user"],
769
+ llm_args=parsed["llm_args_user"],
770
+ )
771
+ from tau2.orchestrator.orchestrator import Orchestrator
772
+
773
+ orchestrator = Orchestrator(
774
+ domain="banking_knowledge",
775
+ agent=agent,
776
+ user=user,
777
+ environment=environment,
778
+ task=task,
779
+ max_steps=_orchestrator_step_cap(
780
+ remaining_budget=remaining_budget,
781
+ materialized=state,
782
+ ),
783
+ max_errors=parsed["max_errors"],
784
+ seed=seed,
785
+ simulation_id=f"agentinputlab-{role}-{task_id}-{seed}",
786
+ timeout=parsed["timeout"],
787
+ )
788
+ try:
789
+ simulation = run_simulation(
790
+ orchestrator,
791
+ env_kwargs=env_kwargs,
792
+ )
793
+ except json.JSONDecodeError as error:
794
+ result = _model_output_invalid_result(
795
+ replay_record=replay_record,
796
+ role=role,
797
+ seed=seed,
798
+ work_dir=work_dir,
799
+ error=error,
800
+ messages=orchestrator.get_messages(),
801
+ )
802
+ return self._history_result(result, agent, work_dir)
803
+ except ValueError as error:
804
+ if _is_generated_user_message_validation(error):
805
+ measured_usage = _downstream_agent_usage(
806
+ orchestrator.get_messages(),
807
+ initial_history_length=len(initial_history),
808
+ assistant_message_type=AssistantMessage,
809
+ measured_model_calls=agent._budget_model_calls,
810
+ measured_input_tokens=agent._budget_input_tokens,
811
+ measured_output_tokens=agent._budget_output_tokens,
812
+ )
813
+ result = _model_output_invalid_result(
814
+ replay_record=replay_record,
815
+ role=role,
816
+ seed=seed,
817
+ work_dir=work_dir,
818
+ error=error,
819
+ messages=orchestrator.get_messages(),
820
+ termination_reason="generated_user_message_invalid",
821
+ failed_generation_scope="benchmark_user_simulator",
822
+ measured_usage={
823
+ **measured_usage,
824
+ "cost_usd": 0.0,
825
+ "unpriced_model_calls": measured_usage["model_calls"],
826
+ },
827
+ )
828
+ return self._history_result(result, agent, work_dir)
829
+ if not _is_generated_tool_argument_validation(error):
830
+ raise
831
+ result = _model_output_invalid_result(
832
+ replay_record=replay_record,
833
+ role=role,
834
+ seed=seed,
835
+ work_dir=work_dir,
836
+ error=error,
837
+ messages=orchestrator.get_messages(),
838
+ )
839
+ return self._history_result(result, agent, work_dir)
840
+ finally:
841
+ if getattr(agent, "_history_session", None) is not None:
842
+ agent._history_session.write_receipt(work_dir / "live-history.json",
843
+ target_usage=agent._history_target_usage())
844
+ if simulation.reward_info is None:
845
+ raise OfficialArenaError("tau3 official evaluator did not return reward_info")
846
+
847
+ messages = simulation.get_messages()
848
+ # The replay history contains the fixed initial assistant greeting. It
849
+ # is not a model call made by this arm and must not consume the shared
850
+ # execution budget a second time. The user simulator is benchmark
851
+ # environment state, so its calls and tokens are deliberately excluded.
852
+ agent_usage = _downstream_agent_usage(
853
+ messages,
854
+ initial_history_length=len(initial_history),
855
+ assistant_message_type=AssistantMessage,
856
+ # The intervened agent owns the authoritative counters for real
857
+ # generations. A materialized clarification and the deterministic
858
+ # budget-stop response are AssistantMessages too, but neither is a
859
+ # model call and neither may consume the shared generation budget.
860
+ measured_model_calls=agent._budget_model_calls,
861
+ measured_input_tokens=agent._budget_input_tokens,
862
+ measured_output_tokens=agent._budget_output_tokens,
863
+ )
864
+ if agent_usage["tool_calls"] != agent._budget_tool_calls:
865
+ raise OfficialArenaError(
866
+ "tau3 executed tool-call trace differs from its enforced budget counter"
867
+ )
868
+ trace = {
869
+ "provider": "tau3",
870
+ "source_revision": self.source_revision,
871
+ "task_id": task_id,
872
+ "role": role,
873
+ "seed": seed,
874
+ "cost_accounting": (
875
+ "local_unpriced_zero"
876
+ if str(parsed["llm_agent"]).rsplit("/", 1)[-1] in local_unpriced_models
877
+ else "upstream_litellm"
878
+ ),
879
+ "nl_assertion_evaluator": (
880
+ {
881
+ "model": nl_evaluator_binding["model"],
882
+ "provenance": nl_evaluator_binding["provenance"],
883
+ }
884
+ if nl_evaluator_binding is not None
885
+ else {"model": "upstream_default", "provenance": "pinned_tau3"}
886
+ ),
887
+ "termination_reason": simulation.termination_reason.value,
888
+ "messages": [message.model_dump(mode="json") for message in messages],
889
+ "reward_info": simulation.reward_info.model_dump(mode="json"),
890
+ "generation_controls": {
891
+ "hard_stops": agent._budget_hard_stops,
892
+ "executed_tool_calls": agent._budget_tool_calls,
893
+ "discarded_tool_calls": agent._budget_discarded_tool_calls,
894
+ },
895
+ }
896
+ work_dir.joinpath("simulation.json").write_text(
897
+ json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
898
+ encoding="utf-8",
899
+ )
900
+ total_cost = float(simulation.agent_cost or 0.0) + float(simulation.user_cost or 0.0)
901
+ result = ProviderArmResult(
902
+ metrics={
903
+ "official_reward": float(simulation.reward_info.reward),
904
+ "cost": total_cost,
905
+ "adapter_hard_stops": float(agent._budget_hard_stops),
906
+ },
907
+ trace=trace,
908
+ usage={
909
+ **agent_usage,
910
+ "cost_usd": total_cost,
911
+ "unpriced_model_calls": (
912
+ agent_usage["model_calls"]
913
+ if (
914
+ str(parsed["llm_agent"]).rsplit("/", 1)[-1] in local_unpriced_models
915
+ or (agent_usage["input_tokens"] == 0 and agent_usage["output_tokens"] == 0)
916
+ )
917
+ else 0
918
+ ),
919
+ },
920
+ runtime_instance_id=content_hash(
921
+ {
922
+ "provider": "tau3",
923
+ "checkpoint_id": replay_record["checkpoint_id"],
924
+ "role": role,
925
+ "seed": seed,
926
+ "simulation_id": simulation.id,
927
+ "work_dir": str(work_dir),
928
+ }
929
+ ),
930
+ )
931
+ return self._history_result(result, agent, work_dir)
932
+
933
+ def _run_config(
934
+ self,
935
+ config: Mapping[str, Any],
936
+ *,
937
+ budget: Mapping[str, Any] | None = None,
938
+ ) -> Any:
939
+ from tau2.data_model.simulation import TextRunConfig
940
+
941
+ agent_args = dict(config["llm_args_agent"])
942
+ max_steps = int(config["max_steps"])
943
+ if budget is not None:
944
+ max_steps = min(max_steps, int(budget["max_model_calls"]))
945
+ override = config.get("budget_override")
946
+ declared_per_call = (
947
+ int(override["max_output_tokens_per_call"])
948
+ if isinstance(override, Mapping)
949
+ else max(1, int(budget["max_output_tokens"]) // max_steps)
950
+ )
951
+ per_call = int(
952
+ min(
953
+ declared_per_call,
954
+ max(1, int(budget["max_output_tokens"]) // max_steps),
955
+ )
956
+ )
957
+ agent_args["max_tokens"] = min(
958
+ int(agent_args.get("max_tokens", per_call)),
959
+ per_call,
960
+ int(budget["max_output_tokens"]),
961
+ )
962
+
963
+ return TextRunConfig(
964
+ domain="banking_knowledge",
965
+ agent="llm_agent",
966
+ llm_agent=config["llm_agent"],
967
+ llm_args_agent=agent_args,
968
+ user="user_simulator",
969
+ llm_user=config["llm_user"],
970
+ llm_args_user=dict(config["llm_args_user"]),
971
+ max_steps=max_steps,
972
+ max_errors=config["max_errors"],
973
+ timeout=config["timeout"],
974
+ retrieval_config=config["retrieval_config"],
975
+ retrieval_config_kwargs=dict(config["retrieval_config_kwargs"]),
976
+ )
977
+
978
+ def _validate_config(self, config: Mapping[str, Any]) -> Mapping[str, Any]:
979
+ config = resolve_shared_budget_config(resolve_retriever_config(config))
980
+ if not isinstance(config, Mapping) or not (
981
+ set(config) >= self._CONFIG_FIELDS
982
+ and set(config) <= self._CONFIG_FIELDS | self._OPTIONAL_CONFIG_FIELDS
983
+ ):
984
+ raise OfficialArenaError("tau3 provider config fields are invalid")
985
+ for name in (
986
+ "task_set_name",
987
+ "task_split_name",
988
+ "llm_agent",
989
+ "llm_user",
990
+ "retrieval_config",
991
+ ):
992
+ if not isinstance(config[name], str) or not config[name].strip():
993
+ raise OfficialArenaError(f"tau3 provider {name} is invalid")
994
+ for name in ("llm_args_agent", "llm_args_user", "retrieval_config_kwargs"):
995
+ if not isinstance(config[name], Mapping):
996
+ raise OfficialArenaError(f"tau3 provider {name} must be an object")
997
+ for name in ("limit", "max_steps", "max_errors"):
998
+ value = config[name]
999
+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:
1000
+ raise OfficialArenaError(f"tau3 provider {name} must be positive")
1001
+ timeout = config["timeout"]
1002
+ if timeout is not None and (
1003
+ isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or timeout <= 0
1004
+ ):
1005
+ raise OfficialArenaError("tau3 provider timeout is invalid")
1006
+ _capture_attempt_policy(
1007
+ config.get("capture_max_attempts", 1),
1008
+ config.get("capture_text_only_fallback", False),
1009
+ )
1010
+ if config.get("capture_text_only_fallback", False) and not _is_gpt_oss_model_name(config["llm_user"]):
1011
+ raise OfficialArenaError("tau3 text-only capture fallback requires GPT-OSS")
1012
+ if "capture_task_ids" in config:
1013
+ selected = config["capture_task_ids"]
1014
+ if (
1015
+ not isinstance(selected, list)
1016
+ or not selected
1017
+ or len(selected) > config["limit"]
1018
+ or any(not isinstance(value, str) or not value.strip() for value in selected)
1019
+ or len(set(selected)) != len(selected)
1020
+ ):
1021
+ raise OfficialArenaError("tau3 capture_task_ids is invalid")
1022
+ validate_dependency_hashes(
1023
+ config["locked_dependency_hashes"],
1024
+ label="tau3",
1025
+ )
1026
+ if "retriever_override" in config:
1027
+ _validate_retriever_override(config["retriever_override"])
1028
+ return config
1029
+
1030
+
1031
+ _RETRIEVER_OVERRIDE_FIELDS = frozenset(
1032
+ {
1033
+ "profile_id",
1034
+ "backend",
1035
+ "model_path",
1036
+ "model_revision",
1037
+ "model_artifact_hash",
1038
+ "query_prefix",
1039
+ "document_prefix",
1040
+ "expected_dimensions",
1041
+ "top_k",
1042
+ "device",
1043
+ "prompt_template",
1044
+ "prompt_sha256",
1045
+ }
1046
+ )
1047
+ _LOCAL_ENCODERS: dict[str, Any] = {}
1048
+ RETRIEVER_OVERRIDE_PATH_ENV = "AILAB_TAU3_RETRIEVER_OVERRIDE_PATH"
1049
+ RETRIEVER_OVERRIDE_SHA256_ENV = "AILAB_TAU3_RETRIEVER_OVERRIDE_SHA256"
1050
+ _EMPTY_ASSISTANT_RESPONSE = "[AgentInputLab: empty assistant response]"
1051
+ _CAPTURE_MAX_ATTEMPTS = 16
1052
+ _CAPTURE_MAX_STEPS = 32
1053
+ _CAPTURE_RETRY_SEED_STRIDE = 1_000_003
1054
+ _GPT_OSS_CAPTURE_FALLBACK_ATTEMPTS = 1
1055
+
1056
+
1057
+ def _capture_attempt_policy(max_attempts: int, allow_text_only_fallback: bool) -> tuple[int, bool]:
1058
+ """Validate explicit preparation policies before building a simulator."""
1059
+
1060
+ if type(allow_text_only_fallback) is not bool:
1061
+ raise OfficialArenaError("tau3 capture_text_only_fallback must be boolean")
1062
+ maximum = _CAPTURE_MAX_ATTEMPTS + (
1063
+ _GPT_OSS_CAPTURE_FALLBACK_ATTEMPTS if allow_text_only_fallback else 0
1064
+ )
1065
+ if type(max_attempts) is not int or not 1 <= max_attempts <= maximum:
1066
+ raise OfficialArenaError("tau3 capture_max_attempts is invalid")
1067
+ if allow_text_only_fallback and max_attempts != maximum:
1068
+ raise OfficialArenaError("tau3 capture_text_only_fallback requires explicit 17-attempt preparation")
1069
+ return max_attempts, allow_text_only_fallback
1070
+
1071
+
1072
+ def _capture_retry_seed(base_seed: int, attempt: int) -> int:
1073
+ """Derive a distinct deterministic seed for a simulator-only capture retry."""
1074
+
1075
+ if attempt < 0:
1076
+ raise ValueError("capture attempt must be non-negative")
1077
+ return base_seed + attempt * _CAPTURE_RETRY_SEED_STRIDE
1078
+
1079
+
1080
+ def _capture_initial_visible_user(
1081
+ *,
1082
+ build_orchestrator: Any,
1083
+ run_config: Any,
1084
+ task: Any,
1085
+ base_seed: int,
1086
+ assistant_message_type: type,
1087
+ user_message_type: type,
1088
+ max_attempts: int = 1,
1089
+ allow_text_only_fallback: bool = False,
1090
+ ) -> tuple[Any, Any, int, int]:
1091
+ """Capture a visible initial user turn; default to one simulator attempt.
1092
+
1093
+ The benchmark user simulator occasionally emits a long environment-tool
1094
+ preamble, or malformed tool-argument JSON, before the evaluated agent is
1095
+ allowed to run. Those are capture-infrastructure failures, not treatment
1096
+ outcomes. Additional preparation attempts require an explicit bound and
1097
+ retain deterministic attempt/seed metadata. Removing the simulator's tools
1098
+ is a separate explicit opt-in variant, never an automatic repair. Count
1099
+ all native user/agent/judge generations separately: an attempt counter is
1100
+ not token/cost telemetry, and missing usage is not free preparation.
1101
+ """
1102
+
1103
+ attempts, allow_text_only_fallback = _capture_attempt_policy(max_attempts, allow_text_only_fallback)
1104
+ last_failure: Exception | None = None
1105
+ for attempt in range(attempts):
1106
+ capture_seed = _capture_retry_seed(base_seed, attempt)
1107
+ orchestrator = build_orchestrator(
1108
+ run_config,
1109
+ task,
1110
+ seed=capture_seed,
1111
+ simulation_id=f"agentinputlab-capture-{task.id}-attempt-{attempt + 1}",
1112
+ )
1113
+ capture_user = getattr(orchestrator, "user", None)
1114
+ capture_model = str(getattr(capture_user, "llm", ""))
1115
+ if allow_text_only_fallback and (
1116
+ not _is_gpt_oss_model_name(capture_model) or not hasattr(capture_user, "tools")
1117
+ ):
1118
+ orchestrator._cleanup()
1119
+ raise OfficialArenaError("tau3 text-only capture fallback requires a GPT-OSS simulator with tools")
1120
+ text_only_fallback = allow_text_only_fallback and attempt >= _CAPTURE_MAX_ATTEMPTS
1121
+ if text_only_fallback:
1122
+ # GPT-OSS can deterministically loop on user-only environment tools
1123
+ # before emitting the first visible request. After the normal,
1124
+ # bounded capture window is exhausted, make one final simulator
1125
+ # call without those pre-agent tools. The user scenario remains
1126
+ # unchanged, but this is an explicit preparation variant, not the
1127
+ # default capture policy or an evaluated-agent retry.
1128
+ capture_user.tools = []
1129
+ try:
1130
+ orchestrator.initialize()
1131
+ for _capture_step in range(_CAPTURE_MAX_STEPS):
1132
+ orchestrator.step()
1133
+ candidate = orchestrator.message
1134
+ if isinstance(candidate, user_message_type) and _is_replayable_initial_user_message(
1135
+ candidate
1136
+ ):
1137
+ return orchestrator, candidate, attempt + 1, capture_seed
1138
+ if isinstance(candidate, assistant_message_type):
1139
+ raise OfficialArenaError(
1140
+ "tau3 agent executed before the first visible user message"
1141
+ )
1142
+ last_failure = OfficialArenaError(
1143
+ "tau3 capture did not produce an initial visible user message"
1144
+ )
1145
+ except json.JSONDecodeError as error:
1146
+ last_failure = error
1147
+ except Exception:
1148
+ orchestrator._cleanup()
1149
+ raise
1150
+ orchestrator._cleanup()
1151
+ raise OfficialArenaError(
1152
+ "tau3 capture did not produce an initial visible user message "
1153
+ f"after {attempts} simulator attempts"
1154
+ ) from last_failure
1155
+
1156
+
1157
+ def _is_gpt_oss_model_name(model_name: str) -> bool:
1158
+ return model_name.casefold().removeprefix("openai/") == "gpt-oss-20b"
1159
+
1160
+
1161
+ def _is_gpt_oss_generation_boundary_failure(error: Exception, *, model_name: str) -> bool:
1162
+ """Recognize model-output failures that should become scored hard stops."""
1163
+
1164
+ if not _is_gpt_oss_model_name(model_name):
1165
+ return False
1166
+ text = str(error).casefold()
1167
+ return (
1168
+ "unexpected tokens remaining in message header" in text
1169
+ or "unexpected token " in text
1170
+ or ("input length" in text and "exceeds model's maximum context length" in text)
1171
+ )
1172
+
1173
+
1174
+ def _file_sha256(path: Path) -> str:
1175
+ digest = hashlib.sha256()
1176
+ with path.open("rb") as handle:
1177
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
1178
+ digest.update(block)
1179
+ return digest.hexdigest()
1180
+
1181
+
1182
+ def _validate_retriever_override(value: Any) -> Mapping[str, Any]:
1183
+ if not isinstance(value, Mapping) or set(value) != _RETRIEVER_OVERRIDE_FIELDS:
1184
+ raise OfficialArenaError("tau3 retriever_override fields are invalid")
1185
+ if value["backend"] not in {"bm25", "sentence_transformers_dense"}:
1186
+ raise OfficialArenaError("tau3 retriever_override backend is invalid")
1187
+ for name in ("profile_id", "model_artifact_hash", "prompt_template", "prompt_sha256"):
1188
+ if not isinstance(value[name], str) or not value[name]:
1189
+ raise OfficialArenaError(f"tau3 retriever_override {name} is invalid")
1190
+ for name in ("model_artifact_hash", "prompt_sha256"):
1191
+ if len(value[name]) != 64 or any(
1192
+ character not in "0123456789abcdef" for character in value[name]
1193
+ ):
1194
+ raise OfficialArenaError(f"tau3 retriever_override {name} is not SHA-256")
1195
+ for name in ("top_k", "expected_dimensions"):
1196
+ if isinstance(value[name], bool) or not isinstance(value[name], int) or value[name] < 1:
1197
+ raise OfficialArenaError(f"tau3 retriever_override {name} must be positive")
1198
+ raw_prompt = Path(str(value["prompt_template"])).expanduser()
1199
+ prompt = raw_prompt.resolve()
1200
+ if (
1201
+ not prompt.is_file()
1202
+ or raw_prompt.is_symlink()
1203
+ or _file_sha256(prompt) != value["prompt_sha256"]
1204
+ ):
1205
+ raise OfficialArenaError("tau3 retriever_override prompt is absent or changed")
1206
+ if value["backend"] == "sentence_transformers_dense":
1207
+ model_path = Path(str(value["model_path"])).expanduser().resolve()
1208
+ if not model_path.is_dir() or not value["model_revision"] or not value["device"]:
1209
+ raise OfficialArenaError("tau3 dense retriever_override model binding is invalid")
1210
+ return value
1211
+
1212
+
1213
+ def resolve_retriever_config(config: Mapping[str, Any]) -> Mapping[str, Any]:
1214
+ """Resolve a hash-bound process-level τ³ retriever factor.
1215
+
1216
+ Every main-arena method runner uses this provider. The two-variable
1217
+ binding therefore lets RQ4 vary the shared downstream retrieval dependency
1218
+ without changing method logic or editing the pinned tau2 checkout.
1219
+ """
1220
+
1221
+ raw_path = os.environ.get(RETRIEVER_OVERRIDE_PATH_ENV)
1222
+ raw_sha256 = os.environ.get(RETRIEVER_OVERRIDE_SHA256_ENV)
1223
+ if not raw_path and not raw_sha256:
1224
+ return config
1225
+ if not raw_path or not raw_sha256:
1226
+ raise OfficialArenaError("tau3 retriever override environment binding is incomplete")
1227
+ unresolved = Path(raw_path).expanduser()
1228
+ path = unresolved.resolve()
1229
+ if unresolved.is_symlink() or not path.is_file() or _file_sha256(path) != raw_sha256:
1230
+ raise OfficialArenaError("tau3 retriever override environment artifact differs")
1231
+ try:
1232
+ loaded = json.loads(path.read_text(encoding="utf-8"))
1233
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
1234
+ raise OfficialArenaError("tau3 retriever override artifact is invalid JSON") from exc
1235
+ override = _validate_retriever_override(loaded)
1236
+ existing = config.get("retriever_override")
1237
+ if existing is not None and existing != override:
1238
+ raise OfficialArenaError("tau3 explicit and environment retriever overrides differ")
1239
+ variant_name = (
1240
+ "agentinputlab_rq4_"
1241
+ + str(override["profile_id"]).replace(".", "_")
1242
+ + f"_k{override['top_k']}"
1243
+ )
1244
+ return {
1245
+ **config,
1246
+ "retrieval_config": variant_name,
1247
+ "retrieval_config_kwargs": {"top_k": int(override["top_k"])},
1248
+ "retriever_override": override,
1249
+ }
1250
+
1251
+
1252
+ def _normalized_tool_arguments(value: object) -> str | None:
1253
+ """Return canonical object JSON for normal or double-encoded arguments."""
1254
+
1255
+ if not isinstance(value, str):
1256
+ return None
1257
+ candidate = value
1258
+ for _ in range(4):
1259
+ try:
1260
+ parsed = json.loads(candidate)
1261
+ except (TypeError, ValueError):
1262
+ return None
1263
+ if isinstance(parsed, Mapping):
1264
+ return json.dumps(parsed, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
1265
+ if isinstance(parsed, str):
1266
+ candidate = parsed
1267
+ continue
1268
+ return None
1269
+ return None
1270
+
1271
+
1272
+ def _normalized_tool_name(value: object, *, advertised_tool_names: set[str]) -> str | None:
1273
+ """Strip only a leaked Harmony channel suffix from an advertised tool name."""
1274
+
1275
+ if not isinstance(value, str) or "<|channel|>" not in value:
1276
+ return None
1277
+ candidate = value.split("<|channel|>", 1)[0]
1278
+ if candidate and candidate in advertised_tool_names:
1279
+ return candidate
1280
+ return None
1281
+
1282
+
1283
+ def _openai_tool_names(tools: object) -> set[str]:
1284
+ if isinstance(tools, (str, bytes)) or not isinstance(tools, (list, tuple)):
1285
+ return set()
1286
+ names: set[str] = set()
1287
+ for tool in tools:
1288
+ if not isinstance(tool, Mapping):
1289
+ continue
1290
+ function = tool.get("function")
1291
+ if not isinstance(function, Mapping):
1292
+ continue
1293
+ name = function.get("name")
1294
+ if isinstance(name, str) and name:
1295
+ names.add(name)
1296
+ return names
1297
+
1298
+
1299
+ def _normalize_empty_assistant_message(message: Any) -> bool:
1300
+ """Make an empty OpenAI assistant turn explicit for the pinned tau2 runtime.
1301
+
1302
+ Some local OpenAI-compatible models emit a successful completion whose
1303
+ assistant message contains neither text nor tool calls. The model event is
1304
+ valid at the transport layer, but tau2 rejects it before the official
1305
+ scorer can assign a task outcome. Preserve any refusal text and otherwise
1306
+ insert a deterministic, visibly synthetic sentinel so the trajectory is
1307
+ scored as produced instead of disappearing from the experiment matrix.
1308
+
1309
+ The function returns whether it changed the response. It deliberately
1310
+ does not alter non-empty content, tool calls, finish reasons, or usage.
1311
+ """
1312
+
1313
+ if message is None or getattr(message, "tool_calls", ()):
1314
+ return False
1315
+ content = getattr(message, "content", None)
1316
+ if content is not None and (not isinstance(content, str) or content.strip()):
1317
+ return False
1318
+ refusal = getattr(message, "refusal", None)
1319
+ message.content = (
1320
+ refusal if isinstance(refusal, str) and refusal.strip() else _EMPTY_ASSISTANT_RESPONSE
1321
+ )
1322
+ return True
1323
+
1324
+
1325
+ def _install_tau_tool_argument_normalizer() -> None:
1326
+ """Bridge vLLM/OpenAI tool arguments to tau2's dictionary contract.
1327
+
1328
+ OpenAI-compatible servers serialize function arguments as JSON text. A
1329
+ few valid generations quote that JSON one additional time. The pinned
1330
+ tau2 runtime decodes exactly once and then validates a dictionary, so the
1331
+ double-encoded form otherwise aborts an official trajectory. Normalize
1332
+ only this transport representation; tool name, values, and scorer state
1333
+ remain unchanged.
1334
+ """
1335
+
1336
+ import tau2.utils.llm_utils as llm_utils
1337
+
1338
+ completion = llm_utils.completion
1339
+ if getattr(completion, "_agentinputlab_tool_argument_normalizer", False):
1340
+ return
1341
+
1342
+ def normalized_completion(*args: Any, **kwargs: Any) -> Any:
1343
+ response = completion(*args, **kwargs)
1344
+ advertised_tool_names = _openai_tool_names(kwargs.get("tools"))
1345
+ choices = getattr(response, "choices", ()) or ()
1346
+ for choice in choices:
1347
+ message = getattr(choice, "message", None)
1348
+ _normalize_empty_assistant_message(message)
1349
+ for tool_call in getattr(message, "tool_calls", ()) or ():
1350
+ function = getattr(tool_call, "function", None)
1351
+ raw_name = getattr(function, "name", None)
1352
+ normalized_name = _normalized_tool_name(
1353
+ raw_name,
1354
+ advertised_tool_names=advertised_tool_names,
1355
+ )
1356
+ if normalized_name is not None:
1357
+ function.name = normalized_name
1358
+ raw = getattr(function, "arguments", None)
1359
+ normalized = _normalized_tool_arguments(raw)
1360
+ if normalized is not None:
1361
+ function.arguments = normalized
1362
+ return response
1363
+
1364
+ normalized_completion._agentinputlab_tool_argument_normalizer = True # type: ignore[attr-defined]
1365
+ llm_utils.completion = normalized_completion
1366
+
1367
+
1368
+ def _install_agentinputlab_retriever(config: Mapping[str, Any]) -> None:
1369
+ raw = config.get("retriever_override")
1370
+ if raw is None:
1371
+ return
1372
+ override = _validate_retriever_override(raw)
1373
+ from tau2.domains.banking_knowledge.retrieval import (
1374
+ RETRIEVAL_VARIANTS,
1375
+ PipelineSpec,
1376
+ RetrievalVariant,
1377
+ standard_prompt,
1378
+ )
1379
+
1380
+ variant_name = str(config["retrieval_config"])
1381
+ prompt = Path(str(override["prompt_template"])).expanduser().resolve()
1382
+ if override["backend"] == "bm25":
1383
+ spec = PipelineSpec(type="bm25", top_k=int(override["top_k"]))
1384
+ else:
1385
+ from tau2.knowledge.document_preprocessors.embedding_indexer import (
1386
+ EMBEDDER_REGISTRY as document_registry,
1387
+ )
1388
+ from tau2.knowledge.input_preprocessors.embedding_encoder import (
1389
+ EMBEDDER_REGISTRY as query_registry,
1390
+ )
1391
+
1392
+ registry_key = "agentinputlab_" + str(override["model_artifact_hash"])[:16]
1393
+
1394
+ def embedder_class(*, prefix: str) -> type[Any]:
1395
+ class LocalSentenceTransformerEmbedder:
1396
+ def __init__(self, model: str):
1397
+ if model != override["profile_id"]:
1398
+ raise ValueError("tau3 local retriever profile marker differs")
1399
+ self.query_instruction = prefix
1400
+ identity = str(override["model_artifact_hash"])
1401
+ encoder = _LOCAL_ENCODERS.get(identity)
1402
+ if encoder is None:
1403
+ from sentence_transformers import SentenceTransformer
1404
+
1405
+ encoder = SentenceTransformer(
1406
+ str(override["model_path"]),
1407
+ device=str(override["device"]),
1408
+ local_files_only=True,
1409
+ trust_remote_code=True,
1410
+ )
1411
+ _LOCAL_ENCODERS[identity] = encoder
1412
+ self._encoder = encoder
1413
+
1414
+ def embed(self, texts: list[str]) -> Any:
1415
+ values = [prefix + text for text in texts]
1416
+ vectors = self._encoder.encode(
1417
+ values,
1418
+ batch_size=min(16, max(1, len(values))),
1419
+ normalize_embeddings=True,
1420
+ show_progress_bar=False,
1421
+ convert_to_numpy=True,
1422
+ )
1423
+ if vectors.ndim != 2 or vectors.shape[1] != override["expected_dimensions"]:
1424
+ raise ValueError("tau3 local retriever returned unexpected dimensions")
1425
+ return vectors
1426
+
1427
+ def get_name(self) -> str:
1428
+ return str(override["profile_id"])
1429
+
1430
+ return LocalSentenceTransformerEmbedder
1431
+
1432
+ document_registry[registry_key] = embedder_class(prefix=str(override["document_prefix"]))
1433
+ query_registry[registry_key] = embedder_class(prefix=str(override["query_prefix"]))
1434
+ spec = PipelineSpec(
1435
+ type="embedding",
1436
+ embedder_type=registry_key,
1437
+ embedder_model=str(override["profile_id"]),
1438
+ top_k=int(override["top_k"]),
1439
+ )
1440
+ RETRIEVAL_VARIANTS[variant_name] = RetrievalVariant(
1441
+ name=variant_name,
1442
+ prompt_template=prompt,
1443
+ build_prompt=standard_prompt,
1444
+ kb_search=spec,
1445
+ supports_top_k=True,
1446
+ )
1447
+
1448
+
1449
+ def _intervened_tau_agent_class() -> type[Any]:
1450
+ from tau2.agent.llm_agent import LLMAgent, LLMAgentState
1451
+ from tau2.data_model.message import (
1452
+ AssistantMessage,
1453
+ MultiToolMessage,
1454
+ UserMessage,
1455
+ )
1456
+ from tau2.utils.llm_utils import generate
1457
+
1458
+ class IntervenedTauAgent(LLMAgent[LLMAgentState]):
1459
+ def __init__(
1460
+ self,
1461
+ *args: Any,
1462
+ materialized: Mapping[str, Any],
1463
+ budget: Mapping[str, Any],
1464
+ history_transform: Any = None,
1465
+ **kwargs: Any,
1466
+ ):
1467
+ super().__init__(*args, **kwargs)
1468
+ self._materialized = materialized
1469
+ self._first_turn = True
1470
+ self._budget = budget
1471
+ self._budget_stopped = False
1472
+ self._budget_model_calls = 0
1473
+ self._budget_tool_calls = 0
1474
+ self._budget_input_tokens = 0
1475
+ self._budget_output_tokens = 0
1476
+ self._budget_hard_stops = 0
1477
+ self._budget_discarded_tool_calls = 0
1478
+ self._history_session = (NativeHistorySession(history_transform, materialized, budget)
1479
+ if history_transform is not None else None)
1480
+ self._budget_max_context_tokens = (
1481
+ 2**63 - 1
1482
+ if budget.get("max_context_tokens") is None
1483
+ else int(budget["max_context_tokens"])
1484
+ )
1485
+
1486
+ def _history_target_usage(self):
1487
+ return dict(model_calls=self._budget_model_calls, input_tokens=self._budget_input_tokens,
1488
+ output_tokens=self._budget_output_tokens, tool_calls=self._budget_tool_calls)
1489
+
1490
+ @property
1491
+ def system_prompt(self) -> str:
1492
+ base = super().system_prompt
1493
+ additions = [self._materialized.get("instruction", "")]
1494
+ additions.extend(
1495
+ block.get("text", "")
1496
+ for block in self._materialized.get("context", [])
1497
+ if isinstance(block, Mapping)
1498
+ )
1499
+ visible = "\n\n".join(item for item in additions if item)
1500
+ return base + (
1501
+ "\n\n<optimizer_visible_input>\n" + visible + "\n</optimizer_visible_input>"
1502
+ if visible
1503
+ else ""
1504
+ )
1505
+
1506
+ def generate_next_message(
1507
+ self,
1508
+ message: Any,
1509
+ state: LLMAgentState,
1510
+ ) -> tuple[AssistantMessage, LLMAgentState]:
1511
+ metadata = self._materialized.get("metadata", {})
1512
+ clarification = (
1513
+ metadata.get("clarification_request") if isinstance(metadata, Mapping) else None
1514
+ )
1515
+ if self._first_turn and isinstance(clarification, str) and clarification.strip():
1516
+ self._first_turn = False
1517
+ if message is not None:
1518
+ state.messages.append(message)
1519
+ response = AssistantMessage(
1520
+ role="assistant",
1521
+ content=clarification,
1522
+ cost=0.0,
1523
+ )
1524
+ state.messages.append(response)
1525
+ return response, state
1526
+ if self._first_turn and isinstance(message, UserMessage):
1527
+ message = message.model_copy(
1528
+ update={"content": self._materialized["query"]},
1529
+ deep=True,
1530
+ )
1531
+ self._first_turn = False
1532
+ return super().generate_next_message(message, state)
1533
+
1534
+ def _generate_next_message(
1535
+ self,
1536
+ message: Any,
1537
+ state: LLMAgentState,
1538
+ ) -> AssistantMessage:
1539
+ if isinstance(message, MultiToolMessage):
1540
+ state.messages.extend(message.tool_messages)
1541
+ else:
1542
+ state.messages.append(message)
1543
+ if self._budget_stopped:
1544
+ return AssistantMessage(
1545
+ role="assistant",
1546
+ content=_SHARED_BUDGET_STOP_MESSAGE,
1547
+ cost=0.0,
1548
+ )
1549
+ messages = state.system_messages + state.messages
1550
+ if self._history_session is not None:
1551
+ remaining = self._history_session.remaining(self._history_target_usage())
1552
+ if remaining["max_model_calls"] < 1 or remaining["max_output_tokens"] < 1:
1553
+ self._budget_stopped = True
1554
+ self._budget_hard_stops += 1
1555
+ return AssistantMessage(role="assistant", content=_SHARED_BUDGET_STOP_MESSAGE, cost=0.0)
1556
+ from tau2.utils.llm_utils import to_litellm_messages
1557
+ messages = self._history_session.messages(messages, encode=to_litellm_messages,
1558
+ agent_usage=self._history_target_usage())
1559
+ try:
1560
+ from litellm import token_counter
1561
+
1562
+ normalized_messages = [
1563
+ item.model_dump(mode="json") if hasattr(item, "model_dump") else item
1564
+ for item in messages
1565
+ ]
1566
+ # tau2's internal tool-call objects are not OpenAI-shaped.
1567
+ # Count their canonical JSON as plain text so every argument,
1568
+ # result, system message, and tool schema remains included
1569
+ # without asking LiteLLM to reinterpret the structure.
1570
+ token_messages = [
1571
+ {
1572
+ "role": "user",
1573
+ "content": json.dumps(
1574
+ {
1575
+ "messages": normalized_messages,
1576
+ "tools": [tool.openai_schema for tool in self.tools],
1577
+ },
1578
+ ensure_ascii=False,
1579
+ sort_keys=True,
1580
+ ),
1581
+ },
1582
+ ]
1583
+ next_input_tokens = int(token_counter(model=self.llm, messages=token_messages))
1584
+ except Exception as exc:
1585
+ raise OfficialArenaError(
1586
+ "tau3 could not measure input tokens before generation"
1587
+ ) from exc
1588
+ history_usage = self._history_session.usage if self._history_session is not None else {}
1589
+ history_calls = int(history_usage.get("model_calls", 0))
1590
+ history_input = int(history_usage.get("input_tokens", 0))
1591
+ history_output = int(history_usage.get("output_tokens", 0))
1592
+ remaining_output = int(self._budget["max_output_tokens"]) - self._budget_output_tokens - history_output
1593
+ if (
1594
+ self._budget_model_calls + history_calls >= int(self._budget["max_model_calls"])
1595
+ or self._budget_input_tokens + history_input + next_input_tokens > self._budget_max_context_tokens
1596
+ or remaining_output < 1
1597
+ ):
1598
+ self._budget_stopped = True
1599
+ self._budget_hard_stops += 1
1600
+ return AssistantMessage(
1601
+ role="assistant",
1602
+ content=_SHARED_BUDGET_STOP_MESSAGE,
1603
+ cost=0.0,
1604
+ )
1605
+ call_args = dict(self.llm_args)
1606
+ call_args["max_tokens"] = min(
1607
+ int(call_args.get("max_tokens", remaining_output)),
1608
+ remaining_output,
1609
+ )
1610
+ try:
1611
+ response = generate(
1612
+ model=self.llm,
1613
+ tools=self.tools,
1614
+ messages=messages,
1615
+ call_name="agent_response",
1616
+ **call_args,
1617
+ )
1618
+ except Exception as error:
1619
+ if not _is_gpt_oss_generation_boundary_failure(
1620
+ error,
1621
+ model_name=self.llm,
1622
+ ):
1623
+ raise
1624
+ # A malformed Harmony header or a request beyond the serving
1625
+ # context window is an evaluated trajectory outcome, not a
1626
+ # missing experiment row. Preserve it as the same scored hard
1627
+ # stop used for an exhausted shared budget.
1628
+ self._budget_model_calls += 1
1629
+ self._budget_input_tokens += next_input_tokens
1630
+ self._budget_stopped = True
1631
+ self._budget_hard_stops += 1
1632
+ return AssistantMessage(
1633
+ role="assistant",
1634
+ content=_SHARED_BUDGET_STOP_MESSAGE,
1635
+ cost=0.0,
1636
+ )
1637
+ usage = getattr(response, "usage", None)
1638
+ if isinstance(usage, Mapping):
1639
+ actual_input = int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0)
1640
+ actual_output = int(
1641
+ usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0
1642
+ )
1643
+ else:
1644
+ actual_input = next_input_tokens
1645
+ actual_output = 0
1646
+ self._budget_model_calls += 1
1647
+ self._budget_input_tokens += actual_input
1648
+ self._budget_output_tokens += actual_output
1649
+ if (
1650
+ self._budget_input_tokens + history_input > self._budget_max_context_tokens
1651
+ or self._budget_output_tokens + history_output > int(self._budget["max_output_tokens"])
1652
+ ):
1653
+ raise OfficialArenaError("tau3 provider exceeded its measured shared budget")
1654
+ response, kept_tool_calls, discarded_tool_calls = _apply_tool_call_budget(
1655
+ response,
1656
+ remaining_tool_calls=max(
1657
+ 0,
1658
+ int(self._budget["max_tool_calls"]) - self._budget_tool_calls,
1659
+ ),
1660
+ )
1661
+ self._budget_tool_calls += kept_tool_calls
1662
+ if discarded_tool_calls:
1663
+ self._budget_stopped = True
1664
+ self._budget_hard_stops += 1
1665
+ self._budget_discarded_tool_calls += discarded_tool_calls
1666
+ return response
1667
+
1668
+ return IntervenedTauAgent
1669
+
1670
+
1671
+ def factory() -> Tau3Provider:
1672
+ return Tau3Provider()