agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,463 @@
1
+ """ACE feedback on frozen Tau tool plans, using native tools and reward logic.
2
+
3
+ Initial user messages are existing captures, not generated by this adapter.
4
+ Retail uses the official training split; Banking uses declared disjoint groups.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import copy
10
+ import inspect
11
+ import time
12
+ from collections.abc import Mapping, Sequence
13
+ from contextlib import nullcontext
14
+ from datetime import datetime, timezone
15
+ from pathlib import Path
16
+ from urllib.parse import urlsplit
17
+
18
+ from agentinputlab.api import _state
19
+ from agentinputlab.contracts import ExternalDependencyError, ValidationError
20
+ from agentinputlab.contracts.serialization import canonical_json, content_hash, to_jsonable
21
+
22
+ from .ace import ACEExample
23
+ from .ace_native_plan import (
24
+ NativePlanFeedback,
25
+ _save,
26
+ _sha,
27
+ native_process_scope,
28
+ source_file_pins,
29
+ verify_native_checkout,
30
+ )
31
+
32
+ SCHEMA = "agentinputlab.ace.Tau.native_feedback_input.v1"
33
+ PLAN_INSTRUCTION = (
34
+ "Solve the supplied task as one noninteractive tool plan. Preserve all visible task "
35
+ "rules and function schemas. Put exactly one JSON object in your final_answer string: "
36
+ '{"actions":[{"name":"visible_function_name","arguments":{}}],"response":"message to user"}. '
37
+ "Use only advertised tools with their required arguments. Actions execute in order without "
38
+ "another model call, access to tool replies while planning, or another user turn. The response "
39
+ "ends this declared plan. Do not output Python, code fences, placeholder variables or extra fields.\n"
40
+ )
41
+
42
+
43
+ def _provider(domain):
44
+ if domain == "retail":
45
+ from workers.official_arenas.providers.tau_retail import TauRetailProvider
46
+
47
+ return TauRetailProvider()
48
+ if domain == "banking_knowledge":
49
+ from workers.official_arenas.providers.tau3 import Tau3Provider
50
+
51
+ return Tau3Provider()
52
+ raise ValidationError("ACE Tau supports explicit retail or banking_knowledge domains")
53
+
54
+
55
+ def _same_backbone(configuration, target):
56
+ model = target.get("model_id") if isinstance(target, Mapping) else None
57
+ agent = configuration.get("llm_agent")
58
+ if (
59
+ not isinstance(model, str)
60
+ or not model
61
+ or not isinstance(agent, str)
62
+ or agent not in (model, "openai/" + model)
63
+ or configuration.get("llm_user") != agent
64
+ or configuration.get("llm_judge", agent) != agent
65
+ ):
66
+ raise ValidationError("ACE Tau user and judge must follow the declared target backbone")
67
+ seed = configuration.get("llm_args_agent", {}).get("seed")
68
+ if type(seed) is not int:
69
+ raise ValidationError("ACE Tau needs an explicit integer evaluated-agent seed")
70
+
71
+
72
+ def _native_inputs(checkout, domain, rows, provider):
73
+ from tau2.data_model.message import AssistantMessage, ToolMessage, UserMessage
74
+ from tau2.data_model.tasks import InitialState
75
+ from tau2.runner.build import _build_env_kwargs, build_environment
76
+ from tau2.runner.helpers import get_tasks, load_task_splits
77
+ from tau2.utils.utils import DATA_DIR
78
+
79
+ if Path(DATA_DIR).resolve() != checkout / "data":
80
+ raise ExternalDependencyError("ACE Tau data must belong to the verified native checkout")
81
+ split = load_task_splits(domain)
82
+ if domain == "retail":
83
+ if not isinstance(split, dict) or not {"train", "test"} <= split.keys():
84
+ raise ValidationError("Retail official task split is unavailable")
85
+ train = set(map(str, split["train"]))
86
+ if train & set(map(str, split["test"])):
87
+ raise ValidationError("Retail native training and test splits overlap")
88
+ elif split is not None:
89
+ raise ValidationError("Banking split semantics changed; re-declare the fitting protocol")
90
+ tasks, histories, schemas, kwargs, configs = {}, {}, {}, {}, {}
91
+ kinds = {"assistant": AssistantMessage, "tool": ToolMessage, "user": UserMessage}
92
+ for row in rows:
93
+ key = row["case_id"]
94
+ cfg = provider._validate_config(row["native_configuration"])
95
+ state = _state(row["state"])
96
+ protected = row["protected_checkpoint"]["protected"]
97
+ if (
98
+ cfg["task_set_name"] != domain
99
+ or cfg["task_split_name"] != "train"
100
+ or (domain == "retail" and key not in train)
101
+ or protected["task_id"] != key
102
+ or state.task_id != key
103
+ or protected["locked_dependency_hashes"] != cfg["locked_dependency_hashes"]
104
+ ):
105
+ raise ValidationError("Tau task, training origin or dependency binding differs")
106
+ matches = get_tasks(domain, task_split_name=cfg["task_split_name"], task_ids=[key])
107
+ if len(matches) != 1 or str(matches[0].id) != key:
108
+ raise ValidationError("Tau fitting task is missing from the native inventory")
109
+ task = matches[0]
110
+ history = [kinds[m["role"]].model_validate(m) for m in protected["initial_message_history"]]
111
+ if (
112
+ not history
113
+ or not isinstance(history[-1], UserMessage)
114
+ or history[-1].is_tool_call()
115
+ or state.query != history[-1].content
116
+ ):
117
+ raise ValidationError("Tau feedback requires its frozen initial visible user message")
118
+ initial = task.initial_state
119
+ task = task.model_copy(
120
+ update=dict(
121
+ initial_state=InitialState(
122
+ initialization_data=initial.initialization_data if initial else None,
123
+ initialization_actions=initial.initialization_actions if initial else None,
124
+ message_history=copy.deepcopy(history),
125
+ )
126
+ ),
127
+ deep=True,
128
+ )
129
+ env_args = _build_env_kwargs(
130
+ provider._run_config(cfg, budget=to_jsonable(state.budget)), task
131
+ )
132
+ environment = build_environment(domain, env_kwargs=env_args)
133
+ tool_schema = [to_jsonable(t.openai_schema) for t in environment.get_tools()]
134
+ names = {t.get("name") or t.get("function", {}).get("name") for t in tool_schema}
135
+ if (
136
+ not names
137
+ or names != set(state.tool_names)
138
+ or state.instruction != environment.get_policy()
139
+ ):
140
+ raise ValidationError("Tau visible native policy or tool inventory differs")
141
+ catalogs = [c for c in state.context if c.block_id == "official_tool_catalog"]
142
+ if len(catalogs) != 1:
143
+ raise ValidationError("Tau frozen visible state needs its native tool catalog")
144
+ import json
145
+
146
+ if to_jsonable(json.loads(catalogs[0].text)) != tool_schema:
147
+ raise ValidationError("Tau frozen function schemas differ from the native environment")
148
+ tasks[key], histories[key], schemas[key] = task, history, tool_schema
149
+ configs[key], kwargs[key] = cfg, env_args
150
+ files = list((checkout / "src/tau2").rglob("*.py"))
151
+ data = checkout / "data/tau2/domains" / domain
152
+ if domain == "retail":
153
+ files.extend(
154
+ data / name for name in ("db.json", "policy.md", "tasks.json", "split_tasks.json")
155
+ )
156
+ else:
157
+ files.append(data / "db.json")
158
+ files.extend(p for p in (data / "documents").rglob("*") if p.is_file())
159
+ files.extend(data / "tasks" / (key + ".json") for key in tasks)
160
+ return tasks, histories, schemas, kwargs, configs, source_file_pins(checkout, files)
161
+
162
+
163
+ def build_ace_tau_feedback_input(
164
+ *,
165
+ checkout,
166
+ domain: str,
167
+ captures: Sequence[Mapping],
168
+ native_configurations: Mapping,
169
+ train_ids: Sequence[str],
170
+ validation_ids: Sequence[str],
171
+ heldout_ids: Sequence[str],
172
+ partition_label: str,
173
+ target_model: Mapping,
174
+ output_path,
175
+ ):
176
+ """Prepare existing captures, without user/agent/judge model calls."""
177
+ provider = _provider(domain)
178
+ root = verify_native_checkout(checkout, revision=provider.source_revision)
179
+ ids = [*train_ids, *validation_ids]
180
+ if (
181
+ not train_ids
182
+ or not validation_ids
183
+ or any(not isinstance(k, str) or not k for k in ids)
184
+ or len(ids) != len(set(ids))
185
+ or set(ids) & set(heldout_ids)
186
+ or not isinstance(partition_label, str)
187
+ or not partition_label.strip()
188
+ or not isinstance(native_configurations, Mapping)
189
+ or set(ids) != set(native_configurations)
190
+ ):
191
+ raise ValidationError(
192
+ "Tau needs disjoint fitting groups, per-case configurations and a label"
193
+ )
194
+ selected = [c for c in captures if c["checkpoint_id"] in ids]
195
+ if len(selected) != len(ids) or {c["checkpoint_id"] for c in selected} != set(ids):
196
+ raise ValidationError("Tau capture IDs are missing or duplicated")
197
+ by_id = {c["checkpoint_id"]: to_jsonable(c) for c in selected}
198
+ rows = []
199
+ for role, group in (("train", train_ids), ("validation", validation_ids)):
200
+ for key in group:
201
+ cfg = to_jsonable(native_configurations[key])
202
+ _same_backbone(cfg, target_model)
203
+ state = _state(by_id[key]["visible_state"])
204
+ rows.append(
205
+ dict(
206
+ case_id=key,
207
+ role=role,
208
+ state=to_jsonable(state),
209
+ state_hash=state.state_hash,
210
+ protected_checkpoint=by_id[key],
211
+ native_configuration=cfg,
212
+ )
213
+ )
214
+ with native_process_scope(root, package="tau2"):
215
+ _, _, schemas, _, _, pins = _native_inputs(root, domain, rows, provider)
216
+ value = dict(
217
+ schema=SCHEMA,
218
+ benchmark=provider.benchmark_id,
219
+ domain=domain,
220
+ native_source_revision=provider.source_revision,
221
+ source_files=pins,
222
+ fitting_rows=rows,
223
+ heldout_ids=list(heldout_ids),
224
+ source_official_split="train" if domain == "retail" else None,
225
+ partition_source=partition_label,
226
+ target_model=to_jsonable(target_model),
227
+ environment_model=to_jsonable(target_model),
228
+ native_function_schemas=schemas,
229
+ initial_user_captures_reused_NOT_regenerated=True,
230
+ builder_model_calls=0,
231
+ ToolTalk_not_used=True,
232
+ )
233
+ path = Path(output_path).expanduser().absolute()
234
+ if path.resolve().is_relative_to(root) or path.exists() or path.is_symlink():
235
+ raise ValidationError("Tau input must be a new file outside the clean native checkout")
236
+ _save(path, value)
237
+ return dict(path=str(path), sha256=_sha(path))
238
+
239
+
240
+ class ACETauFeedback(NativePlanFeedback):
241
+ """Native Tau reward for a frozen, explicitly noninteractive tool plan."""
242
+
243
+ processor_id = "agentinputlab.ace.Tau.native_tool_plan_feedback.v1"
244
+ metric = "official_reward"
245
+
246
+ def __init__(
247
+ self, *, checkout, source_input, output_root, mode="offline", environment_base_url=None
248
+ ):
249
+ self._initialize(
250
+ checkout=checkout, source_input=source_input, output_root=output_root, mode=mode
251
+ )
252
+ self.domain = self.source.get("domain")
253
+ self.provider = _provider(self.domain)
254
+ if (
255
+ self.source.get("schema") != SCHEMA
256
+ or self.source.get("benchmark") != self.provider.benchmark_id
257
+ ):
258
+ raise ValidationError("Tau feedback schema or benchmark differs")
259
+ self.checkout = verify_native_checkout(checkout, revision=self.provider.source_revision)
260
+ self._verify_source()
261
+ expected_split = "train" if self.domain == "retail" else None
262
+ if self.source["source_official_split"] != expected_split or not self.source.get(
263
+ "partition_source"
264
+ ):
265
+ raise ValidationError("Tau official/caller-defined split provenance differs")
266
+ for row in self.rows.values():
267
+ _same_backbone(row["native_configuration"], self.source["target_model"])
268
+ if self.source["target_model"] != self.source["environment_model"]:
269
+ raise ValidationError("Tau environment and target backbone bindings differ")
270
+ with native_process_scope(self.checkout, package="tau2"):
271
+ self.tasks, self.histories, self.schemas, self.env_kwargs, self.configs, pins = (
272
+ _native_inputs(self.checkout, self.domain, list(self.rows.values()), self.provider)
273
+ )
274
+ if (
275
+ pins != self.source["source_files"]
276
+ or self.schemas != self.source["native_function_schemas"]
277
+ ):
278
+ raise ValidationError("Tau native source or visible function schemas changed")
279
+ self.environment_base_url = environment_base_url
280
+ if environment_base_url is not None:
281
+ url = urlsplit(environment_base_url) if isinstance(environment_base_url, str) else None
282
+ if (
283
+ not url
284
+ or url.scheme not in ("http", "https")
285
+ or not url.hostname
286
+ or url.username
287
+ or url.password
288
+ ):
289
+ raise ValidationError(
290
+ "Tau environment endpoint must be an explicit credential-free HTTP URL"
291
+ )
292
+ self.tool_names = {k: set(s.tool_names) for k, s in self.states.items()}
293
+ self._create_output()
294
+
295
+ def fingerprint(self):
296
+ self._verify_source()
297
+ return dict(
298
+ processor_id=self.processor_id,
299
+ mode=self.mode,
300
+ source_input=self.source_input,
301
+ source_files=self.source["source_files"],
302
+ source_revision=self.provider.source_revision,
303
+ scorer_id=self.provider.scorer_id,
304
+ scorer_revision=self.provider.scorer_revision,
305
+ provider_source_sha256=_sha(inspect.getfile(type(self.provider))),
306
+ task_adapter_source_sha256=_sha(__file__),
307
+ target_model=self.source["target_model"],
308
+ environment_model=self.source["environment_model"],
309
+ environment_base_url=self.environment_base_url,
310
+ source_official_split=self.source["source_official_split"],
311
+ partition_source=self.source["partition_source"],
312
+ groups={
313
+ role: [
314
+ dict(case_id=k, state_hash=r["state_hash"])
315
+ for k, r in self.rows.items()
316
+ if r["role"] == role
317
+ ]
318
+ for role in ("train", "validation")
319
+ },
320
+ visible_tool_schemas_sha256={k: content_hash(v) for k, v in self.schemas.items()},
321
+ feedback_metric=self.metric,
322
+ initial_user_captures_reused_NOT_regenerated=True,
323
+ noninteractive_plan_NOT_native_closed_loop_agent=True,
324
+ heldout_feedback_NOT_used=True,
325
+ transductive_NOT_independent_heldout=self.mode == "online",
326
+ ToolTalk_NOT_used=True,
327
+ native_user_simulator_calls=0,
328
+ native_judge_controls_and_costs_separate=True,
329
+ automatic_retry=False,
330
+ )
331
+
332
+ def _example(self, key, split):
333
+ state = self.states[key]
334
+ visible = dict(
335
+ query=state.query,
336
+ instruction=state.instruction,
337
+ messages=state.messages,
338
+ context=state.context,
339
+ tool_names=state.tool_names,
340
+ native_function_tools=self.schemas[key],
341
+ )
342
+ return ACEExample(
343
+ example_id=key,
344
+ input=dict(question=PLAN_INSTRUCTION + canonical_json(visible)),
345
+ expected_output=key,
346
+ split=split,
347
+ metadata=dict(
348
+ source_state_hash=state.state_hash,
349
+ task_processor=self.processor_id,
350
+ evaluator_target_is_opaque_case_id=True,
351
+ ),
352
+ )
353
+
354
+ def _native_score(self, key, plan, folder):
355
+ with native_process_scope(self.checkout, package="tau2"):
356
+ from tau2.data_model.message import AssistantMessage, ToolCall
357
+ from tau2.data_model.simulation import SimulationRun, TerminationReason
358
+ from tau2.evaluator import evaluator_nl_assertions
359
+ from tau2.evaluator.evaluator import EvaluationType, evaluate_simulation
360
+ from tau2.runner.build import build_environment
361
+ from tau2.utils import llm_utils
362
+
363
+ from workers.official_arenas.providers import tau3, tau_retail
364
+
365
+ started, start_time = time.monotonic(), datetime.now(timezone.utc).isoformat()
366
+ task, cfg = self.tasks[key], copy.deepcopy(self.configs[key])
367
+ history = copy.deepcopy(self.histories[key])
368
+ if self.environment_base_url is not None:
369
+ for role in ("agent", "user", "judge"):
370
+ if "llm_args_" + role in cfg:
371
+ cfg["llm_args_" + role]["api_base"] = self.environment_base_url
372
+ original_cost = llm_utils.get_response_cost
373
+ original_judge = (
374
+ evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS,
375
+ evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS_ARGS,
376
+ )
377
+ try:
378
+ tau3._install_local_tau_cost_accounting(cfg)
379
+ if self.domain == "banking_knowledge":
380
+ tau3._install_local_tau_nl_evaluator(cfg)
381
+ environment = build_environment(self.domain, env_kwargs=self.env_kwargs[key])
382
+ initial = task.initial_state
383
+ environment.set_state(
384
+ initial.initialization_data,
385
+ initial.initialization_actions,
386
+ copy.deepcopy(history),
387
+ strict=True,
388
+ )
389
+ for index, action in enumerate(plan["actions"]):
390
+ call = ToolCall(
391
+ id="ace_plan_" + str(index),
392
+ name=action["name"],
393
+ arguments=action["arguments"],
394
+ requestor="assistant",
395
+ )
396
+ history.append(AssistantMessage.text("", tool_calls=[call]))
397
+ history.append(environment.get_response(call))
398
+ history.append(AssistantMessage.text(plan["response"]))
399
+ simulation = SimulationRun(
400
+ id="ace-fitting-" + key,
401
+ task_id=key,
402
+ start_time=start_time,
403
+ end_time=datetime.now(timezone.utc).isoformat(),
404
+ duration=time.monotonic() - started,
405
+ termination_reason=TerminationReason.AGENT_STOP,
406
+ seed=cfg["llm_args_agent"]["seed"],
407
+ messages=history,
408
+ agent_cost=0.0,
409
+ user_cost=0.0,
410
+ )
411
+ _save(
412
+ folder / "before-native-scorer.json",
413
+ dict(
414
+ case_id=key,
415
+ plan=plan,
416
+ messages=[m.model_dump(mode="json") for m in history],
417
+ source_state_hash=self.states[key].state_hash,
418
+ native_noninteractive_plan_NOT_interactive_rollout=True,
419
+ evaluated_agent_model_calls=0,
420
+ native_user_simulator_calls=0,
421
+ ),
422
+ )
423
+ context = (
424
+ tau_retail.bound_retail_judge(
425
+ cfg, list(task.evaluation_criteria.nl_assertions or [])
426
+ )
427
+ if self.domain == "retail"
428
+ else nullcontext()
429
+ )
430
+ with context:
431
+ reward = evaluate_simulation(
432
+ simulation,
433
+ task,
434
+ evaluation_type=EvaluationType.ALL,
435
+ solo_mode=False,
436
+ domain=self.domain,
437
+ env_kwargs=self.env_kwargs[key],
438
+ strict_replay=True,
439
+ )
440
+ finally:
441
+ llm_utils.get_response_cost = original_cost
442
+ (
443
+ evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS,
444
+ evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS_ARGS,
445
+ ) = original_judge
446
+ result = dict(
447
+ status="native_Tau_noninteractive_tool_plan_scored",
448
+ case_id=key,
449
+ native_metrics=dict(official_reward=float(reward.reward)),
450
+ native_reward_info=reward.model_dump(mode="json"),
451
+ ordered_action_count=len(plan["actions"]),
452
+ source_state_hash=self.states[key].state_hash,
453
+ evaluated_agent_model_calls=0,
454
+ native_user_simulator_calls=0,
455
+ noninteractive_plan_agent_stop_explicit_NOT_native_interactive_success_claim=True,
456
+ )
457
+ _save(folder / "native-score.json", result)
458
+ return result
459
+
460
+
461
+ def create_tau_feedback(**configuration):
462
+ """Flat-JSON startup factory for the isolated native ACE worker."""
463
+ return ACETauFeedback(**configuration)