agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,169 @@
1
+ """Frozen ACE playbook replay for the three common-support paper arenas.
2
+
3
+ ACE learns its playbook offline through the unchanged pinned upstream training
4
+ loop. This controlled adapter performs no further optimization at evaluation
5
+ time: it appends the frozen, hash-addressed playbook as one visible context
6
+ block while preserving the captured request, tools, messages, and evidence.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass
12
+
13
+ from agentinputlab.contracts import (
14
+ ContextBlock,
15
+ ExecutionTrace,
16
+ FrozenAgentState,
17
+ InputPatch,
18
+ InterventionContext,
19
+ PatchOp,
20
+ PatchOperation,
21
+ SemanticFamily,
22
+ TraceEvent,
23
+ TracePhase,
24
+ ValidationError,
25
+ )
26
+ from agentinputlab.contracts.serialization import content_hash
27
+
28
+ from .ace import ACEContextArtifact
29
+
30
+ ACE_COMMON_SUPPORT_RELEASE = "recipe.ace.eval_only"
31
+ ACE_COMMON_SUPPORT_PLACEMENT_RULE = "append_frozen_playbook_context.v1"
32
+
33
+
34
+ @dataclass(slots=True)
35
+ class ACEPlaybookCommonSupport:
36
+ """Replay one leakage-audited ACE artifact through the unified patch seam."""
37
+
38
+ artifact: ACEContextArtifact
39
+ method_id: str = "ace"
40
+ release_id: str = ACE_COMMON_SUPPORT_RELEASE
41
+ family: SemanticFamily = SemanticFamily.CURATE
42
+
43
+ def __post_init__(self) -> None:
44
+ if not isinstance(self.artifact, ACEContextArtifact):
45
+ raise ValidationError("ACE common support requires an ACEContextArtifact")
46
+
47
+ def propose(self, state: FrozenAgentState, context: InterventionContext) -> InputPatch:
48
+ del context
49
+ playbook_text = _render_playbook(self.artifact)
50
+ applied_characters = len(playbook_text) if self.artifact.playbook.bullets else 0
51
+ playbook_hash = self.artifact.playbook.playbook_hash
52
+ model_fingerprints_hash = content_hash(self.artifact.model_fingerprints)
53
+ worker_fingerprint_hash = content_hash(self.artifact.worker_fingerprint)
54
+ block = ContextBlock(
55
+ text=playbook_text,
56
+ block_id=f"ace-playbook:{self.artifact.artifact_id[:20]}",
57
+ source="ACE frozen offline playbook",
58
+ metadata={
59
+ "artifact_id": self.artifact.artifact_id,
60
+ "playbook_hash": playbook_hash,
61
+ "split_audit_hash": self.artifact.split_audit_hash,
62
+ "upstream_commit": self.artifact.upstream_commit,
63
+ "placement_rule": ACE_COMMON_SUPPORT_PLACEMENT_RULE,
64
+ },
65
+ )
66
+ trace = ExecutionTrace(
67
+ method_id=self.method_id,
68
+ release_id=self.release_id,
69
+ events=(
70
+ TraceEvent(
71
+ TracePhase.PREPARE,
72
+ "ace_frozen_artifact_verified",
73
+ {
74
+ "artifact_id": self.artifact.artifact_id,
75
+ "playbook_hash": playbook_hash,
76
+ "split_audit_hash": self.artifact.split_audit_hash,
77
+ "upstream_commit": self.artifact.upstream_commit,
78
+ "selected_iteration": self.artifact.selected_iteration,
79
+ "model_fingerprints_hash": model_fingerprints_hash,
80
+ "worker_fingerprint_hash": worker_fingerprint_hash,
81
+ },
82
+ ),
83
+ TraceEvent(
84
+ TracePhase.FINALIZE,
85
+ "ace_playbook_appended_to_visible_context"
86
+ if self.artifact.playbook.bullets
87
+ else "ace_native_empty_playbook_no_op",
88
+ {
89
+ "placement_rule": ACE_COMMON_SUPPORT_PLACEMENT_RULE,
90
+ "source_context_blocks": len(state.context),
91
+ "output_context_blocks": len(state.context)
92
+ + bool(self.artifact.playbook.bullets),
93
+ "playbook_bullets": len(self.artifact.playbook.bullets),
94
+ "playbook_characters": applied_characters,
95
+ },
96
+ ),
97
+ ),
98
+ counters={
99
+ "model_calls": 0,
100
+ "retrieval_calls": 0,
101
+ "tool_calls": 0,
102
+ "input_tokens": 0,
103
+ "output_tokens": 0,
104
+ "source_context_blocks": len(state.context),
105
+ "output_context_blocks": len(state.context) + bool(self.artifact.playbook.bullets),
106
+ "playbook_bullets": len(self.artifact.playbook.bullets),
107
+ "playbook_characters": applied_characters,
108
+ "offline_training_model_calls": int(self.artifact.usage.get("model_calls", 0)),
109
+ "offline_training_tokens": int(self.artifact.usage.get("tokens", 0)),
110
+ },
111
+ warnings=(
112
+ "offline_training_cost_is_reported_on_the_artifact_not_charged_per_eval_unit",
113
+ "controlled_common_support_playbook_mapping_not_paper_result_parity",
114
+ ),
115
+ )
116
+ return InputPatch(
117
+ method_id=self.method_id,
118
+ release_id=self.release_id,
119
+ source_state_hash=state.state_hash,
120
+ operations=(
121
+ PatchOperation(
122
+ PatchOp.ADD_CONTEXT,
123
+ block,
124
+ target="context",
125
+ rationale="Expose the frozen ACE playbook without changing captured evidence.",
126
+ ),
127
+ )
128
+ if self.artifact.playbook.bullets
129
+ else (
130
+ PatchOperation(PatchOp.NO_OP, rationale="Native ACE selected an empty playbook."),
131
+ ),
132
+ trace=trace,
133
+ implementation_identity=(
134
+ "pinned_upstream_ace_offline+" + ACE_COMMON_SUPPORT_PLACEMENT_RULE
135
+ ),
136
+ profile_kind="controlled_adapter",
137
+ metadata={
138
+ "artifact_id": self.artifact.artifact_id,
139
+ "playbook_hash": playbook_hash,
140
+ "split_audit_hash": self.artifact.split_audit_hash,
141
+ "upstream_commit": self.artifact.upstream_commit,
142
+ "selected_iteration": self.artifact.selected_iteration,
143
+ "model_fingerprints_hash": model_fingerprints_hash,
144
+ "worker_fingerprint_hash": worker_fingerprint_hash,
145
+ "offline_training_usage": dict(self.artifact.usage),
146
+ "common_support_placement_rule": ACE_COMMON_SUPPORT_PLACEMENT_RULE,
147
+ "source_state_hash": state.state_hash,
148
+ "paper_result_parity_claimed": False,
149
+ "native_empty_playbook_no_op": not self.artifact.playbook.bullets,
150
+ },
151
+ )
152
+
153
+
154
+ def _render_playbook(artifact: ACEContextArtifact) -> str:
155
+ lines = [
156
+ "ACE LEARNED PLAYBOOK (frozen offline artifact)",
157
+ "Use these learned strategies when relevant; the user request and visible evidence "
158
+ "remain authoritative.",
159
+ ]
160
+ for bullet in artifact.playbook.bullets:
161
+ lines.append(f"- [{bullet.bullet_id}] {bullet.content}")
162
+ return "\n".join(lines)
163
+
164
+
165
+ __all__ = [
166
+ "ACE_COMMON_SUPPORT_PLACEMENT_RULE",
167
+ "ACE_COMMON_SUPPORT_RELEASE",
168
+ "ACEPlaybookCommonSupport",
169
+ ]
@@ -0,0 +1,322 @@
1
+ """Common contracts for ACE's explicitly noninteractive native tool-plan feedback.
2
+
3
+ Task-specific implementations invoke registered native tools and scorers. This
4
+ module neither executes generated Python nor supplies a replacement scorer.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import contextlib
10
+ import hashlib
11
+ import json
12
+ import math
13
+ import os
14
+ import subprocess
15
+ import sys
16
+ import threading
17
+ from collections.abc import Mapping
18
+ from pathlib import Path, PurePosixPath
19
+
20
+ from agentinputlab.api import _state
21
+ from agentinputlab.artifacts import load_pinned_json_mapping
22
+ from agentinputlab.contracts import ExternalDependencyError, ValidationError
23
+ from agentinputlab.contracts.serialization import canonical_json, content_hash, to_jsonable
24
+
25
+ from .ace import ACEExample, validate_split_isolation
26
+
27
+ _NATIVE_LOCK = threading.RLock()
28
+ PLAN_INSTRUCTION = (
29
+ "Solve the supplied task as one noninteractive tool plan. Preserve all visible "
30
+ "task rules and function schemas. Put exactly one JSON object in your final_answer string: "
31
+ '{"actions":[{"name":"visible_function_name","arguments":{}}],"response":"message to user"}. '
32
+ "Use only visible function names with their required arguments. Actions execute in order "
33
+ "without another model call or access to their returned values while planning. "
34
+ "Do not output Python, code fences, unknown functions, or extra fields.\n"
35
+ )
36
+
37
+
38
+ def _sha(path):
39
+ return hashlib.sha256(Path(path).read_bytes()).hexdigest()
40
+
41
+
42
+ def _save(path, value):
43
+ path = Path(path)
44
+ path.parent.mkdir(parents=True, exist_ok=True)
45
+ with path.open("x", encoding="utf-8") as stream:
46
+ json.dump(to_jsonable(value), stream, sort_keys=True, indent=2, allow_nan=False)
47
+
48
+
49
+ def _unique_object(pairs):
50
+ result = {}
51
+ for key, value in pairs:
52
+ if key in result:
53
+ raise ValueError("duplicate tool-plan JSON key")
54
+ result[key] = value
55
+ return result
56
+
57
+
58
+ def parse_native_tool_plan(text, *, tool_names, tool_cap):
59
+ """Decode a complete plan with advertised names, never fences or code repair."""
60
+ if not isinstance(text, str) or type(tool_cap) is not int or tool_cap < 0:
61
+ raise ValueError("tool plan requires a string and nonnegative integer tool cap")
62
+ value = json.loads(
63
+ text,
64
+ object_pairs_hook=_unique_object,
65
+ parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON")),
66
+ )
67
+ canonical_json(value)
68
+ if not isinstance(value, dict) or set(value) != {"actions", "response"}:
69
+ raise ValueError("tool plan requires actions and response")
70
+ if (
71
+ not isinstance(value["actions"], list)
72
+ or len(value["actions"]) > tool_cap
73
+ or not isinstance(value["response"], str)
74
+ or not value["response"].strip()
75
+ ):
76
+ raise ValueError("tool-plan actions/response violate the declared contract")
77
+ for action in value["actions"]:
78
+ if (
79
+ not isinstance(action, dict)
80
+ or set(action) != {"name", "arguments"}
81
+ or not isinstance(action["name"], str)
82
+ or action["name"] not in tool_names
83
+ or not isinstance(action["arguments"], dict)
84
+ ):
85
+ raise ValueError("tool plan must use advertised tools and argument objects")
86
+ return value
87
+
88
+
89
+ def verify_native_checkout(checkout, *, revision):
90
+ root = Path(checkout).expanduser().resolve()
91
+ if not root.is_dir():
92
+ raise ExternalDependencyError("native benchmark checkout is unavailable")
93
+ try:
94
+ head = subprocess.check_output(
95
+ ["git", "-C", str(root), "rev-parse", "HEAD"],
96
+ text=True,
97
+ stderr=subprocess.DEVNULL,
98
+ ).strip()
99
+ dirty = subprocess.check_output(
100
+ ["git", "-C", str(root), "status", "--porcelain"],
101
+ text=True,
102
+ stderr=subprocess.DEVNULL,
103
+ ).strip()
104
+ except subprocess.CalledProcessError as exc:
105
+ raise ExternalDependencyError("native benchmark must be a verified Git checkout") from exc
106
+ if head != revision or dirty:
107
+ raise ExternalDependencyError("native benchmark checkout revision or clean state differs")
108
+ return root
109
+
110
+
111
+ @contextlib.contextmanager
112
+ def native_process_scope(root, *, package):
113
+ """Native runtimes use process-global imports/CWD; serialize and restore them."""
114
+ root = Path(root).resolve()
115
+ with _NATIVE_LOCK:
116
+ for name, module in tuple(sys.modules.items()):
117
+ if name == package or name.startswith(package + "."):
118
+ path = getattr(module, "__file__", None)
119
+ if path and not Path(path).resolve().is_relative_to(root):
120
+ raise ExternalDependencyError(
121
+ "a different native benchmark checkout is imported"
122
+ )
123
+ previous_cwd, previous_path = Path.cwd(), list(sys.path)
124
+ try:
125
+ os.chdir(root)
126
+ sys.path[:0] = [str(root / "src"), str(root)]
127
+ yield
128
+ finally:
129
+ sys.path[:] = previous_path
130
+ os.chdir(previous_cwd)
131
+
132
+
133
+ def source_file_pins(root, paths):
134
+ root = Path(root).resolve()
135
+ result = {}
136
+ for path in paths:
137
+ path = Path(path).resolve()
138
+ if not path.is_file() or not path.is_relative_to(root):
139
+ raise ValidationError("native feedback source file must be inside its checkout")
140
+ result[path.relative_to(root).as_posix()] = _sha(path)
141
+ return result
142
+
143
+
144
+ def verify_source_files(root, files):
145
+ if not isinstance(files, Mapping) or not files:
146
+ raise ValidationError("native feedback needs pinned source files")
147
+ for name, digest in files.items():
148
+ path = PurePosixPath(name)
149
+ if path.is_absolute() or ".." in path.parts or str(path) != name:
150
+ raise ValidationError("native feedback source path must be relative")
151
+ actual = (Path(root) / name).resolve()
152
+ if not actual.is_relative_to(root) or not actual.is_file() or _sha(actual) != digest:
153
+ raise ValidationError("native feedback source checksum changed")
154
+
155
+
156
+ class NativePlanFeedback:
157
+ """Shared fit/stream validation and audited feedback, not a native agent."""
158
+
159
+ def _initialize(self, *, source_input, checkout, output_root, mode):
160
+ if mode not in ("offline", "online"):
161
+ raise ValidationError("native ACE feedback mode must be offline or online")
162
+ if not isinstance(source_input, Mapping) or set(source_input) != {"path", "sha256"}:
163
+ raise ValidationError("native ACE feedback source needs a path and trusted SHA-256")
164
+ self.source_input = dict(source_input)
165
+ self.source = load_pinned_json_mapping(
166
+ source_input["path"], expected_sha256=source_input["sha256"], max_bytes=16 * 1024 * 1024
167
+ )
168
+ self.mode, self.checkout = mode, Path(checkout).expanduser().resolve()
169
+ rows = self.source.get("fitting_rows")
170
+ required = {"case_id", "role", "state", "state_hash", "protected_checkpoint"}
171
+ if (
172
+ not isinstance(rows, list)
173
+ or not rows
174
+ or any(
175
+ not isinstance(r, dict)
176
+ or not required <= r.keys()
177
+ or not isinstance(r["case_id"], str)
178
+ or not isinstance(r["role"], str)
179
+ for r in rows
180
+ )
181
+ ):
182
+ raise ValidationError("native ACE feedback needs nonempty frozen fitting groups")
183
+ self.rows = {r["case_id"]: r for r in rows}
184
+ if len(self.rows) != len(rows) or any(
185
+ not isinstance(k, str) or not k.strip() for k in self.rows
186
+ ):
187
+ raise ValidationError("native ACE fitting IDs must be unique strings")
188
+ self.states = {key: _state(row["state"]) for key, row in self.rows.items()}
189
+ if (
190
+ set(row["role"] for row in rows) != {"train", "validation"}
191
+ or any(self.states[k].state_hash != r["state_hash"] for k, r in self.rows.items())
192
+ or len({s.state_hash for s in self.states.values()}) != len(rows)
193
+ or set(self.rows) & set(self.source.get("heldout_ids", ()))
194
+ ):
195
+ raise ValidationError("native ACE fitting groups, states or heldout isolation differ")
196
+ self.root = Path(output_root).expanduser().absolute()
197
+ if self.root.resolve().is_relative_to(self.checkout):
198
+ raise ValidationError("native ACE feedback output must be outside the clean checkout")
199
+ if self.root.exists() or self.root.is_symlink():
200
+ raise ValidationError("native ACE feedback output must be create-only")
201
+ self.events, self.cache = [], {}
202
+
203
+ def _create_output(self):
204
+ validate_split_isolation(self.fit_examples("train"), self.fit_examples("validation"))
205
+ self.root.mkdir(parents=True, exist_ok=False, mode=0o700)
206
+
207
+ def _verify_source(self):
208
+ original = load_pinned_json_mapping(
209
+ self.source_input["path"],
210
+ expected_sha256=self.source_input["sha256"],
211
+ max_bytes=16 * 1024 * 1024,
212
+ )
213
+ if (
214
+ original != self.source
215
+ or self.rows != {r["case_id"]: r for r in original["fitting_rows"]}
216
+ or set(self.states) != set(self.rows)
217
+ or any(self.states[k].state_hash != r["state_hash"] for k, r in self.rows.items())
218
+ ):
219
+ raise ValidationError("native feedback in-memory source or states changed")
220
+ verify_native_checkout(self.checkout, revision=self.source["native_source_revision"])
221
+ verify_source_files(self.checkout, self.source["source_files"])
222
+
223
+ def fit_examples(self, split):
224
+ if split not in ("train", "validation"):
225
+ raise ValidationError("native ACE fitting exposes no heldout examples")
226
+ return tuple(
227
+ self._example(key, split) for key, row in self.rows.items() if row["role"] == split
228
+ )
229
+
230
+ def validate_examples(self, train, validation):
231
+ self.fingerprint()
232
+ for supplied, split in ((train, "train"), (validation, "validation")):
233
+ if tuple(to_jsonable(e) for e in supplied) != tuple(
234
+ e.as_dict() for e in self.fit_examples(split)
235
+ ):
236
+ raise ValidationError("native ACE examples differ from the frozen fitting groups")
237
+
238
+ def stream_examples(self):
239
+ if self.mode != "online":
240
+ raise ValidationError("native ACE stream feedback needs explicit online mode")
241
+ return tuple(
242
+ ACEExample.from_value(
243
+ dict(
244
+ e.as_dict(),
245
+ metadata=dict(
246
+ e.as_dict()["metadata"],
247
+ original_operator_role=e.split,
248
+ source_official_split=self.source["source_official_split"],
249
+ online_protocol_label_NOT_official_test_split=True,
250
+ ),
251
+ ),
252
+ split="test",
253
+ )
254
+ for split in ("train", "validation")
255
+ for e in self.fit_examples(split)
256
+ )
257
+
258
+ def validate_online_examples(self, supplied):
259
+ self.fingerprint()
260
+ if tuple(ACEExample.from_value(e, split="test").as_dict() for e in supplied) != tuple(
261
+ e.as_dict() for e in self.stream_examples()
262
+ ):
263
+ raise ValidationError("native ACE online order, source or feedback targets changed")
264
+
265
+ def score_answer(self, predicted, target):
266
+ if not isinstance(target, str) or target not in self.rows:
267
+ raise ValidationError("native ACE feedback target is outside the frozen fitting groups")
268
+ fingerprint = self.fingerprint()
269
+ state = self.states[target]
270
+ try:
271
+ plan = parse_native_tool_plan(
272
+ predicted, tool_names=self.tool_names[target], tool_cap=state.budget.max_tool_calls
273
+ )
274
+ except (ValueError, TypeError):
275
+ self.events.append(
276
+ dict(
277
+ target=target,
278
+ prediction_sha256=content_hash(predicted),
279
+ status="invalid_tool_plan_training_feedback",
280
+ score=0.0,
281
+ source_state_hash=state.state_hash,
282
+ native_task_score_NOT_claimed=True,
283
+ native_metrics=None,
284
+ new_environment_calls=0,
285
+ )
286
+ )
287
+ return 0.0
288
+ key = content_hash([target, state.state_hash, plan, fingerprint])
289
+ with _NATIVE_LOCK:
290
+ reused = key in self.cache
291
+ if not reused:
292
+ folder = self.root / "native-feedback" / key
293
+ folder.mkdir(parents=True, exist_ok=False)
294
+ self.cache[key] = self._native_score(target, plan, folder)
295
+ result = self.cache[key]
296
+ score = result["native_metrics"][self.metric]
297
+ if type(score) not in (int, float) or not math.isfinite(score) or not 0 <= score <= 1:
298
+ raise ValidationError("native ACE task feedback score is invalid")
299
+ self.events.append(
300
+ dict(
301
+ target=target,
302
+ prediction_sha256=content_hash(predicted),
303
+ score=score,
304
+ result=result,
305
+ source_state_hash=state.state_hash,
306
+ exact_feedback_reuse_NOT_new_model_draw=reused,
307
+ )
308
+ )
309
+ return float(score)
310
+
311
+ def answer_is_correct(self, predicted, ground_truth):
312
+ return self.score_answer(predicted, ground_truth) > 0
313
+
314
+ def evaluate_accuracy(self, predictions, ground_truths):
315
+ if len(predictions) != len(ground_truths):
316
+ raise ValidationError("native ACE prediction/target lengths differ")
317
+ return (
318
+ sum(self.score_answer(p, t) for p, t in zip(predictions, ground_truths, strict=True))
319
+ / len(predictions)
320
+ if predictions
321
+ else 0.0
322
+ )