agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,94 @@
1
+ # Third-party source and asset boundaries
2
+
3
+ AgentInputLab standardizes interfaces and supplies clean-room control logic,
4
+ thin adapters, replay schemas, and external-worker protocols. It does not
5
+ vendor the audited upstream repositories, model weights, or benchmark data.
6
+ Small worker-side compatibility paths may follow a permissively licensed
7
+ upstream control flow; those paths name the source symbol and exact revision in
8
+ their local manifest and preserve the upstream attribution.
9
+
10
+ The authoritative, commit-pinned inventory is
11
+ `manifests/algorithms/registry.yaml`. The following summary is a release
12
+ safeguard, not legal advice.
13
+
14
+ | Method | Source license disposition | Core distribution rule |
15
+ |---|---|---|
16
+ | Query2doc official generations | CC-BY-4.0 asset | Download by manifest; preserve attribution |
17
+ | QueryGym | Apache-2.0 | Optional community adapter; source not vendored |
18
+ | Rewrite-Retrieve-Read | NOASSERTION | External checkout only |
19
+ | ReFormeR | NOASSERTION | External checkout only |
20
+ | InfoCQR | NOASSERTION | External checkout only |
21
+ | Plan-and-Solve | NOASSERTION | Clean-room paper-spec or external checkout |
22
+ | Decomposed Prompting | Apache-2.0 | Pinned external worker; bounded best-first compatibility path follows `BestFirstDecomposer.find_answer_decomp` and preserves attribution |
23
+ | Self-Ask | MIT | Minimal faithful engineering; preserve notice |
24
+ | LLMCompiler | MIT | Optional external/full-system integration |
25
+ | Plan-and-Budget | MIT | Minimal logic port requires preserved notice |
26
+ | HyDE | NOASSERTION | Clean-room paper reimplementation only |
27
+ | IRCoT | Apache-2.0 | Isolated external worker |
28
+ | Adaptive-RAG | Apache-2.0 | Rebuild recipe; assets separately licensed |
29
+ | R²AG | MIT | Native/factorial integration; weights separately gated |
30
+ | Q-RAG | NOASSERTION | User-provided checkout/checkpoints only |
31
+ | Ask or Assume? | MIT root; PolyForm enterprise subtree | Enterprise subtree excluded |
32
+ | UDCS | Apache-2.0 | Prompt/component integration; preserve notice |
33
+ | LongLLMLingua and LLMLingua-2 code | MIT | Lazy official adapter; weights/data separately governed |
34
+ | LLMLingua-2 MeetingBank training data | CC-BY-NC-SA-4.0 | Never bundled by runtime package |
35
+ | RECOMP code | MIT | Weights and training data remain NOASSERTION |
36
+ | ACON | MIT | Runtime/recipe integration; paper/code drift disclosed |
37
+ | SmartSearch | MIT | Optional isolated inference/training environment |
38
+ | RQ-RAG | NOASSERTION | External checkout only |
39
+ | ACE | Apache-2.0 | External worker/replay protocol |
40
+ | MCE | MIT | Sandboxed external recipe only |
41
+
42
+ The Plan-and-Budget allocation/schedule port follows
43
+ `junhongmit/P-and-B@2ca2d90c6d3d630ec2b11cc22e0f56751f7f2b7e`.
44
+ Copyright (c) 2024 Junhong Lin. Its upstream MIT permission notice is reproduced
45
+ in `NOTICE`; cache datasets and their underlying terms are separate and are not
46
+ bundled. Solver references/answers are excluded from the public cache view.
47
+
48
+ The Query2doc cached loader audits only the author's two native TREC split files
49
+ at `intfloat/query2doc_msmarco@3019b760cd849830aec6d0654dbe8a4a03e791d6`.
50
+ Attribution: Liang Wang, Nan Yang and Furu Wei, *Query2doc: Query Expansion with
51
+ Large Language Models* (EMNLP 2023). These are `text-davinci-003` generations;
52
+ the package contains checksums and loading code, not the generations. Use also
53
+ requires review of the underlying MS MARCO data terms.
54
+
55
+ ## Optional retrieval model profiles
56
+
57
+ AgentInputLab distributes only model identifiers, immutable repository
58
+ revisions, and preprocessing contracts for the profiles below. It does not
59
+ bundle, mirror, or redistribute model weights. A configured runtime obtains
60
+ weights from the user's local cache or, only when explicitly enabled, from the
61
+ named upstream repository.
62
+
63
+ | Profile asset | License disposition | Core distribution rule |
64
+ |---|---|---|
65
+ | BAAI BGE base English v1.5 | MIT | Pinned downloader/local-cache profile; no bundled weights |
66
+ | intfloat E5 base v2 | MIT | Pinned downloader/local-cache profile; no bundled weights |
67
+ | Qwen3 Embedding 0.6B | Apache-2.0 | Pinned downloader/local-cache profile; no bundled weights |
68
+ | facebook Contriever MS MARCO | CC-BY-NC-4.0 | Included for the current academic research scope; pinned downloader/local-cache profile; no bundled weights |
69
+
70
+ ## Benchmark and metric dependencies
71
+
72
+ | Asset or package | License disposition | Distribution rule |
73
+ |---|---|---|
74
+ | BEIR ArguAna official archive | CC-BY-SA-4.0 | Checksum-pinned downloader only; data is not included in wheels, sdists, containers, or Git |
75
+ | BRIGHT Earth Science and Economics | CC-BY-4.0 | Official query/qrel and corpus Parquet components are independently checksum-pinned; downloader only |
76
+ | MuSiQue v1.0 | CC-BY-4.0 | Official archive is checksum-pinned; only answerable dev and its evaluator-only overlap audit are normalized locally |
77
+ | 2WikiMultiHopQA `data_ids_april7` | Apache-2.0 | Official corrected archive and alias table are checksum-pinned; only dev is normalized locally |
78
+ | HotpotQA distractor validation | CC-BY-SA-4.0 | Official Hugging Face organization mirror is checksum-pinned; source annotation defects remain auditable |
79
+ | QASPER v0.3 train/dev archive | CC-BY-4.0 | Official archive is checksum-pinned; only the caption-augmented dev setting is normalized locally |
80
+ | tau2 1.0.1 Banking Knowledge source | MIT | Exact-commit archive is checksum-pinned and downloader-only; hidden simulator, evaluator, and database state remain evaluator-side in local prepared caches |
81
+ | AppWorld / AppWorld-UL | Apache-2.0 code repository; benchmark assets separately governed upstream | Exact-commit external runtime bridge; no runtime, simulator state, evaluator gold, or task rows are vendored |
82
+ | Microsoft STATE-Bench 0.8.1 | MIT | Exact-commit external runtime bridge; only a project-authored hash-locked task-ID membership index is packaged |
83
+ | Apple ToolSandbox | Apple source-code license in the repository `LICENSE` | Exact-commit external runtime bridge for academic evaluation; upstream runtime, scenarios, and evaluator implementation are not redistributed |
84
+ | ClariQ official dev TSV | NOASSERTION | Explicit local license-review gate; no rows or derived cache are redistributed |
85
+ | `ir-measures` | MIT | Optional `evaluation` dependency; resolved version is recorded in each evaluation environment |
86
+ | `pyarrow` | Apache-2.0 | Optional `benchmarks` dependency used to read pinned upstream Parquet assets; no benchmark rows are bundled |
87
+ | `rouge-score` | Apache-2.0 | Optional `evaluation` dependency; resolved version is recorded in each evaluation environment |
88
+
89
+ The byte locks and source provenance for audited smoke data are maintained in
90
+ `manifests/benchmarks/smoke_assets.json`. Synthetic unit-test fixtures are
91
+ project-authored and contain no copied benchmark rows.
92
+
93
+ For all assets, source, model, and data licenses are evaluated independently.
94
+ An open-source code license never propagates automatically to weights or data.
@@ -0,0 +1,2 @@
1
+ agentinputlab
2
+ workers
workers/README.md ADDED
@@ -0,0 +1,94 @@
1
+ # Pinned upstream workers
2
+
3
+ This directory contains auditable drivers for algorithms whose original runtime
4
+ cannot safely live inside the dependency-free AgentInputLab wheel. A worker is
5
+ not a vendored copy of an upstream repository: users provide the exact checkout
6
+ and optional weights, while the driver verifies the declared commit, license,
7
+ source files, configuration artifacts, and protocol identity before execution.
8
+
9
+ The worker boundary standardizes transport, limits, trace projection, and
10
+ fair-comparison settings. It does not replace the upstream algorithm. Any
11
+ compatibility or security patch is listed in the method's manifest and included
12
+ in its runtime fingerprint.
13
+
14
+ The following claims are deliberately separate:
15
+
16
+ 1. **Protocol tested** — malformed inputs and outputs fail closed.
17
+ 2. **Pinned plumbing tested** — a small checkout exercises the real upstream
18
+ symbols through the AgentInputLab bridge.
19
+ 3. **Upstream boot tested** — the audited checkout imports in its locked
20
+ dependency environment.
21
+ 4. **Result parity tested** — official fixtures or benchmark outputs match the
22
+ upstream reference within a declared tolerance.
23
+
24
+ A worker remains `external_only`, `planned`, or `retrain_required` until the
25
+ applicable upstream boot and parity gates pass. Fake fixtures are never evidence
26
+ of paper-result reproduction.
27
+
28
+ Current worker directories are documented independently. Each README lists its
29
+ upstream revision, supported profile, required environment variables, execution
30
+ mode, and remaining parity limitations.
31
+
32
+ The `ask_or_assume/` worker is intentionally narrower than a full OpenHands
33
+ deployment: it runs only the pinned IntentAgent decision and forced-question
34
+ component path and labels all outputs as component-only.
35
+
36
+ The `reformer/` worker executes source only from a user-mounted pinned checkout.
37
+ Its upstream has no license file, so it requires operator and per-request
38
+ `NOASSERTION` acknowledgements and never vendors upstream code.
39
+
40
+ The `recomp/` worker imports the pinned official sentence-scoring API and
41
+ replays the official abstractive prompt/generate path against user-mounted,
42
+ hash-attested checkpoints. It fixes each input block as one candidate sentence,
43
+ requires a retriever fingerprint without performing retrieval, and keeps
44
+ paper-native top-N assembly separate from controlled assembly.
45
+
46
+ The `rrr/` worker verifies the unlicensed, credential-risk pinned tarball
47
+ without extracting/importing it, then runs a hash-bound compatible inference
48
+ chain over user-mounted prompts, frozen search artifacts, and local checkpoints.
49
+ Warm-up/PPO checkpoints are inference-only; controlled retrieval is explicitly
50
+ separate from the frozen Bing-cache profile.
51
+
52
+ The `infocqr/` worker directly calls pinned official teacher prompt functions
53
+ or replays checksum-locked author caches. Because neither source/cache licensing
54
+ nor a released T5 student checkpoint is available, it is dual-gated and never
55
+ vendors upstream material or fabricates student weights.
56
+
57
+ The `smartsearch/` worker runs only frozen inference with the upstream
58
+ 10-round/top-5/8192-token loop. It replaces the unsafe code-execution search
59
+ sandbox with a fixed retriever shim and labels the result as a controlled
60
+ adapter; training and paper-result parity are outside this worker.
61
+
62
+ The `qrag/` worker runs a user-mounted, hash-attested frozen checkpoint through
63
+ the two-step E5-Hotpot action path and records every available Q-vector, action,
64
+ and selected candidate. Source and checkpoint licenses are acknowledged
65
+ separately; reward, gold labels, and training are unavailable at evaluation
66
+ time.
67
+
68
+ The `adaptive_rag/` worker verifies a clean pinned checkout and exact router
69
+ snapshot/weights/tokenizer, generator, retriever, corpus, index, and prompt
70
+ closures. A supervisor applies the upstream first-step A/B/C routing rule,
71
+ reserves budget before every call, executes only the selected **controlled
72
+ reimplementation** arm, and emits a hash-linked event log that the host
73
+ independently recomputes. Its hermetic runtime is fixture-only; neither the
74
+ source-checkout smoke nor this controlled arm is evidence of upstream arm,
75
+ batch/postprocess-script, or paper-result parity.
76
+
77
+ The `r2ag/` worker verifies a clean pinned RRAG checkout plus separately
78
+ hash-bound R²-Former, projection, feature schema, and complete reader/tokenizer/
79
+ feature-embedder file closures. It is designed to execute RFormer → projection
80
+ → retrieval-marker-only embedding replacement → reader generation on an
81
+ allowlisted, fixed candidate DTO and is labeled `controlled_source_informed`.
82
+ CI exercises the complete host/real-driver boundary with a fixture runtime; it
83
+ does not run a production neural runtime, official paper weights, or
84
+ paper-result parity.
85
+
86
+ The `mce/` worker loads the hash-pinned upstream `run_iteration` orchestration
87
+ but applies a hash-bound semantic patch at four unsafe/unimplemented seams:
88
+ meta/base agents run through an isolated manager, interface validation is
89
+ remote/static without local import, and evaluation runs in a separate trusted
90
+ oracle supervisor. Generated code remains in a verified worker-side
91
+ content-addressed store; the host receives only an opaque handle, hashes,
92
+ scores, usage, and a closed regular-file manifest. This is a controlled thin
93
+ wrapper—not `official_native` E2B or paper-result parity—and production
94
+ attestation/credentials remain operator supplied.
workers/ace/README.md ADDED
@@ -0,0 +1,261 @@
1
+ # ACE generic text-task worker
2
+
3
+ This directory contains the real upstream worker for
4
+ `recipe.ace.offline`. It does not reimplement ACE: every successful request
5
+ constructs the pinned upstream `ace.ace.ACE` object and calls
6
+ `ACE.run(mode="offline")`.
7
+
8
+ ## Pinned upstream
9
+
10
+ - Repository: `https://github.com/ace-agent/ace.git`
11
+ - Commit: `bcb7cea0504afad6f55fec4845dd4864c9f9eee7`
12
+ - License: Apache-2.0
13
+ - License SHA-256:
14
+ `c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4`
15
+
16
+ `compatibility_manifest.json` pins the license, package metadata, ACE
17
+ orchestrator, Generator, Reflector, Curator, prompts, LLM helper, logging, and
18
+ playbook helper sources. The worker verifies the Git HEAD and every listed hash
19
+ before importing anything from the mounted checkout. Symlinked pinned files,
20
+ source drift, a different commit, a dirty checkout (including untracked files),
21
+ and AppWorld imports fail closed.
22
+
23
+ The checkout is an operator startup argument. A JSONL request cannot select a
24
+ checkout, path, module, callable, command, endpoint, or environment variable.
25
+ The driver never invokes a shell and only invokes a fixed, argument-list
26
+ `git status --porcelain` subprocess during startup verification; requests
27
+ cannot influence that command. It never adds `ace-appworld` to the import path.
28
+ BulletpointAnalyzer is disabled.
29
+
30
+ ## Install upstream dependencies
31
+
32
+ Use an isolated Python 3.10+ worker environment:
33
+
34
+ ```bash
35
+ pip install 'agentinputlab[ace]'
36
+ python -m workers.ace.driver --checkout /opt/upstreams/ace --verify-only
37
+ ```
38
+
39
+ The `ace` extra installs the dependencies used by this generic-text worker:
40
+ NumPy, the OpenAI-compatible client, Python-dotenv and Tiktoken. It does not
41
+ install the upstream checkout or unrelated embedding models, FAISS, AppWorld,
42
+ or the optional BulletpointAnalyzer stack. BulletpointAnalyzer remains disabled
43
+ in this profile. Mounted source and license hashes are still verified before
44
+ imports; installing dependencies is not a substitute for that verification.
45
+ The worker itself adds no hidden model client. Credentials must already be
46
+ exported in the worker environment using the upstream provider variable:
47
+
48
+ - `OPENAI_API_KEY`
49
+ - `SAMBANOVA_API_KEY`
50
+ - `TOGETHER_API_KEY`
51
+ - `COMMONSTACK_API_KEY`
52
+
53
+ Loading credentials from a checkout `.env` is disabled. The default path keeps
54
+ the endpoints implemented by the pinned upstream. An operator may instead bind
55
+ an OpenAI-compatible deployment once at worker startup with `base_url` and
56
+ `api_key_env`; JSONL requests still cannot choose or mutate an endpoint,
57
+ credential name, model, or deployment option.
58
+
59
+ Verify the checkout and import without starting JSONL service:
60
+
61
+ ```bash
62
+ python -m workers.ace.driver \
63
+ --checkout /opt/upstreams/ace \
64
+ --verify-only
65
+ ```
66
+
67
+ ## Explicit LLM configuration
68
+
69
+ Serve mode requires `--llm-config-json`. There are no implicit model or pricing
70
+ defaults. The three fingerprints must exactly equal the
71
+ `ace_model_fingerprints` used by `ACEOfflineRecipe.fit`.
72
+
73
+ ```json
74
+ {
75
+ "api_provider": "openai",
76
+ "max_tokens_per_call": 4096,
77
+ "json_mode": true,
78
+ "base_url": "http://127.0.0.1:8000/v1",
79
+ "api_key_env": "OPENAI_API_KEY",
80
+ "extra_body": {
81
+ "chat_template_kwargs": {"enable_thinking": false}
82
+ },
83
+ "roles": {
84
+ "target": {
85
+ "model": "generator-model-id",
86
+ "fingerprint": {
87
+ "provider": "openai",
88
+ "model": "generator-model-id",
89
+ "revision": "deployment-revision"
90
+ },
91
+ "input_cost_per_million": 1.0,
92
+ "output_cost_per_million": 2.0
93
+ },
94
+ "reflector": {
95
+ "model": "reflector-model-id",
96
+ "fingerprint": {
97
+ "provider": "openai",
98
+ "model": "reflector-model-id",
99
+ "revision": "deployment-revision"
100
+ },
101
+ "input_cost_per_million": 1.0,
102
+ "output_cost_per_million": 2.0
103
+ },
104
+ "curator": {
105
+ "model": "curator-model-id",
106
+ "fingerprint": {
107
+ "provider": "openai",
108
+ "model": "curator-model-id",
109
+ "revision": "deployment-revision"
110
+ },
111
+ "input_cost_per_million": 1.0,
112
+ "output_cost_per_million": 2.0
113
+ }
114
+ }
115
+ }
116
+ ```
117
+
118
+ Zero prices are valid for explicitly unpriced local deployments; they must
119
+ still be stated. `base_url`, `api_key_env`, and `extra_body` are optional,
120
+ startup-only deployment controls and are accepted only for `api_provider` set
121
+ to `openai`. For an operator-bound endpoint, the worker sends the fit request's
122
+ `controls.seed` to all three role models as well as seeding local algorithm
123
+ randomness. A different seed in `extra_body` is rejected before a provider call;
124
+ the worker does not silently override either declaration. Model/server seed
125
+ support determines reproducibility. Start the worker with the serialized object
126
+ as one argument:
127
+
128
+ ```bash
129
+ python workers/ace/driver.py \
130
+ --checkout /opt/upstreams/ace \
131
+ --llm-config-json "$ACE_LLM_CONFIG"
132
+ ```
133
+
134
+ The process reads one strict JSON object per stdin line and writes exactly one
135
+ JSON object per stdout line. Upstream stdout is discarded into a one-MiB
136
+ bounded hash sink; the response retains only its byte count and SHA-256 digest,
137
+ never raw prompts or responses. Exceeding the bound fails the request closed.
138
+ Duplicate JSON keys, non-finite values, unknown protocol fields, and model
139
+ fingerprint mismatches are rejected.
140
+
141
+ ## Generic text-task mapping
142
+
143
+ The existing recipe sends `ACEExample` records. The worker maps each record to
144
+ the upstream task-neutral shape:
145
+
146
+ ```text
147
+ ACEExample.input -> question and optional context
148
+ ACEExample.expected_output -> target
149
+ ```
150
+
151
+ A string input becomes the question directly. A mapping may contain exactly one
152
+ of `question`, `query`, `prompt`, `instruction`, or `q`, plus optional
153
+ `context`. With no recognized question field, the whole JSON value is rendered
154
+ canonically as text. Non-string targets are also rendered as canonical JSON.
155
+ By default, evaluation is stripped, case-insensitive exact match.
156
+
157
+ ### Native retrieval feedback
158
+
159
+ For retrieval fitting, `ACERetrievalFeedback` binds an explicit benchmark
160
+ adapter, retriever, and disjoint training/validation groups. It preserves the
161
+ adapter's native qrels and exclusion rules. Training inputs contain the visible
162
+ information need and evidence; their evaluator target is an opaque case ID,
163
+ not a gold query or an answer. The upstream ACE reflection branch tests whether
164
+ the declared native metric is positive; validation selection uses its mean
165
+ (for example, nDCG@10), not text exact match. Invalid query JSON has zero
166
+ training feedback without extraction or another model call. Infrastructure
167
+ errors remain errors. This controlled task mapping is not a paper-parity claim.
168
+
169
+ ```python
170
+ from agentinputlab.recipes import ACERetrievalFeedback
171
+
172
+ feedback = ACERetrievalFeedback(
173
+ adapter=adapter, retriever=retriever,
174
+ train=train_examples, validation=validation_examples,
175
+ metric="ndcg_at_10", top_k=15,
176
+ )
177
+ train = feedback.fit_examples("train")
178
+ validation = feedback.fit_examples("validation")
179
+ ```
180
+
181
+ An in-process service can pass `data_processor=feedback` to `ACEWorkerRuntime`.
182
+ For a subprocess worker, provide an operator-owned factory function that
183
+ constructs this object from verified local dataset and corpus configuration:
184
+
185
+ ```bash
186
+ python -m workers.ace.driver \
187
+ --checkout /opt/upstreams/ace \
188
+ --llm-config-json "$ACE_LLM_CONFIG" \
189
+ --task-processor-factory my_project.ir_feedback:make_feedback \
190
+ --task-processor-config-json '{"dataset_config":"/opt/data/ir.json"}'
191
+ ```
192
+
193
+ The factory receives the startup JSON as keyword arguments. It must construct
194
+ the same feedback binding as the fit caller, perform no model calls, and expose
195
+ `answer_is_correct`, `evaluate_accuracy`, and `fingerprint`. Dataset/corpus
196
+ acquisition belongs before fitting. The driver records the factory module and
197
+ configuration hashes and the processor fingerprint; factory stdout is bounded
198
+ and excluded from JSONL. A processor may expose `validate_examples` to reject
199
+ altered or held-out fit inputs before any provider attempt. Requests cannot
200
+ select or change this factory. Put credentials only in the environment, never
201
+ in the factory configuration. Without these options the original worker and
202
+ its text-scoring behavior are unchanged.
203
+
204
+ The worker independently recomputes the split-audit hash and recipe request ID.
205
+ The request ID binds the complete canonical train and validation records,
206
+ including expected outputs and metadata, as well as controls, model
207
+ fingerprints, playbook hash, split-audit hash, and upstream commit. The worker
208
+ requires disjoint train/validation IDs and never accepts test examples in the
209
+ offline request.
210
+
211
+ ## Upstream execution and playbook fidelity
212
+
213
+ For every request the worker:
214
+
215
+ 1. renders native bullets as upstream lines of the form
216
+ `[id] helpful=X harmful=Y :: content`;
217
+ 2. initializes the pinned `ACE` with the configured Generator, Reflector, and
218
+ Curator;
219
+ 3. derives `next_global_id` with the pinned upstream helper;
220
+ 4. calls `ACE.run(mode="offline")` once, with one epoch, curator frequency one,
221
+ and validation at the final training step;
222
+ 5. observes, but does not reorder, the upstream Generator → Reflector → Curator
223
+ calls and their shared mutable playbook state;
224
+ 6. parses upstream `best_playbook`, preserving existing IDs, content, counters,
225
+ and metadata;
226
+ 7. emits replayable `add`, `edit`, and `delete` operations plus the exact
227
+ AgentInputLab `ACEPlaybook` candidate hash.
228
+
229
+ To remain compatible with the current diff protocol, structured order is
230
+ canonicalized as existing bullet order followed by upstream-created bullet
231
+ order. This does not change ACE's text supplied during execution; it only makes
232
+ the returned diff deterministic and exactly replayable by
233
+ `ACEWorkerResponse.from_mapping`.
234
+
235
+ Run artifacts and detailed upstream logs live in a per-request temporary
236
+ directory and are deleted after the response. Only validated usage totals and
237
+ the selected playbook cross the JSONL boundary.
238
+
239
+ ## Current parity limits
240
+
241
+ - The pinned `ACE.run` exposes only its whole-run validation-best playbook, not
242
+ validation scores for every curator step. Therefore one worker `iteration`
243
+ represents one complete real upstream offline run. Internal sample-level
244
+ Generator/Reflector/Curator state transitions still execute unchanged.
245
+ - Request `max_rounds` maps to upstream `max_num_rounds`, i.e. reflection and
246
+ regeneration rounds per incorrect sample. It is not the number of curator
247
+ steps.
248
+ - At the pinned commit, upstream `apply_curator_operations` fully applies only
249
+ `ADD`; counter changes appear as replayable `edit` operations in the worker
250
+ response. The worker does not invent unsupported merge semantics.
251
+ - `max_model_calls` counts actual calls to the pinned components'
252
+ `api_client.chat.completions.create` seam. Retry attempts share the same
253
+ request counter and are rejected before the next provider attempt when the
254
+ ceiling is reached. Successful agent results must still expose token metadata
255
+ or the request fails rather than silently undercounting it.
256
+ - Token and cost ceilings are checked from provider-reported successful-call
257
+ usage and explicit startup pricing. A call that itself crosses a ceiling is
258
+ rejected after that call returns; the upstream API has no preflight token
259
+ estimate. Failed provider attempts count toward `max_model_calls`, but their
260
+ billable tokens or cost cannot be added unless the provider returns usage
261
+ metadata for that failed attempt.
@@ -0,0 +1 @@
1
+ """Pinned ACE JSONL worker."""
@@ -0,0 +1,39 @@
1
+ {
2
+ "schema_version": "agentinputlab.ace.compatibility.v1",
3
+ "upstream": {
4
+ "repository": "https://github.com/ace-agent/ace.git",
5
+ "commit": "bcb7cea0504afad6f55fec4845dd4864c9f9eee7",
6
+ "license_path": "LICENSE.txt",
7
+ "license_spdx": "Apache-2.0",
8
+ "license_sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
9
+ "require_clean_checkout": true
10
+ },
11
+ "source_files": {
12
+ "pyproject.toml": "38856347e37fd50a80114e3f129a3767197e602e8d7b88ba87c7967dcd32ba09",
13
+ "ace/__init__.py": "5cffd7669d4b499d6e2de648cdc186297d7d0b663206fe4760d8a0ce64f39766",
14
+ "ace/ace.py": "70f7af3ff9088498afa8d6939c5e142cd8b0046179516ee336c57586b92ec914",
15
+ "ace/ace_batch.py": "6038bd54cc68597ed764e2dc5b011b4cb2523e31dbf942bcc7e1888597b29ff6",
16
+ "ace/core/__init__.py": "6dd2fc846149071e71a0b0f401c6ad703e6b9fde566b9c751bfc25dabc8c221f",
17
+ "ace/core/generator.py": "4b750a59752bd5bdb24b2f27b80938844d8a7b5d89e86d7d6bb726f3685c25f5",
18
+ "ace/core/reflector.py": "fe5ae8fb006fd16fdef7b8e7ad38dc746195a4bf0a366a82e69c473c9982b811",
19
+ "ace/core/curator.py": "863aa928b28538ded4b86ed330ce40c6012f6ca215f94618bef0615fb4de281b",
20
+ "ace/core/bulletpoint_analyzer.py": "c8348995c2e443a65343efb460939837655521d6962022d29aec1710947be4d2",
21
+ "ace/prompts/__init__.py": "803a8d38db890045fd344e36026a781947787e9b7d9d03e2d046faad33411a5b",
22
+ "ace/prompts/generator.py": "b8b12d4a6281ea4c422a2ee0bd919b50580d388027917e8740f82f97eb24eebb",
23
+ "ace/prompts/reflector.py": "72b343ce2781531dac2fd765543410b6abfc4817c39078dd4f83187c4d217e9c",
24
+ "ace/prompts/curator.py": "cca641bf025896b97f0daabf6208b110c081082930802e1f6d7ccf54fc90c9d7",
25
+ "llm.py": "cc615c5b26773fddc99c702bf2521a1c929e6363a905824b1be3e300337d1026",
26
+ "logger.py": "c54eda5fe3cdb8bc2cbef01be25f3f0a4eb6725123425c4a3674eed84d9cac23",
27
+ "utils.py": "7824aa262de2cd030b6914d1f7de698ec620dbec9c9e5f14ed1fe6322079570c",
28
+ "playbook_utils.py": "c67943f06cb0ee747be04d0599216e40d6c00e0cdf96ddc6af38fc26143e3a07"
29
+ },
30
+ "execution": {
31
+ "entrypoint": "ace.ace.ACE.run",
32
+ "mode": "offline",
33
+ "appworld_allowed": false,
34
+ "request_controlled_paths_allowed": false,
35
+ "request_controlled_imports_allowed": false,
36
+ "upstream_subprocess_allowed": false,
37
+ "verification_git_status_subprocess": true
38
+ }
39
+ }