agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,355 @@
1
+ """Full native IRCoT-QA, with source-locked and explicit dependency substitutions."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ from collections.abc import Mapping
8
+ from dataclasses import asdict, dataclass, field
9
+ from pathlib import Path
10
+
11
+ from agentinputlab.artifacts import load_pinned_json_mapping
12
+ from agentinputlab.contracts import (
13
+ AlgorithmOutputError,
14
+ ArtifactError,
15
+ BudgetExceededError,
16
+ CapabilityError,
17
+ CompositeResult,
18
+ ContextBlock,
19
+ ExecutionTrace,
20
+ ExternalDependencyError,
21
+ FrozenAgentState,
22
+ InterventionContext,
23
+ SemanticFamily,
24
+ TraceEvent,
25
+ TracePhase,
26
+ )
27
+ from agentinputlab.contracts.serialization import content_hash, freeze_mapping, to_jsonable
28
+ from agentinputlab.integrations.external_workers.ircot import (
29
+ IRCOT_UPSTREAM_COMMIT,
30
+ IRCoTControls,
31
+ IRCoTWorkerRequest,
32
+ IRCoTWorkerResponse,
33
+ )
34
+
35
+ IRCOT_NATIVE_QA_RELEASE = "acquire.ircot.native_like_substituted"
36
+ IRCOT_NATIVE_QA_BINDING_MODE = "agentinputlab.ircot.native_like_qa.v1"
37
+ IRCOT_NATIVE_QA_MANIFEST_SCHEMA = "agentinputlab.ircot.native_like_substitutions.v1"
38
+ _ROLES = {
39
+ "retrieve_and_reset_paragraphs",
40
+ "step_by_step_cot_gen",
41
+ "step_by_step_exit_controller",
42
+ "copy_question",
43
+ "llmqa",
44
+ "answer_extractor",
45
+ }
46
+
47
+
48
+ def _manifest(fingerprint):
49
+ if (
50
+ not isinstance(fingerprint, Mapping)
51
+ or fingerprint.get("binding_mode") != IRCOT_NATIVE_QA_BINDING_MODE
52
+ or fingerprint.get("upstream_commit") != IRCOT_UPSTREAM_COMMIT
53
+ or fingerprint.get("full_six_participant_QA_graph") is not True
54
+ or fingerprint.get("native_final_prediction_extraction") is not True
55
+ ):
56
+ raise ArtifactError("IRCoT native-like profile requires the complete pinned QA runtime")
57
+ for key in ("generator", "retriever", "tokenizer", "native_dependencies"):
58
+ if not isinstance(fingerprint.get(key), Mapping) or not fingerprint[key]:
59
+ raise ArtifactError("IRCoT native-like runtime dependency fingerprints are incomplete")
60
+ for key in ("corpus_revision", "index_revision", "prompt_sha256"):
61
+ if not isinstance(fingerprint.get(key), str) or not fingerprint[key].strip():
62
+ raise ArtifactError("IRCoT native-like source identities are incomplete")
63
+ for key in (
64
+ "max_model_calls",
65
+ "max_output_tokens",
66
+ "max_retrieval_calls",
67
+ "per_call_output_tokens",
68
+ ):
69
+ if type(fingerprint.get(key)) is not int or fingerprint[key] < 1:
70
+ raise ArtifactError("IRCoT native-like runtime budgets must be positive integers")
71
+ if type(fingerprint.get("seed")) is not int or fingerprint["seed"] < 0:
72
+ raise ArtifactError("IRCoT native-like runtime seed must be a nonnegative integer")
73
+ return dict(
74
+ schema=IRCOT_NATIVE_QA_MANIFEST_SCHEMA,
75
+ release_id=IRCOT_NATIVE_QA_RELEASE,
76
+ runtime_fingerprint=to_jsonable(fingerprint),
77
+ runtime_fingerprint_sha256=content_hash(fingerprint),
78
+ substitutions=[
79
+ "generation_backend",
80
+ "retrieval_backend_and_index",
81
+ "gpt2_tokenizer_resolver",
82
+ ],
83
+ original_controller_and_six_participants=True,
84
+ native_answer_extraction=True,
85
+ original_native_generator_default_cap=300,
86
+ per_call_limit_is_a_ceiling_not_an_increase=True,
87
+ shared_final_call_and_output_budget=True,
88
+ original_cache_or_retry=False,
89
+ historical_Codex_full_Elasticsearch_or_published_score_parity=False,
90
+ )
91
+
92
+
93
+ def write_ircot_substitution_manifest(runtime, output, *, accept_substitutions=False):
94
+ """Create-only operator acceptance of the exact native-like dependency set."""
95
+ if accept_substitutions is not True:
96
+ raise ArtifactError(
97
+ "review the IRCoT model/index substitutions and acknowledge them explicitly"
98
+ )
99
+ value = _manifest(runtime.fingerprint())
100
+ path = Path(output).expanduser().absolute()
101
+ path.parent.mkdir(parents=True, exist_ok=True)
102
+ with path.open("x") as f:
103
+ json.dump(value, f, indent=2, sort_keys=True, allow_nan=False)
104
+ return dict(path=str(path), sha256=hashlib.sha256(path.read_bytes()).hexdigest())
105
+
106
+
107
+ @dataclass(frozen=True, slots=True)
108
+ class IRCoTNativeQARequest:
109
+ legacy: IRCoTWorkerRequest
110
+ max_model_calls: int
111
+ per_call_output_tokens: int
112
+ seed: int
113
+ expected_runtime_fingerprint_sha256: str
114
+
115
+ def external_reservation(self):
116
+ return {"model_calls": 1, "retrieval_calls": 1}
117
+
118
+ def raise_for_external_status(self, result):
119
+ if not isinstance(result, Mapping) or result.get("status") != "ok":
120
+ code = (
121
+ result.get("failure_code", "INVALID_RESPONSE")
122
+ if isinstance(result, Mapping)
123
+ else "INVALID_RESPONSE"
124
+ )
125
+ if code == "NATIVE_QA_EMPTY_OR_NON_TEXT":
126
+ raise AlgorithmOutputError(
127
+ "native IRCoT-QA preserved an empty/non-text final prediction"
128
+ )
129
+ raise ExternalDependencyError(f"native IRCoT-QA failed with {code}")
130
+
131
+
132
+ @dataclass(frozen=True, slots=True)
133
+ class IRCoTNativeLikeSubstituted:
134
+ manifest_path: str | Path
135
+ expected_manifest_sha256: str
136
+ controls: IRCoTControls | Mapping = field(
137
+ default_factory=lambda: IRCoTControls(
138
+ per_step_top_k=6, max_steps=7, max_total_documents=15, max_output_tokens=4096
139
+ )
140
+ )
141
+ method_id: str = "ircot"
142
+ release_id: str = IRCOT_NATIVE_QA_RELEASE
143
+ family: SemanticFamily = SemanticFamily.ACQUIRE
144
+ _binding: Mapping = field(init=False, repr=False)
145
+
146
+ def __post_init__(self):
147
+ if isinstance(self.controls, Mapping):
148
+ try:
149
+ object.__setattr__(self, "controls", IRCoTControls(**self.controls))
150
+ except (TypeError, ValueError) as exc:
151
+ raise ArtifactError("IRCoT full QA control fields are invalid") from exc
152
+ if not isinstance(self.controls, IRCoTControls):
153
+ raise ArtifactError("IRCoT full QA requires explicit controls")
154
+ for key in ("per_step_top_k", "max_steps", "max_total_documents", "max_output_tokens"):
155
+ if type(getattr(self.controls, key)) is not int or getattr(self.controls, key) < 1:
156
+ raise ArtifactError("IRCoT full QA controls must be positive integers")
157
+ if self.controls.deduplicate is not True:
158
+ raise ArtifactError("the pinned native IRCoT source always deduplicates")
159
+ if (
160
+ self.method_id != "ircot"
161
+ or self.release_id != IRCOT_NATIVE_QA_RELEASE
162
+ or self.family is not SemanticFamily.ACQUIRE
163
+ ):
164
+ raise ArtifactError("IRCoT full QA registered profile identity differs")
165
+ value = load_pinned_json_mapping(
166
+ self.manifest_path, expected_sha256=self.expected_manifest_sha256
167
+ )
168
+ if value != _manifest(value.get("runtime_fingerprint")):
169
+ raise ArtifactError(
170
+ "IRCoT full QA substitution manifest differs from the exact supported protocol"
171
+ )
172
+ object.__setattr__(self, "_binding", freeze_mapping(value))
173
+
174
+ def run(self, state: FrozenAgentState, context: InterventionContext) -> CompositeResult:
175
+ if not state.query.strip() or not state.task_id:
176
+ raise CapabilityError("IRCoT full QA requires question text and an explicit task ID")
177
+ if state.context or state.messages or state.tool_names or state.instruction:
178
+ raise CapabilityError(
179
+ "IRCoT full QA accepts question-only state plus an explicitly bound external corpus"
180
+ )
181
+ binding = load_pinned_json_mapping(
182
+ self.manifest_path, expected_sha256=self.expected_manifest_sha256
183
+ )
184
+ if content_hash(binding) != content_hash(self._binding):
185
+ raise ArtifactError("IRCoT full QA substitution manifest changed")
186
+ runtime = context.external_runtime
187
+ if (
188
+ runtime is None
189
+ or not callable(getattr(runtime, "run", None))
190
+ or not callable(getattr(runtime, "fingerprint", None))
191
+ ):
192
+ raise ExternalDependencyError(
193
+ "IRCoT full QA requires the startup-bound native QA runtime"
194
+ )
195
+ fingerprint = runtime.fingerprint()
196
+ if content_hash(fingerprint) != binding["runtime_fingerprint_sha256"]:
197
+ raise ArtifactError("IRCoT full QA exact runtime/model/index fingerprint mismatch")
198
+ for component, key in ((context.backend, "generator"), (context.retriever, "retriever")):
199
+ if component is None or not callable(getattr(component, "fingerprint", None)):
200
+ raise ExternalDependencyError(
201
+ "IRCoT full QA requires backend and retriever fingerprints"
202
+ )
203
+ if content_hash(component.fingerprint()) != content_hash(fingerprint[key]):
204
+ raise ArtifactError(
205
+ "IRCoT full QA context is not the acknowledged model/index substitution"
206
+ )
207
+ controls = self.controls
208
+ if (
209
+ context.seed != fingerprint["seed"]
210
+ or controls.max_steps + 1 > state.budget.max_model_calls
211
+ or controls.max_steps + 1 > fingerprint["max_model_calls"]
212
+ or controls.max_output_tokens
213
+ > min(state.budget.max_output_tokens, fingerprint["max_output_tokens"])
214
+ or controls.max_steps + 1
215
+ > min(state.budget.max_retrieval_calls, fingerprint["max_retrieval_calls"])
216
+ ):
217
+ raise BudgetExceededError(
218
+ "IRCoT full QA cannot reserve its declared reasoning/final workflow under the bound budget/seed"
219
+ )
220
+ base = dict(
221
+ state_hash=state.state_hash,
222
+ task_id=state.task_id,
223
+ question=state.query,
224
+ controls=asdict(controls),
225
+ generator_fingerprint=fingerprint["generator"],
226
+ retriever_fingerprint=fingerprint["retriever"],
227
+ corpus_revision=fingerprint["corpus_revision"],
228
+ index_revision=fingerprint["index_revision"],
229
+ prompt_hash=fingerprint["prompt_sha256"],
230
+ )
231
+ legacy = IRCoTWorkerRequest(
232
+ request_id=content_hash(
233
+ dict(
234
+ base,
235
+ release_id=self.release_id,
236
+ manifest_sha256=self.expected_manifest_sha256,
237
+ seed=context.seed,
238
+ )
239
+ ),
240
+ **{**base, "controls": controls},
241
+ )
242
+ request = IRCoTNativeQARequest(
243
+ legacy,
244
+ min(state.budget.max_model_calls, fingerprint["max_model_calls"]),
245
+ fingerprint["per_call_output_tokens"],
246
+ context.seed,
247
+ content_hash(fingerprint),
248
+ )
249
+ raw = runtime.run(request)
250
+ request.raise_for_external_status(raw)
251
+ if raw.get("identity") != fingerprint or content_hash(
252
+ runtime.fingerprint()
253
+ ) != content_hash(fingerprint):
254
+ raise ArtifactError("IRCoT full QA response/runtime identity changed")
255
+ response = IRCoTWorkerResponse.from_mapping(raw, request=legacy)
256
+ roles = [e["role"] for e in raw["native_controller_events"]]
257
+ # The original CopyQuestion participant schedules a task. After answer
258
+ # extraction the source controller visits it again to drain that task;
259
+ # its eoq_after_n_calls=1 guard ends without another model generation.
260
+ tail = ["copy_question", "llmqa", "answer_extractor", "copy_question"]
261
+ prefix = roles[:-4]
262
+ loop = [
263
+ "retrieve_and_reset_paragraphs",
264
+ "step_by_step_cot_gen",
265
+ "step_by_step_exit_controller",
266
+ ]
267
+ if (
268
+ not set(roles) >= _ROLES
269
+ or roles[-4:] != tail
270
+ or not prefix
271
+ or len(prefix) % 3
272
+ or any(prefix[i : i + 3] != loop for i in range(0, len(prefix), 3))
273
+ ):
274
+ raise AlgorithmOutputError(
275
+ "IRCoT full QA did not execute the complete native final-answer graph"
276
+ )
277
+ if (
278
+ not response.answer.strip()
279
+ or response.usage["model_calls"] > request.max_model_calls
280
+ or response.usage["model_calls"] != roles.count("step_by_step_cot_gen") + 1
281
+ ):
282
+ raise AlgorithmOutputError(
283
+ "IRCoT full QA answer/model accounting differs from its native graph"
284
+ )
285
+ by_id = {d.document_id: d for d in response.documents}
286
+ evidence = tuple(
287
+ ContextBlock(
288
+ text=by_id[i].text,
289
+ block_id=i,
290
+ source=response.documents[0].corpus_revision,
291
+ score=by_id[i].score,
292
+ metadata={
293
+ "title": by_id[i].title,
294
+ "corpus_revision": by_id[i].corpus_revision,
295
+ "evidence_kind": "real_corpus_document",
296
+ },
297
+ )
298
+ for i in response.active_document_ids
299
+ )
300
+ return CompositeResult(
301
+ method_id=self.method_id,
302
+ release_id=self.release_id,
303
+ source_state_hash=state.state_hash,
304
+ final_answer=response.answer,
305
+ evidence=evidence,
306
+ trace=ExecutionTrace(
307
+ method_id=self.method_id,
308
+ release_id=self.release_id,
309
+ events=(
310
+ TraceEvent(TracePhase.PREPARE, "native_qa_substitution_binding", binding),
311
+ TraceEvent(
312
+ TracePhase.OBSERVE,
313
+ "native_qa_controller",
314
+ {
315
+ "events": raw["native_controller_events"],
316
+ "model_classes": raw["native_model_classes"],
317
+ "runtime_events": raw["controlled_runtime_events"],
318
+ },
319
+ ),
320
+ TraceEvent(
321
+ TracePhase.FINALIZE,
322
+ "native_qa_prediction",
323
+ {
324
+ "answer": response.answer,
325
+ "exit_reason": response.exit_reason,
326
+ "active_document_ids": response.active_document_ids,
327
+ "native_chain": raw["native_chain"],
328
+ },
329
+ ),
330
+ ),
331
+ counters=dict(response.usage),
332
+ ),
333
+ implementation_identity=IRCOT_NATIVE_QA_BINDING_MODE,
334
+ profile_kind="controlled_adapter",
335
+ metadata={
336
+ "substitution_manifest_sha256": self.expected_manifest_sha256,
337
+ "runtime_fingerprint": fingerprint,
338
+ "native_full_QA_graph_executed": True,
339
+ "native_final_prediction_extraction": True,
340
+ "independent_downstream_answer_call_added": False,
341
+ "native_generator_default_output_cap": 300,
342
+ "current_provider_usage_is_measured": raw["usage_is_complete"],
343
+ "original_historical_backbone_index_or_published_score_parity": False,
344
+ "automatic_retry": False,
345
+ "official_scoring": "caller_owned_independent_scorer",
346
+ },
347
+ )
348
+
349
+
350
+ __all__ = [
351
+ "IRCOT_NATIVE_QA_RELEASE",
352
+ "IRCoTNativeLikeSubstituted",
353
+ "IRCoTNativeQARequest",
354
+ "write_ircot_substitution_manifest",
355
+ ]
@@ -0,0 +1,327 @@
1
+ """Checksum-bound replay of original IRCoT ID-to-answer prediction files.
2
+
3
+ The archive remains operator-owned. Loading verifies bytes and declared input/
4
+ model bindings, not authorship or published-score parity. No prediction is
5
+ passed off as corpus evidence and no model request is made during replay.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Mapping
11
+ from dataclasses import dataclass, field
12
+ from pathlib import Path, PurePosixPath
13
+ from typing import Any
14
+
15
+ from agentinputlab.artifacts import load_pinned_json_mapping
16
+ from agentinputlab.contracts import (
17
+ AlgorithmOutputError,
18
+ ArtifactError,
19
+ CompositeResult,
20
+ ExecutionTrace,
21
+ FrozenAgentState,
22
+ InterventionContext,
23
+ SemanticFamily,
24
+ TraceEvent,
25
+ TracePhase,
26
+ ValidationError,
27
+ )
28
+ from agentinputlab.contracts.serialization import content_hash, freeze_mapping
29
+
30
+ IRCOT_REPLAY_RELEASE = "acquire.ircot.prediction_replay"
31
+ IRCOT_REPLAY_SCHEMA = "agentinputlab.ircot.prediction_replay.v1"
32
+
33
+
34
+ def _sha(value, label):
35
+ if (
36
+ not isinstance(value, str)
37
+ or len(value) != 64
38
+ or any(c not in "0123456789abcdef" for c in value)
39
+ ):
40
+ raise ArtifactError(f"IRCoT replay {label} must be a lowercase SHA-256")
41
+ return value
42
+
43
+
44
+ def _mounted_json(root, spec):
45
+ if not isinstance(spec, Mapping) or set(spec) != {"path", "sha256"}:
46
+ raise ArtifactError("IRCoT replay files require a relative path and trusted checksum")
47
+ name = spec["path"]
48
+ if not isinstance(name, str) or not name or "\\" in name:
49
+ raise ArtifactError("IRCoT replay file path is invalid")
50
+ path = PurePosixPath(name)
51
+ if path.is_absolute() or ".." in path.parts or not path.parts:
52
+ raise ArtifactError("IRCoT replay file path must remain under its manifest directory")
53
+ target = root.joinpath(*path.parts)
54
+ if not target.parent.resolve().is_relative_to(root):
55
+ raise ArtifactError("IRCoT replay file parent escapes its manifest directory")
56
+ return load_pinned_json_mapping(
57
+ target, expected_sha256=_sha(spec["sha256"], "file checksum"), max_bytes=8 * 1024 * 1024
58
+ )
59
+
60
+
61
+ @dataclass(frozen=True, slots=True)
62
+ class IRCoTPredictionArchive:
63
+ """Immutable selected scope, including empty predictions and historical cost."""
64
+
65
+ manifest_sha256: str
66
+ manifest: Mapping[str, Any]
67
+ records: Mapping[str, Any]
68
+ source_prediction_count: int
69
+
70
+ def __post_init__(self):
71
+ object.__setattr__(self, "manifest", freeze_mapping(self.manifest))
72
+ object.__setattr__(self, "records", freeze_mapping(self.records))
73
+
74
+ @property
75
+ def generator_fingerprint_sha256(self):
76
+ return content_hash(self.manifest["generator_fingerprint"])
77
+
78
+
79
+ def load_ircot_prediction_archive(manifest_path, *, expected_sha256):
80
+ """Load a bounded operator manifest plus an unchanged upstream-format file.
81
+
82
+ ``inputs`` is a label-free binding map, never a ground-truth answer file.
83
+ It may select a documented ID-based scope from a larger prediction file.
84
+ Unselected predictions remain counted in ``source_prediction_count``.
85
+ """
86
+ pin = _sha(expected_sha256, "manifest checksum")
87
+ path = Path(manifest_path).expanduser().absolute()
88
+ manifest = load_pinned_json_mapping(path, expected_sha256=pin, max_bytes=1024 * 1024)
89
+ required = {
90
+ "schema",
91
+ "origin_kind",
92
+ "run_revision",
93
+ "benchmark",
94
+ "split",
95
+ "generator_fingerprint",
96
+ "dependency_fingerprint",
97
+ "provenance",
98
+ "selection_rule",
99
+ "predictions",
100
+ "inputs",
101
+ }
102
+ if set(manifest) != required or manifest["schema"] != IRCOT_REPLAY_SCHEMA:
103
+ raise ArtifactError("IRCoT replay manifest schema/fields differ")
104
+ if manifest["origin_kind"] not in ("author_release", "controlled_capture"):
105
+ raise ArtifactError(
106
+ "IRCoT replay origin must be an author release or separately labelled controlled capture"
107
+ )
108
+ revision = manifest["run_revision"]
109
+ if (
110
+ not isinstance(revision, str)
111
+ or len(revision) != 40
112
+ or any(c not in "0123456789abcdef" for c in revision)
113
+ ):
114
+ raise ArtifactError(
115
+ "IRCoT replay run_revision must be the actual full immutable Git revision"
116
+ )
117
+ for key in ("benchmark", "split", "selection_rule"):
118
+ if not isinstance(manifest[key], str) or not manifest[key].strip():
119
+ raise ArtifactError(f"IRCoT replay requires {key}")
120
+ for key in ("generator_fingerprint", "dependency_fingerprint"):
121
+ if not isinstance(manifest[key], Mapping) or not manifest[key]:
122
+ raise ArtifactError(f"IRCoT replay requires {key}")
123
+ content_hash(manifest[key])
124
+ provenance = manifest["provenance"]
125
+ if not isinstance(provenance, Mapping) or set(provenance) != {
126
+ "source_uri",
127
+ "source_artifact_sha256",
128
+ "run_config_sha256",
129
+ "query_source_sha256",
130
+ }:
131
+ raise ArtifactError(
132
+ "IRCoT replay requires explicit source, configuration and query provenance"
133
+ )
134
+ if not isinstance(provenance["source_uri"], str) or not provenance["source_uri"].strip():
135
+ raise ArtifactError("IRCoT replay source_uri is required")
136
+ for key in ("source_artifact_sha256", "run_config_sha256", "query_source_sha256"):
137
+ _sha(provenance[key], key)
138
+ root = path.parent.resolve()
139
+ predictions = _mounted_json(root, manifest["predictions"])
140
+ bindings = _mounted_json(root, manifest["inputs"])
141
+ if not predictions or not bindings or not set(bindings) <= set(predictions):
142
+ raise ArtifactError("IRCoT replay selected IDs must exist in the original prediction file")
143
+ if any(
144
+ not isinstance(k, str) or not k or not isinstance(v, str) for k, v in predictions.items()
145
+ ):
146
+ raise ArtifactError(
147
+ "IRCoT answer replay requires the original ID-to-text format, not retrieval IDs or gold answer lists"
148
+ )
149
+ records = {}
150
+ for key, binding in bindings.items():
151
+ if not isinstance(binding, Mapping) or set(binding) != {
152
+ "query",
153
+ "input_state_hash",
154
+ "historical_usage",
155
+ "origin_receipt_sha256",
156
+ }:
157
+ raise ArtifactError("IRCoT replay input binding fields differ; labels are not allowed")
158
+ if not isinstance(binding["query"], str) or not binding["query"].strip():
159
+ raise ArtifactError("IRCoT replay input binding requires its exact source query")
160
+ for name in ("input_state_hash", "origin_receipt_sha256"):
161
+ if binding[name] is not None:
162
+ _sha(binding[name], name)
163
+ historical = binding["historical_usage"]
164
+ if historical is not None:
165
+ if not isinstance(historical, Mapping) or not historical:
166
+ raise ArtifactError(
167
+ "IRCoT replay historical_usage must be measured data or null, never a guessed zero"
168
+ )
169
+ from agentinputlab.observability import normalize_external_usage
170
+
171
+ normalize_external_usage(historical)
172
+ if (
173
+ manifest["origin_kind"] == "controlled_capture"
174
+ and binding["origin_receipt_sha256"] is None
175
+ ):
176
+ raise ArtifactError(
177
+ "IRCoT controlled capture requires the original inference receipt checksum"
178
+ )
179
+ records[key] = {
180
+ **dict(binding),
181
+ "answer": predictions[key],
182
+ "status": "archived_prediction"
183
+ if predictions[key].strip()
184
+ else "archived_empty_prediction",
185
+ }
186
+ return IRCoTPredictionArchive(pin, manifest, records, len(predictions))
187
+
188
+
189
+ @dataclass(frozen=True, slots=True)
190
+ class IRCoTPredictionReplay:
191
+ manifest_path: str | Path
192
+ expected_manifest_sha256: str
193
+ expected_generator_fingerprint_sha256: str
194
+ benchmark: str
195
+ split: str
196
+ method_id: str = "ircot"
197
+ release_id: str = IRCOT_REPLAY_RELEASE
198
+ family: SemanticFamily = SemanticFamily.ACQUIRE
199
+ _archive: IRCoTPredictionArchive = field(init=False, repr=False)
200
+
201
+ def __post_init__(self):
202
+ if (
203
+ self.method_id != "ircot"
204
+ or self.release_id != IRCOT_REPLAY_RELEASE
205
+ or self.family is not SemanticFamily.ACQUIRE
206
+ ):
207
+ raise ValidationError("IRCoT replay identity differs from the registered profile")
208
+ archive = load_ircot_prediction_archive(
209
+ self.manifest_path, expected_sha256=self.expected_manifest_sha256
210
+ )
211
+ if archive.generator_fingerprint_sha256 != _sha(
212
+ self.expected_generator_fingerprint_sha256, "expected generator"
213
+ ):
214
+ raise ArtifactError("IRCoT replay belongs to a different model/deployment")
215
+ if (
216
+ archive.manifest["benchmark"] != self.benchmark
217
+ or archive.manifest["split"] != self.split
218
+ ):
219
+ raise ArtifactError("IRCoT replay benchmark/split does not match the requested scope")
220
+ object.__setattr__(self, "_archive", archive)
221
+
222
+ @property
223
+ def archive(self):
224
+ """Inspect original empty outcomes/provenance without fabricating answers."""
225
+ return self._archive
226
+
227
+ def run(self, state: FrozenAgentState, context: InterventionContext) -> CompositeResult:
228
+ # Recheck trusted files before reuse; there is never a live fallback.
229
+ archive = load_ircot_prediction_archive(
230
+ self.manifest_path, expected_sha256=self.expected_manifest_sha256
231
+ )
232
+ if archive != self._archive:
233
+ raise ArtifactError("IRCoT replay archive changed after construction")
234
+ record = archive.records.get(state.task_id)
235
+ if record is None or record["query"] != state.query:
236
+ raise ArtifactError("IRCoT replay exact task ID/query mismatch; no generation fallback")
237
+ if (
238
+ record["input_state_hash"] is not None
239
+ and record["input_state_hash"] != state.state_hash
240
+ ):
241
+ raise ArtifactError("IRCoT replay original input-state hash differs")
242
+ for key, value in (("benchmark_cell", self.benchmark), ("source_split", self.split)):
243
+ if key in state.metadata and state.metadata[key] != value:
244
+ raise ArtifactError("IRCoT replay visible dataset scope differs")
245
+ if (
246
+ context.backend is not None
247
+ and content_hash(context.backend.fingerprint()) != archive.generator_fingerprint_sha256
248
+ ):
249
+ raise ArtifactError(
250
+ "IRCoT replay cannot relabel an archived answer as another live backbone"
251
+ )
252
+ if not record["answer"].strip():
253
+ raise AlgorithmOutputError(
254
+ "IRCoT archived empty prediction preserved; no retry or replacement answer"
255
+ )
256
+ manifest = archive.manifest
257
+ return CompositeResult(
258
+ method_id=self.method_id,
259
+ release_id=self.release_id,
260
+ source_state_hash=state.state_hash,
261
+ final_answer=record["answer"],
262
+ evidence=(),
263
+ trace=ExecutionTrace(
264
+ method_id=self.method_id,
265
+ release_id=self.release_id,
266
+ events=(
267
+ TraceEvent(
268
+ TracePhase.PREPARE,
269
+ "prediction_archive_binding",
270
+ {
271
+ "manifest_sha256": archive.manifest_sha256,
272
+ "source_prediction_count": archive.source_prediction_count,
273
+ "selected_record_count": len(archive.records),
274
+ "task_id": state.task_id,
275
+ "generator_fingerprint_sha256": archive.generator_fingerprint_sha256,
276
+ },
277
+ ),
278
+ TraceEvent(
279
+ TracePhase.FINALIZE,
280
+ "archived_prediction",
281
+ {
282
+ "answer": record["answer"],
283
+ "origin_receipt_sha256": record["origin_receipt_sha256"],
284
+ "historical_usage": record["historical_usage"],
285
+ },
286
+ ),
287
+ ),
288
+ counters={
289
+ "model_calls": 0,
290
+ "retrieval_calls": 0,
291
+ "input_tokens": 0,
292
+ "output_tokens": 0,
293
+ },
294
+ ),
295
+ implementation_identity=IRCOT_REPLAY_SCHEMA,
296
+ profile_kind="official_native"
297
+ if manifest["origin_kind"] == "author_release"
298
+ else "controlled_adapter",
299
+ metadata={
300
+ "artifact_replay": True,
301
+ "origin_kind": manifest["origin_kind"],
302
+ "run_revision": manifest["run_revision"],
303
+ "manifest_sha256": archive.manifest_sha256,
304
+ "generator_fingerprint": manifest["generator_fingerprint"],
305
+ "dependency_fingerprint": manifest["dependency_fingerprint"],
306
+ "provenance": manifest["provenance"],
307
+ "historical_usage": record["historical_usage"],
308
+ "historical_cost_recharged": False,
309
+ "historical_usage_unknown": record["historical_usage"] is None,
310
+ "new_provider_requests": 0,
311
+ "current_seed_or_budget_rerun_claimed": False,
312
+ "input_state_hash_verified": record["input_state_hash"] is not None,
313
+ "answers_are_not_corpus_evidence": True,
314
+ "checksum_proves_authorship": False,
315
+ "benchmark_scoring": "caller_owned_independent_scorer",
316
+ "automatic_retry": False,
317
+ },
318
+ )
319
+
320
+
321
+ __all__ = [
322
+ "IRCOT_REPLAY_RELEASE",
323
+ "IRCOT_REPLAY_SCHEMA",
324
+ "IRCoTPredictionArchive",
325
+ "IRCoTPredictionReplay",
326
+ "load_ircot_prediction_archive",
327
+ ]