agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,727 @@
1
+ """Machine-readable, fail-closed readiness checks for the v0.2 beta gate."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections import Counter
7
+ from collections.abc import Mapping
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from agentinputlab.contracts.enums import ProfileKind, ReleaseTier
13
+ from agentinputlab.contracts.serialization import content_hash
14
+
15
+ from .loader import load_registry_file
16
+ from .result_manifest import validate_result_table_manifest
17
+ from .runtime import AlgorithmRegistry
18
+
19
+
20
+ @dataclass(frozen=True, slots=True)
21
+ class BetaGateCheck:
22
+ """One independently auditable beta release criterion."""
23
+
24
+ check_id: str
25
+ passed: bool
26
+ detail: str
27
+ blockers: tuple[str, ...] = ()
28
+
29
+ def to_dict(self) -> dict[str, Any]:
30
+ return {
31
+ "check_id": self.check_id,
32
+ "status": "PASS" if self.passed else "BLOCKED",
33
+ "passed": self.passed,
34
+ "detail": self.detail,
35
+ "blockers": list(self.blockers),
36
+ }
37
+
38
+
39
+ @dataclass(frozen=True, slots=True)
40
+ class BetaReadinessReport:
41
+ """Deterministic summary suitable for CLI output and release artifacts."""
42
+
43
+ gate_id: str
44
+ manifest_sha256: str
45
+ checks: tuple[BetaGateCheck, ...]
46
+ schema_version: int = 1
47
+
48
+ @property
49
+ def ready(self) -> bool:
50
+ return all(item.passed for item in self.checks)
51
+
52
+ @property
53
+ def blockers(self) -> tuple[str, ...]:
54
+ return tuple(
55
+ f"{item.check_id}: {blocker}" for item in self.checks for blocker in item.blockers
56
+ )
57
+
58
+ def to_dict(self) -> dict[str, Any]:
59
+ return {
60
+ "schema_version": self.schema_version,
61
+ "gate_id": self.gate_id,
62
+ "status": "READY" if self.ready else "BLOCKED",
63
+ "ready": self.ready,
64
+ "manifest_sha256": self.manifest_sha256,
65
+ "checks": [item.to_dict() for item in self.checks],
66
+ "blockers": list(self.blockers),
67
+ }
68
+
69
+
70
+ def _blocked(check_id: str, detail: str, *blockers: str) -> BetaGateCheck:
71
+ return BetaGateCheck(
72
+ check_id=check_id,
73
+ passed=False,
74
+ detail=detail,
75
+ blockers=tuple(blockers) or ("criterion was not satisfied",),
76
+ )
77
+
78
+
79
+ def _passed(check_id: str, detail: str) -> BetaGateCheck:
80
+ return BetaGateCheck(check_id=check_id, passed=True, detail=detail)
81
+
82
+
83
+ def _repository_path(
84
+ root: Path,
85
+ value: object,
86
+ label: str,
87
+ *,
88
+ reject_symlinks: bool = False,
89
+ ) -> tuple[Path | None, str | None]:
90
+ if not isinstance(value, str) or not value.strip():
91
+ return None, f"{label} must be a non-empty repository-relative path"
92
+ relative = Path(value)
93
+ if relative.is_absolute() or ".." in relative.parts:
94
+ return None, f"{label} must be a repository-relative path without parent traversal"
95
+ candidate = root / relative
96
+ if reject_symlinks:
97
+ current = root
98
+ for part in relative.parts:
99
+ current /= part
100
+ if current.is_symlink():
101
+ return None, f"{label} must not contain a symbolic link: {current}"
102
+ candidate = candidate.resolve()
103
+ try:
104
+ candidate.relative_to(root)
105
+ except ValueError:
106
+ return None, f"{label} escapes the repository root"
107
+ return candidate, None
108
+
109
+
110
+ def _check_roster(
111
+ registry: AlgorithmRegistry,
112
+ manifest: Mapping[str, Any],
113
+ ) -> BetaGateCheck:
114
+ expected = manifest.get("required_roster")
115
+ if not isinstance(expected, Mapping):
116
+ return _blocked("roster.20_plus_7", "Frozen algorithm roster", "required_roster missing")
117
+ core_expected = expected.get("core")
118
+ optional_expected = expected.get("optional")
119
+ methods = registry.document.algorithms
120
+ core_actual = sum(item.release_tier is ReleaseTier.CORE for item in methods)
121
+ optional_actual = sum(item.release_tier is ReleaseTier.OPTIONAL for item in methods)
122
+ blockers: list[str] = []
123
+ if core_expected != core_actual:
124
+ blockers.append(f"core roster expected {core_expected!r}, found {core_actual}")
125
+ if optional_expected != optional_actual:
126
+ blockers.append(f"optional roster expected {optional_expected!r}, found {optional_actual}")
127
+ if blockers:
128
+ return _blocked("roster.20_plus_7", "Frozen algorithm roster", *blockers)
129
+ return _passed(
130
+ "roster.20_plus_7",
131
+ f"registry contains {core_actual} core and {optional_actual} optional methods",
132
+ )
133
+
134
+
135
+ def _check_l2_paths(
136
+ root: Path,
137
+ registry: AlgorithmRegistry,
138
+ manifest: Mapping[str, Any],
139
+ ) -> BetaGateCheck:
140
+ raw_paths = manifest.get("l2_paths")
141
+ minimum = manifest.get("minimum_l2_methods_per_family")
142
+ raw_required_families = manifest.get("required_l2_families")
143
+ if not isinstance(raw_paths, list) or not raw_paths:
144
+ return _blocked(
145
+ "l2.family_coverage",
146
+ "Resolvable documented L2 paths",
147
+ "l2_paths must be a non-empty array",
148
+ )
149
+ if not isinstance(minimum, int) or isinstance(minimum, bool) or minimum < 1:
150
+ return _blocked(
151
+ "l2.family_coverage",
152
+ "Resolvable documented L2 paths",
153
+ "minimum_l2_methods_per_family must be a positive integer",
154
+ )
155
+ if (
156
+ not isinstance(raw_required_families, list)
157
+ or not raw_required_families
158
+ or any(not isinstance(item, str) or not item for item in raw_required_families)
159
+ ):
160
+ return _blocked(
161
+ "l2.family_coverage",
162
+ "Resolvable documented L2 paths",
163
+ "required_l2_families must be a non-empty array of family names",
164
+ )
165
+ required_families = list(dict.fromkeys(raw_required_families))
166
+ if len(required_families) != len(raw_required_families):
167
+ return _blocked(
168
+ "l2.family_coverage",
169
+ "Resolvable documented L2 paths",
170
+ "required_l2_families must not contain duplicates",
171
+ )
172
+
173
+ blockers: list[str] = []
174
+ declared_methods: list[str] = []
175
+ family_counts: Counter[str] = Counter()
176
+ registry_method_ids = {item.method_id for item in registry.document.algorithms}
177
+ for index, raw in enumerate(raw_paths):
178
+ label = f"l2_paths[{index}]"
179
+ if not isinstance(raw, Mapping):
180
+ blockers.append(f"{label} must be an object")
181
+ continue
182
+ method_id = raw.get("method_id")
183
+ release_id = raw.get("release_id")
184
+ if not isinstance(method_id, str) or not method_id:
185
+ blockers.append(f"{label}.method_id must be non-empty")
186
+ continue
187
+ declared_methods.append(method_id)
188
+ if raw.get("evidence_level") != "L2":
189
+ blockers.append(f"{label}.evidence_level must equal L2")
190
+ evidence_path, path_error = _repository_path(
191
+ root,
192
+ raw.get("evidence_path"),
193
+ f"{label}.evidence_path",
194
+ )
195
+ if path_error is not None:
196
+ blockers.append(path_error)
197
+ elif evidence_path is not None and not evidence_path.is_file():
198
+ blockers.append(f"{label}.evidence_path does not exist")
199
+ if not isinstance(release_id, str) or not release_id:
200
+ blockers.append(f"{label}.release_id must be non-empty")
201
+ continue
202
+ try:
203
+ method, release = registry.get_release(release_id)
204
+ except Exception as exc:
205
+ blockers.append(f"{label} references unknown release {release_id!r}: {exc}")
206
+ continue
207
+ if method.method_id != method_id:
208
+ blockers.append(
209
+ f"{label} release belongs to {method.method_id}, not declared {method_id}"
210
+ )
211
+ continue
212
+ if not release.entrypoint_available:
213
+ blockers.append(f"{label} release has no integration entrypoint")
214
+ continue
215
+ try:
216
+ resolved = registry.resolve_entrypoint(release_id)
217
+ except Exception as exc:
218
+ blockers.append(f"{label} entrypoint cannot resolve: {exc}")
219
+ continue
220
+ if not callable(resolved):
221
+ blockers.append(f"{label} entrypoint is not callable")
222
+ continue
223
+ family_counts[method.semantic_family.value] += 1
224
+
225
+ duplicates = sorted(
226
+ method_id for method_id, count in Counter(declared_methods).items() if count > 1
227
+ )
228
+ if duplicates:
229
+ blockers.append(f"duplicate L2 method declarations: {', '.join(duplicates)}")
230
+ missing = sorted(registry_method_ids - set(declared_methods))
231
+ extra = sorted(set(declared_methods) - registry_method_ids)
232
+ if missing:
233
+ blockers.append(f"L2 manifest omits methods: {', '.join(missing)}")
234
+ if extra:
235
+ blockers.append(f"L2 manifest declares unknown methods: {', '.join(extra)}")
236
+
237
+ registry_families = {item.semantic_family.value for item in registry.document.algorithms}
238
+ unknown_required_families = sorted(set(required_families) - registry_families)
239
+ if unknown_required_families:
240
+ blockers.append(
241
+ "required_l2_families contains unknown families: "
242
+ + ", ".join(unknown_required_families)
243
+ )
244
+ for family in required_families:
245
+ count = family_counts[family]
246
+ if count < minimum:
247
+ blockers.append(
248
+ f"family {family} has {count} resolvable L2 methods; requires at least {minimum}"
249
+ )
250
+ if blockers:
251
+ return _blocked(
252
+ "l2.family_coverage",
253
+ "Resolvable documented L2 paths",
254
+ *blockers,
255
+ )
256
+ rendered = ", ".join(f"{family}={family_counts[family]}" for family in required_families)
257
+ return _passed(
258
+ "l2.family_coverage",
259
+ f"{len(declared_methods)} methods have resolvable L2 paths; {rendered}",
260
+ )
261
+
262
+
263
+ def _check_selectors(manifest: Mapping[str, Any]) -> BetaGateCheck:
264
+ minimum = manifest.get("minimum_selector_count")
265
+ if not isinstance(minimum, int) or isinstance(minimum, bool) or minimum < 1:
266
+ return _blocked(
267
+ "selector.catalog",
268
+ "Selector catalog size",
269
+ "minimum_selector_count must be a positive integer",
270
+ )
271
+ try:
272
+ from agentinputlab.selectors import list_selector_cards
273
+
274
+ cards = list_selector_cards()
275
+ except Exception as exc:
276
+ return _blocked(
277
+ "selector.catalog",
278
+ "Selector catalog size",
279
+ f"selector catalog could not be loaded: {exc}",
280
+ )
281
+ if len(cards) < minimum:
282
+ return _blocked(
283
+ "selector.catalog",
284
+ "Selector catalog size",
285
+ f"found {len(cards)} selectors; requires at least {minimum}",
286
+ )
287
+ return _passed(
288
+ "selector.catalog",
289
+ f"{len(cards)} selector cards are registered (minimum {minimum})",
290
+ )
291
+
292
+
293
+ def _check_algorithm_licenses(registry: AlgorithmRegistry) -> BetaGateCheck:
294
+ blockers: list[str] = []
295
+ for method in registry.document.algorithms:
296
+ upstream = method.upstream
297
+ if not upstream.license_id.strip():
298
+ blockers.append(f"{method.method_id} has no source license identity")
299
+ if upstream.license_id == "NOASSERTION" and upstream.vendoring_allowed:
300
+ blockers.append(f"{method.method_id} may not vendor a NOASSERTION source")
301
+ if blockers:
302
+ return _blocked(
303
+ "license.algorithm_boundaries",
304
+ "Algorithm source-license boundaries",
305
+ *blockers,
306
+ )
307
+ noassertion_count = sum(
308
+ item.upstream.license_id == "NOASSERTION" for item in registry.document.algorithms
309
+ )
310
+ return _passed(
311
+ "license.algorithm_boundaries",
312
+ f"all source licenses are explicit; {noassertion_count} NOASSERTION sources remain external",
313
+ )
314
+
315
+
316
+ def _validate_benchmark_card(
317
+ raw: Any,
318
+ *,
319
+ expected_cell_id: str,
320
+ registry_cell: Mapping[str, Any],
321
+ label: str,
322
+ ) -> tuple[str, ...]:
323
+ if not isinstance(raw, Mapping):
324
+ return (f"{label} must contain an object",)
325
+ blockers: list[str] = []
326
+ if raw.get("schema_version") != 1:
327
+ blockers.append(f"{label}.schema_version must equal 1")
328
+ if raw.get("cell_id") != expected_cell_id:
329
+ blockers.append(f"{label}.cell_id must equal {expected_cell_id}")
330
+ for key in ("source_url", "source_revision", "license_id"):
331
+ value = raw.get(key)
332
+ if not isinstance(value, str) or not value.strip():
333
+ blockers.append(f"{label}.{key} must be a non-empty string")
334
+ if raw.get("source_revision") != registry_cell.get("source_revision"):
335
+ blockers.append(f"{label}.source_revision does not match benchmark registry")
336
+ vendoring_allowed = raw.get("vendoring_allowed")
337
+ if not isinstance(vendoring_allowed, bool):
338
+ blockers.append(f"{label}.vendoring_allowed must be a boolean")
339
+ if raw.get("license_id") == "NOASSERTION" and vendoring_allowed is not False:
340
+ blockers.append(f"{label} cannot vendor a NOASSERTION source")
341
+
342
+ scorer = raw.get("scorer")
343
+ if not isinstance(scorer, Mapping):
344
+ blockers.append(f"{label}.scorer must be an object")
345
+ else:
346
+ for key in ("identity", "revision"):
347
+ value = scorer.get(key)
348
+ if not isinstance(value, str) or not value.strip():
349
+ blockers.append(f"{label}.scorer.{key} must be a non-empty string")
350
+
351
+ gold = raw.get("gold_boundary")
352
+ if not isinstance(gold, Mapping):
353
+ blockers.append(f"{label}.gold_boundary must be an object")
354
+ else:
355
+ if gold.get("optimizer_visible") is not False:
356
+ blockers.append(f"{label}.gold_boundary.optimizer_visible must be false")
357
+ if gold.get("evaluator_only") is not True:
358
+ blockers.append(f"{label}.gold_boundary.evaluator_only must be true")
359
+ return tuple(blockers)
360
+
361
+
362
+ def _check_benchmark_cards(
363
+ root: Path,
364
+ manifest: Mapping[str, Any],
365
+ ) -> BetaGateCheck:
366
+ registry_path, path_error = _repository_path(
367
+ root,
368
+ manifest.get("benchmark_registry"),
369
+ "benchmark_registry",
370
+ )
371
+ if path_error is not None or registry_path is None:
372
+ return _blocked(
373
+ "benchmark.cards",
374
+ "Benchmark source/license/scorer/gold cards",
375
+ path_error or "benchmark_registry is invalid",
376
+ )
377
+ try:
378
+ registry_raw = json.loads(registry_path.read_text(encoding="utf-8"))
379
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
380
+ return _blocked(
381
+ "benchmark.cards",
382
+ "Benchmark source/license/scorer/gold cards",
383
+ f"cannot load benchmark registry: {exc}",
384
+ )
385
+ raw_cells = registry_raw.get("cells") if isinstance(registry_raw, Mapping) else None
386
+ if not isinstance(raw_cells, list):
387
+ return _blocked(
388
+ "benchmark.cards",
389
+ "Benchmark source/license/scorer/gold cards",
390
+ "benchmark registry cells must be an array",
391
+ )
392
+ cells = {
393
+ item.get("cell_id"): item
394
+ for item in raw_cells
395
+ if isinstance(item, Mapping) and isinstance(item.get("cell_id"), str)
396
+ }
397
+
398
+ requirements = manifest.get("required_benchmark_cards")
399
+ if not isinstance(requirements, list) or not requirements:
400
+ return _blocked(
401
+ "benchmark.cards",
402
+ "Benchmark source/license/scorer/gold cards",
403
+ "required_benchmark_cards must be a non-empty array",
404
+ )
405
+ blockers: list[str] = []
406
+ declared: list[str] = []
407
+ for index, requirement in enumerate(requirements):
408
+ label = f"required_benchmark_cards[{index}]"
409
+ if not isinstance(requirement, Mapping):
410
+ blockers.append(f"{label} must be an object")
411
+ continue
412
+ cell_id = requirement.get("cell_id")
413
+ if not isinstance(cell_id, str) or not cell_id:
414
+ blockers.append(f"{label}.cell_id must be non-empty")
415
+ continue
416
+ declared.append(cell_id)
417
+ registry_cell = cells.get(cell_id)
418
+ if registry_cell is None:
419
+ blockers.append(f"{cell_id} is absent from the benchmark registry")
420
+ continue
421
+ card_path, card_error = _repository_path(
422
+ root,
423
+ requirement.get("path"),
424
+ f"{label}.path",
425
+ )
426
+ if card_error is not None or card_path is None:
427
+ blockers.append(card_error or f"{label}.path is invalid")
428
+ continue
429
+ if not card_path.is_file():
430
+ blockers.append(f"{cell_id} benchmark card is missing at {card_path.relative_to(root)}")
431
+ continue
432
+ try:
433
+ card = json.loads(card_path.read_text(encoding="utf-8"))
434
+ except (OSError, json.JSONDecodeError) as exc:
435
+ blockers.append(f"{cell_id} benchmark card cannot be loaded: {exc}")
436
+ continue
437
+ blockers.extend(
438
+ _validate_benchmark_card(
439
+ card,
440
+ expected_cell_id=cell_id,
441
+ registry_cell=registry_cell,
442
+ label=f"benchmark card {cell_id}",
443
+ )
444
+ )
445
+ duplicates = sorted(cell_id for cell_id, count in Counter(declared).items() if count > 1)
446
+ if duplicates:
447
+ blockers.append(f"duplicate benchmark card requirements: {', '.join(duplicates)}")
448
+ if blockers:
449
+ return _blocked(
450
+ "benchmark.cards",
451
+ "Benchmark source/license/scorer/gold cards",
452
+ *blockers,
453
+ )
454
+ return _passed(
455
+ "benchmark.cards",
456
+ f"{len(declared)} required benchmark cards have frozen source, license, scorer, and gold boundaries",
457
+ )
458
+
459
+
460
+ def _check_result_profile_policy(
461
+ root: Path,
462
+ manifest: Mapping[str, Any],
463
+ ) -> BetaGateCheck:
464
+ policy = manifest.get("result_table_policy")
465
+ if not isinstance(policy, Mapping):
466
+ return _blocked(
467
+ "results.profile_boundary",
468
+ "Single-profile-kind result tables",
469
+ "result_table_policy missing",
470
+ )
471
+ blockers: list[str] = []
472
+ if policy.get("require_single_profile_kind") is not True:
473
+ blockers.append("result_table_policy.require_single_profile_kind must be true")
474
+ allowed = policy.get("allowed_profile_kinds")
475
+ expected = {item.value for item in ProfileKind}
476
+ if not isinstance(allowed, list) or set(allowed) != expected:
477
+ blockers.append(
478
+ "result_table_policy.allowed_profile_kinds must contain exactly "
479
+ + ", ".join(sorted(expected))
480
+ )
481
+ if policy.get("validator") != "agentinputlab.registry:validate_result_table_manifest":
482
+ blockers.append("result_table_policy.validator must name the shipped validator")
483
+ minimum_manifests = policy.get("minimum_manifests")
484
+ if (
485
+ isinstance(minimum_manifests, bool)
486
+ or not isinstance(minimum_manifests, int)
487
+ or minimum_manifests < 1
488
+ ):
489
+ blockers.append("result_table_policy.minimum_manifests must be a positive integer")
490
+ minimum_manifests = 1
491
+ manifest_glob = policy.get("manifest_glob")
492
+ if not isinstance(manifest_glob, str) or not manifest_glob.strip():
493
+ blockers.append("result_table_policy.manifest_glob must be non-empty")
494
+ result_paths: list[Path] = []
495
+ elif Path(manifest_glob).is_absolute() or ".." in Path(manifest_glob).parts:
496
+ blockers.append("result_table_policy.manifest_glob must remain repository-relative")
497
+ result_paths = []
498
+ else:
499
+ result_paths = sorted(root.glob(manifest_glob))
500
+ if len(result_paths) < minimum_manifests:
501
+ blockers.append(
502
+ "result-table manifest inventory is incomplete: "
503
+ f"found {len(result_paths)}, requires at least {minimum_manifests}"
504
+ )
505
+ seen_table_ids: dict[str, Path] = {}
506
+ seen_run_identities: dict[str, Path] = {}
507
+ for path in result_paths:
508
+ try:
509
+ relative_manifest_path = path.relative_to(root)
510
+ except ValueError:
511
+ blockers.append(f"result-table manifest path escapes repository root: {path}")
512
+ continue
513
+ checked_path, checked_path_error = _repository_path(
514
+ root,
515
+ relative_manifest_path.as_posix(),
516
+ f"result-table manifest {relative_manifest_path}",
517
+ reject_symlinks=True,
518
+ )
519
+ if checked_path_error is not None or checked_path is None:
520
+ blockers.append(
521
+ checked_path_error
522
+ or f"result-table manifest {relative_manifest_path} is invalid"
523
+ )
524
+ continue
525
+ if not checked_path.is_file():
526
+ blockers.append(
527
+ f"result-table manifest {relative_manifest_path} is not a regular file"
528
+ )
529
+ continue
530
+ try:
531
+ manifest_bytes = checked_path.read_bytes()
532
+ raw = json.loads(manifest_bytes)
533
+ table = validate_result_table_manifest(raw)
534
+ except Exception as exc:
535
+ blockers.append(f"{relative_manifest_path} violates profile boundary: {exc}")
536
+ continue
537
+ previous_table = seen_table_ids.get(table.table_id)
538
+ if previous_table is not None:
539
+ blockers.append(
540
+ f"{relative_manifest_path} duplicates table_id {table.table_id!r} "
541
+ f"already declared by {previous_table}"
542
+ )
543
+ else:
544
+ seen_table_ids[table.table_id] = relative_manifest_path
545
+ for study in table.studies:
546
+ previous_run = seen_run_identities.get(study.run_identity_hash)
547
+ if previous_run is not None:
548
+ blockers.append(
549
+ f"{relative_manifest_path} duplicates run_identity_hash "
550
+ f"{study.run_identity_hash} already referenced by {previous_run}"
551
+ )
552
+ else:
553
+ seen_run_identities[study.run_identity_hash] = relative_manifest_path
554
+ run_path, run_path_error = _repository_path(
555
+ root,
556
+ study.run_path,
557
+ f"{relative_manifest_path} study {study.study_id} run_path",
558
+ reject_symlinks=True,
559
+ )
560
+ if run_path_error is not None or run_path is None:
561
+ blockers.append(run_path_error or f"{study.study_id} run_path is invalid")
562
+ continue
563
+ try:
564
+ from agentinputlab.experiments import verify_experiment_run_directory
565
+
566
+ verified = verify_experiment_run_directory(run_path)
567
+ except Exception as exc:
568
+ blockers.append(
569
+ f"{relative_manifest_path} study {study.study_id} "
570
+ f"does not reference a verified run: {exc}"
571
+ )
572
+ continue
573
+ expected = {
574
+ "study_id": study.study_id,
575
+ "profile_kind": study.profile_kind.value,
576
+ "comparison_profile_hash": study.profile_hash,
577
+ "identity_hash": study.run_identity_hash,
578
+ "manifest_hash": study.manifest_hash,
579
+ "plan_hash": study.plan_hash,
580
+ "driver_hash": study.driver_hash,
581
+ }
582
+ mismatches = [key for key, value in expected.items() if verified.get(key) != value]
583
+ if mismatches:
584
+ blockers.append(
585
+ f"{relative_manifest_path} study {study.study_id} verified "
586
+ "run identity differs for: " + ", ".join(sorted(mismatches))
587
+ )
588
+ if not verified.get("complete") or not verified.get("all_succeeded"):
589
+ blockers.append(
590
+ f"{relative_manifest_path} study {study.study_id} "
591
+ "run is incomplete or contains failed cells"
592
+ )
593
+ if verified.get("is_shard"):
594
+ blockers.append(
595
+ f"{relative_manifest_path} study {study.study_id} "
596
+ "references a shard rather than a complete parent plan"
597
+ )
598
+ if blockers:
599
+ return _blocked(
600
+ "results.profile_boundary",
601
+ "Single-profile-kind result tables",
602
+ *blockers,
603
+ )
604
+ return _passed(
605
+ "results.profile_boundary",
606
+ f"profile-kind policy is enforced; {len(result_paths)} result-table manifests validated",
607
+ )
608
+
609
+
610
+ def check_beta_readiness(
611
+ root: str | Path,
612
+ *,
613
+ manifest_path: str | Path | None = None,
614
+ registry: AlgorithmRegistry | None = None,
615
+ ) -> BetaReadinessReport:
616
+ """Evaluate integrity-oriented beta gates without claiming execution authenticity."""
617
+
618
+ supplied_root = Path(root).expanduser().absolute()
619
+ repository_root = supplied_root.resolve()
620
+ supplied_manifest = (
621
+ Path(manifest_path).expanduser()
622
+ if manifest_path is not None
623
+ else Path("manifests") / "releases" / "v0.2-beta.json"
624
+ )
625
+ if supplied_manifest.is_absolute():
626
+ try:
627
+ relative_manifest = supplied_manifest.relative_to(supplied_root)
628
+ except ValueError:
629
+ try:
630
+ relative_manifest = supplied_manifest.relative_to(repository_root)
631
+ except ValueError:
632
+ relative_manifest = None
633
+ else:
634
+ relative_manifest = supplied_manifest
635
+ if relative_manifest is None:
636
+ selected_manifest = supplied_manifest
637
+ selected_manifest_error = "beta release manifest escapes the repository root"
638
+ else:
639
+ selected_manifest, selected_manifest_error = _repository_path(
640
+ repository_root,
641
+ relative_manifest.as_posix(),
642
+ "beta release manifest",
643
+ reject_symlinks=True,
644
+ )
645
+ if selected_manifest_error is not None or selected_manifest is None:
646
+ return BetaReadinessReport(
647
+ gate_id="v0.2-beta",
648
+ manifest_sha256="",
649
+ checks=(
650
+ _blocked(
651
+ "manifest.schema",
652
+ "Beta release manifest",
653
+ selected_manifest_error or "beta release manifest path is invalid",
654
+ ),
655
+ ),
656
+ )
657
+ try:
658
+ manifest_bytes = selected_manifest.read_bytes()
659
+ raw = json.loads(manifest_bytes)
660
+ except (OSError, json.JSONDecodeError) as exc:
661
+ return BetaReadinessReport(
662
+ gate_id="v0.2-beta",
663
+ manifest_sha256="",
664
+ checks=(
665
+ _blocked(
666
+ "manifest.schema",
667
+ "Beta release manifest",
668
+ f"cannot load {selected_manifest}: {exc}",
669
+ ),
670
+ ),
671
+ )
672
+ if not isinstance(raw, Mapping):
673
+ return BetaReadinessReport(
674
+ gate_id="v0.2-beta",
675
+ manifest_sha256=content_hash(raw),
676
+ checks=(
677
+ _blocked(
678
+ "manifest.schema",
679
+ "Beta release manifest",
680
+ "manifest root must be an object",
681
+ ),
682
+ ),
683
+ )
684
+ gate_id = raw.get("gate_id")
685
+ schema_version = raw.get("schema_version")
686
+ manifest_blockers: list[str] = []
687
+ if schema_version != 1:
688
+ manifest_blockers.append("schema_version must equal 1")
689
+ if gate_id != "v0.2-beta":
690
+ manifest_blockers.append("gate_id must equal v0.2-beta")
691
+ if manifest_blockers:
692
+ return BetaReadinessReport(
693
+ gate_id=gate_id if isinstance(gate_id, str) else "v0.2-beta",
694
+ manifest_sha256=content_hash(raw),
695
+ checks=(
696
+ _blocked(
697
+ "manifest.schema",
698
+ "Beta release manifest",
699
+ *manifest_blockers,
700
+ ),
701
+ ),
702
+ )
703
+
704
+ active_registry = registry
705
+ if active_registry is None:
706
+ source_registry = repository_root / "manifests" / "algorithms" / "registry.yaml"
707
+ active_registry = AlgorithmRegistry(load_registry_file(source_registry))
708
+ checks = (
709
+ _check_roster(active_registry, raw),
710
+ _check_l2_paths(repository_root, active_registry, raw),
711
+ _check_selectors(raw),
712
+ _check_algorithm_licenses(active_registry),
713
+ _check_benchmark_cards(repository_root, raw),
714
+ _check_result_profile_policy(repository_root, raw),
715
+ )
716
+ return BetaReadinessReport(
717
+ gate_id=gate_id,
718
+ manifest_sha256=content_hash(raw),
719
+ checks=checks,
720
+ )
721
+
722
+
723
+ __all__ = [
724
+ "BetaGateCheck",
725
+ "BetaReadinessReport",
726
+ "check_beta_readiness",
727
+ ]