agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,50 @@
1
+ """Outcome, harm, drift, leakage, cost, and family-process metrics."""
2
+
3
+ from .catalog import METRIC_CATALOG, MetricDefinition, metric_definition
4
+ from .harm import (
5
+ STRICT_HARD_HARM_DEFINITION,
6
+ STRICT_HARD_HARM_PROTOCOL,
7
+ HardHarmAssessment,
8
+ assess_hard_harm,
9
+ assess_hard_harm_from_metrics,
10
+ )
11
+ from .paired import PairedSummary, summarize_pairs
12
+ from .qa import (
13
+ best_reference_score,
14
+ english_qa_exact_match,
15
+ english_qa_f1,
16
+ normalize_english_answer,
17
+ set_f1,
18
+ )
19
+ from .text import compression_ratio, intent_token_recall, token_jaccard
20
+ from .third_party import (
21
+ RougeScore,
22
+ evaluation_dependency_versions,
23
+ retrieval_aggregates,
24
+ rouge_l,
25
+ )
26
+
27
+ __all__ = [
28
+ "METRIC_CATALOG",
29
+ "STRICT_HARD_HARM_DEFINITION",
30
+ "STRICT_HARD_HARM_PROTOCOL",
31
+ "HardHarmAssessment",
32
+ "MetricDefinition",
33
+ "PairedSummary",
34
+ "RougeScore",
35
+ "assess_hard_harm",
36
+ "assess_hard_harm_from_metrics",
37
+ "best_reference_score",
38
+ "compression_ratio",
39
+ "english_qa_exact_match",
40
+ "english_qa_f1",
41
+ "evaluation_dependency_versions",
42
+ "intent_token_recall",
43
+ "metric_definition",
44
+ "normalize_english_answer",
45
+ "retrieval_aggregates",
46
+ "rouge_l",
47
+ "set_f1",
48
+ "summarize_pairs",
49
+ "token_jaccard",
50
+ ]
@@ -0,0 +1,313 @@
1
+ """Machine-readable ownership and readiness of benchmark metric names."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+
8
+ @dataclass(frozen=True, slots=True)
9
+ class MetricDefinition:
10
+ name: str
11
+ owner: str
12
+ status: str
13
+ implementation: str
14
+ reference: str
15
+ note: str = ""
16
+
17
+
18
+ _IR_MEASURES_REFERENCE = "MacAvaney et al., ECIR 2022; https://ir-measur.es/"
19
+ _ROUGE_REFERENCE = "Google Research rouge-score 0.1.2"
20
+ _QA_REFERENCE = "SQuAD/Hotpot-style normalized English token overlap"
21
+
22
+ METRIC_CATALOG: tuple[MetricDefinition, ...] = (
23
+ MetricDefinition("ndcg", "third_party", "implemented", "ir_measures", _IR_MEASURES_REFERENCE),
24
+ MetricDefinition(
25
+ "ndcg_at_10", "third_party", "implemented", "ir_measures", _IR_MEASURES_REFERENCE
26
+ ),
27
+ MetricDefinition("map", "third_party", "implemented", "ir_measures", _IR_MEASURES_REFERENCE),
28
+ MetricDefinition(
29
+ "recall", "benchmark_adapter", "implemented",
30
+ "ir_measures for retrieval; ToolTalk official conversation evaluator for ToolTalk",
31
+ _IR_MEASURES_REFERENCE + "; ToolTalk pinned ToolExecutor",
32
+ "The benchmark/scorer identity disambiguates document recall from tool-call recall.",
33
+ ),
34
+ *(
35
+ MetricDefinition(
36
+ name, "benchmark_adapter", "external_required",
37
+ "workers.official_arenas.providers.tooltalk:ToolTalkProvider",
38
+ "ToolTalk e05f4ce6132c80ed33392b81535b077d56ab28fd ToolExecutor.evaluate_predictions",
39
+ "Requires the pinned official runtime and scorer; registration is not an acceptance result.",
40
+ )
41
+ for name in ("official_success", "precision", "action_precision", "bad_action_rate", "soft_success")
42
+ ),
43
+ MetricDefinition(
44
+ "adapter_hard_stops", "benchmark_adapter", "implemented",
45
+ "workers.official_arenas.providers.tooltalk:_Predictor",
46
+ "ToolTalk adapter generation controls",
47
+ "Diagnostic stop count, distinct from native official_success; not a task-success metric.",
48
+ ),
49
+ MetricDefinition("rouge", "third_party", "implemented", "rouge_score", _ROUGE_REFERENCE),
50
+ MetricDefinition(
51
+ "answer_f1",
52
+ "first_party",
53
+ "implemented",
54
+ "english_qa_f1 + best_reference_score",
55
+ _QA_REFERENCE,
56
+ "A benchmark adapter must still freeze reference selection and language-specific rules.",
57
+ ),
58
+ MetricDefinition(
59
+ "support_f1",
60
+ "first_party",
61
+ "implemented",
62
+ "set_f1",
63
+ "Exact set precision/recall/F1",
64
+ "The adapter defines the canonical support identifier.",
65
+ ),
66
+ MetricDefinition(
67
+ "evidence_f1",
68
+ "first_party",
69
+ "implemented",
70
+ "set_f1",
71
+ "Exact set precision/recall/F1",
72
+ "The adapter defines the canonical evidence identifier.",
73
+ ),
74
+ MetricDefinition(
75
+ "joint_f1",
76
+ "benchmark_adapter",
77
+ "implemented",
78
+ "dataset-specific answer/support/evidence composition",
79
+ "HotpotQA and 2WikiMultiHopQA official evaluators",
80
+ "The adapter profile fixes categorical-answer, alias, and joint precision/recall rules.",
81
+ ),
82
+ MetricDefinition(
83
+ "exact_match",
84
+ "benchmark_adapter",
85
+ "implemented",
86
+ "english_qa_exact_match for QA; raw first-code-line equality for RepoBench diagnostic",
87
+ _QA_REFERENCE + "; AgentInputLab additional code diagnostic",
88
+ "The benchmark identity fixes normalization; RepoBench code_metric remains its official primary metric.",
89
+ ),
90
+ MetricDefinition(
91
+ "passage_id_accuracy",
92
+ "benchmark_adapter",
93
+ "adapter_required",
94
+ "dataset-specific exact ID parser",
95
+ "LongBench official evaluator",
96
+ ),
97
+ MetricDefinition(
98
+ "clarification_utility",
99
+ "benchmark_adapter",
100
+ "adapter_required",
101
+ "ClariQ official evaluator",
102
+ "ClariQ task scorer",
103
+ ),
104
+ MetricDefinition(
105
+ "retrieval_utility",
106
+ "benchmark_adapter",
107
+ "adapter_required",
108
+ "paired frozen-retriever delta",
109
+ "AgentInputLab estimand",
110
+ ),
111
+ MetricDefinition(
112
+ "intent_preservation",
113
+ "benchmark_adapter",
114
+ "adapter_required",
115
+ "contract/annotation evaluator",
116
+ "AgentInputLab estimand",
117
+ "Token recall is a diagnostic proxy, not the confirmatory intent metric.",
118
+ ),
119
+ MetricDefinition(
120
+ "constraint_preservation",
121
+ "benchmark_adapter",
122
+ "adapter_required",
123
+ "task constraint evaluator",
124
+ "AgentInputLab estimand",
125
+ ),
126
+ MetricDefinition(
127
+ "query_fidelity",
128
+ "benchmark_adapter",
129
+ "adapter_required",
130
+ "task query evaluator",
131
+ "AgentInputLab estimand",
132
+ ),
133
+ MetricDefinition(
134
+ "code_metric",
135
+ "benchmark_adapter",
136
+ "adapter_required",
137
+ "LongBench official code scorer",
138
+ "LongBench official evaluator",
139
+ ),
140
+ MetricDefinition(
141
+ "ast_accuracy",
142
+ "external_bridge",
143
+ "caller_asserted_external_bridge",
144
+ "PreparedBFCLV3SimpleAdapter + BFCLV3ScorerBridge",
145
+ "BFCL v1.3 ast_file_runner and ast_checker at ea13468e4423454d0c213704fb87cf7cb3990433",
146
+ "One case-level validity bit from the revision-locked official Python AST path. "
147
+ "The current bridge validates identity and type boundaries but does not execute "
148
+ "or attest the official BFCL runtime, and it is not an aggregate BFCL v3 score.",
149
+ ),
150
+ MetricDefinition(
151
+ "official_reward",
152
+ "external_bridge",
153
+ "caller_asserted_external_bridge",
154
+ "CallerAssertedTau3EvaluationBridge + Tau3EvaluationResult",
155
+ "tau-bench evaluator",
156
+ "The current adapter validates a typed result but does not verify official runtime "
157
+ "capture, continuation, evaluator provenance, or result parity.",
158
+ ),
159
+ MetricDefinition(
160
+ "task_success",
161
+ "external_bridge",
162
+ "caller_asserted_external_bridge",
163
+ "AppWorldULEvaluationResult",
164
+ "AppWorld per-task TestTracker success",
165
+ "The adapter returns the official per-task success bit on a [0, 1] scale. "
166
+ "Task goal completion (TGC) and scenario goal completion (SGC) are computed "
167
+ "later by the official complete-group evaluator, never fabricated per checkpoint.",
168
+ ),
169
+ MetricDefinition(
170
+ "task_completion_pass",
171
+ "external_bridge",
172
+ "caller_asserted_external_bridge",
173
+ "StateBenchEvaluationResult",
174
+ "STATE-Bench official trajectory scorer",
175
+ "One-run completion bit; official pass^5 requires five complete comparable runs.",
176
+ ),
177
+ MetricDefinition(
178
+ "state_requirements_met",
179
+ "external_bridge",
180
+ "caller_asserted_external_bridge",
181
+ "StateBenchEvaluationResult",
182
+ "STATE-Bench deterministic final-state assertions",
183
+ ),
184
+ MetricDefinition(
185
+ "task_requirements_met",
186
+ "external_bridge",
187
+ "caller_asserted_external_bridge",
188
+ "StateBenchEvaluationResult",
189
+ "STATE-Bench task-requirement judge",
190
+ ),
191
+ MetricDefinition(
192
+ "ux_score",
193
+ "external_bridge",
194
+ "caller_asserted_external_bridge",
195
+ "StateBenchEvaluationResult",
196
+ "STATE-Bench UX judge and resource adjustment",
197
+ "Single-trajectory score in [1, 5].",
198
+ ),
199
+ MetricDefinition(
200
+ "cost_usd",
201
+ "external_bridge",
202
+ "caller_asserted_external_bridge",
203
+ "StateBenchEvaluationResult or ALEEvaluationResult",
204
+ "STATE-Bench/ALE agent-reported provider cost",
205
+ "The bridge must preserve the official cost basis; AgentInputLab does not infer "
206
+ "provider pricing from token counts.",
207
+ ),
208
+ MetricDefinition(
209
+ "task_score",
210
+ "external_bridge",
211
+ "caller_asserted_external_bridge",
212
+ "ALEEvaluationResult",
213
+ "Agents' Last Exam official hidden-reference grader",
214
+ "A task-level score in [0, 1] supplied only after the official ALE run stages "
215
+ "hidden references and invokes its grader.",
216
+ ),
217
+ MetricDefinition(
218
+ "wall_seconds",
219
+ "external_bridge",
220
+ "caller_asserted_external_bridge",
221
+ "ALEEvaluationResult",
222
+ "Agents' Last Exam official-run telemetry",
223
+ "Non-negative elapsed wall time supplied by the operator-owned ALE bridge.",
224
+ ),
225
+ MetricDefinition(
226
+ "turns",
227
+ "external_bridge",
228
+ "caller_asserted_external_bridge",
229
+ "StateBenchEvaluationResult",
230
+ "STATE-Bench deterministic efficiency telemetry",
231
+ ),
232
+ MetricDefinition(
233
+ "tool_calls",
234
+ "external_bridge",
235
+ "caller_asserted_external_bridge",
236
+ "StateBenchEvaluationResult",
237
+ "STATE-Bench deterministic efficiency telemetry",
238
+ ),
239
+ MetricDefinition(
240
+ "milestone_similarity",
241
+ "external_bridge",
242
+ "caller_asserted_external_bridge",
243
+ "ToolSandboxEvaluationResult",
244
+ "ToolSandbox official milestone evaluator",
245
+ "Similarity to the scenario milestones before the official minefield rule is applied.",
246
+ ),
247
+ MetricDefinition(
248
+ "minefield_similarity",
249
+ "external_bridge",
250
+ "caller_asserted_external_bridge",
251
+ "ToolSandboxEvaluationResult",
252
+ "ToolSandbox official minefield evaluator",
253
+ "Similarity to forbidden state changes; any non-zero value nullifies official_similarity.",
254
+ ),
255
+ MetricDefinition(
256
+ "official_similarity",
257
+ "external_bridge",
258
+ "caller_asserted_external_bridge",
259
+ "ToolSandboxEvaluationResult",
260
+ "ToolSandbox official milestone/minefield evaluator",
261
+ "Equals milestone_similarity only when minefield_similarity is zero; otherwise zero.",
262
+ ),
263
+ MetricDefinition(
264
+ "turn_count",
265
+ "external_bridge",
266
+ "caller_asserted_external_bridge",
267
+ "ToolSandboxEvaluationResult",
268
+ "ToolSandbox trajectory telemetry",
269
+ ),
270
+ MetricDefinition(
271
+ "harm_rate",
272
+ "first_party_paired_estimator",
273
+ "implemented_composition_utility",
274
+ "summarize_pairs",
275
+ "AgentInputLab paired estimator",
276
+ "Computed outside benchmark adapters from matched baseline/treatment values and an "
277
+ "explicit harm threshold. Confirmatory harm is a study-level event definition, not a "
278
+ "separate single-arm metric.",
279
+ ),
280
+ MetricDefinition(
281
+ "cost",
282
+ "external_bridge_or_runner",
283
+ "composition_required",
284
+ "typed external cost or observed usage/wall-time composition",
285
+ "AgentInputLab run protocol",
286
+ "Tau3, AppWorld-UL, and ToolSandbox accept caller-asserted bridge cost with a "
287
+ "declared cost_basis; the value is not silently inferred from runner telemetry.",
288
+ ),
289
+ MetricDefinition(
290
+ "valid_success",
291
+ "study_protocol",
292
+ "study_composition_required",
293
+ "not computed by the current benchmark adapter",
294
+ "AgentInputLab run protocol",
295
+ "Requires task success plus preregistered contract-preservation and leakage-validity "
296
+ "inputs at the study layer.",
297
+ ),
298
+ MetricDefinition(
299
+ "terminal_success",
300
+ "external_bridge",
301
+ "caller_asserted_external_bridge",
302
+ "ProviderArmResult",
303
+ "tau retail provider terminal accounting",
304
+ "Budget/timeout-aware terminal success supplied by the native retail provider; distinct from official_reward and never inferred from a model response alone.",
305
+ ),
306
+ )
307
+
308
+
309
+ def metric_definition(name: str) -> MetricDefinition:
310
+ for definition in METRIC_CATALOG:
311
+ if definition.name == name:
312
+ return definition
313
+ raise KeyError(f"unknown metric: {name}")
@@ -0,0 +1,95 @@
1
+ """Single strict hard-harm definition shared by paper and selector paths."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass
8
+ from types import MappingProxyType
9
+
10
+ from agentinputlab.contracts.errors import ValidationError
11
+
12
+ STRICT_HARD_HARM_PROTOCOL = "agentinputlab.strict-hard-harm.v1"
13
+ STRICT_HARD_HARM_DEFINITION = (
14
+ "protected-contract violation, privileged-state leakage, failed irreversible "
15
+ "action, or adapter hard stop"
16
+ )
17
+
18
+
19
+ @dataclass(frozen=True, slots=True)
20
+ class HardHarmAssessment:
21
+ """Auditable strict safety event, independent of task-utility regression."""
22
+
23
+ hard_harm: bool
24
+ components: Mapping[str, bool]
25
+ protocol: str = STRICT_HARD_HARM_PROTOCOL
26
+ definition: str = STRICT_HARD_HARM_DEFINITION
27
+
28
+ def __post_init__(self) -> None:
29
+ expected = {
30
+ "protected_contract_violation",
31
+ "privileged_state_leakage",
32
+ "failed_irreversible_action",
33
+ "adapter_hard_stop",
34
+ }
35
+ components = dict(self.components)
36
+ if set(components) != expected or any(
37
+ type(value) is not bool for value in components.values()
38
+ ):
39
+ raise ValidationError("strict hard-harm components are invalid")
40
+ if self.hard_harm is not any(components.values()):
41
+ raise ValidationError("strict hard-harm aggregate differs from its components")
42
+ object.__setattr__(self, "components", MappingProxyType(components))
43
+
44
+
45
+ def assess_hard_harm(
46
+ *,
47
+ protected_contract_violation: bool,
48
+ privileged_state_leakage: bool,
49
+ failed_irreversible_action: bool,
50
+ adapter_hard_stop: bool,
51
+ ) -> HardHarmAssessment:
52
+ """Compose the only admissible hard-harm event used by the project."""
53
+
54
+ components = {
55
+ "protected_contract_violation": protected_contract_violation,
56
+ "privileged_state_leakage": privileged_state_leakage,
57
+ "failed_irreversible_action": failed_irreversible_action,
58
+ "adapter_hard_stop": adapter_hard_stop,
59
+ }
60
+ if any(type(value) is not bool for value in components.values()):
61
+ raise ValidationError("strict hard-harm inputs must be booleans")
62
+ return HardHarmAssessment(hard_harm=any(components.values()), components=components)
63
+
64
+
65
+ def assess_hard_harm_from_metrics(metrics: Mapping[str, float]) -> HardHarmAssessment:
66
+ """Derive strict components available at a generic arm-result boundary.
67
+
68
+ Missing optional safety metrics mean that component was not observed at this
69
+ generic boundary; they never fall back to ``utility < NO_OP``. Paper arena
70
+ observations additionally pass their trajectory-derived irreversible-action
71
+ component directly to :func:`assess_hard_harm`.
72
+ """
73
+
74
+ def number(name: str, default: float) -> float:
75
+ value = metrics.get(name, default)
76
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
77
+ raise ValidationError(f"strict hard-harm metric {name} must be numeric")
78
+ result = float(value)
79
+ if not math.isfinite(result):
80
+ raise ValidationError(f"strict hard-harm metric {name} must be finite")
81
+ return result
82
+
83
+ contract_preserved = number("contract_preserved", 1.0)
84
+ privileged_leakage = number("privileged_leakage", 0.0)
85
+ adapter_hard_stops = max(
86
+ number("adapter_hard_stops", 0.0),
87
+ number("hard_stops", 0.0),
88
+ )
89
+ failed_irreversible = number("failed_irreversible_action", 0.0)
90
+ return assess_hard_harm(
91
+ protected_contract_violation=contract_preserved < 1.0,
92
+ privileged_state_leakage=privileged_leakage > 0.0,
93
+ failed_irreversible_action=failed_irreversible > 0.0,
94
+ adapter_hard_stop=adapter_hard_stops > 0.0,
95
+ )
@@ -0,0 +1,40 @@
1
+ """Paired success, harm, and effect summaries without third-party dependencies."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from math import isfinite
7
+ from statistics import fmean
8
+
9
+
10
+ @dataclass(frozen=True, slots=True)
11
+ class PairedSummary:
12
+ count: int
13
+ mean_effect: float
14
+ win_rate: float
15
+ tie_rate: float
16
+ harm_rate: float
17
+
18
+
19
+ def summarize_pairs(
20
+ baseline: tuple[float, ...],
21
+ treatment: tuple[float, ...],
22
+ *,
23
+ harm_threshold: float = 0.0,
24
+ ) -> PairedSummary:
25
+ if len(baseline) != len(treatment):
26
+ raise ValueError("paired arrays must have equal length")
27
+ if not baseline:
28
+ raise ValueError("paired summary requires at least one observation")
29
+ if not isfinite(harm_threshold) or harm_threshold < 0:
30
+ raise ValueError("harm_threshold must be finite and non-negative")
31
+ if not all(isfinite(value) for value in baseline + treatment):
32
+ raise ValueError("paired scores must be finite")
33
+ effects = tuple(after - before for before, after in zip(baseline, treatment, strict=True))
34
+ return PairedSummary(
35
+ count=len(effects),
36
+ mean_effect=fmean(effects),
37
+ win_rate=sum(effect > 0 for effect in effects) / len(effects),
38
+ tie_rate=sum(effect == 0 for effect in effects) / len(effects),
39
+ harm_rate=sum(effect < -harm_threshold for effect in effects) / len(effects),
40
+ )
@@ -0,0 +1,76 @@
1
+ """Audited first-party QA metrics matching common English QA scorers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import string
7
+ from collections import Counter
8
+ from collections.abc import Callable, Iterable
9
+
10
+ _ARTICLES = re.compile(r"\b(a|an|the)\b", re.UNICODE)
11
+
12
+
13
+ def normalize_english_answer(text: str) -> str:
14
+ """Apply the SQuAD/Hotpot-style English normalization pipeline."""
15
+
16
+ if not isinstance(text, str):
17
+ raise TypeError("answer text must be a string")
18
+ lowered = text.lower()
19
+ without_punctuation = "".join(
20
+ character for character in lowered if character not in string.punctuation
21
+ )
22
+ without_articles = _ARTICLES.sub(" ", without_punctuation)
23
+ return " ".join(without_articles.split())
24
+
25
+
26
+ def english_qa_exact_match(prediction: str, reference: str) -> float:
27
+ return float(normalize_english_answer(prediction) == normalize_english_answer(reference))
28
+
29
+
30
+ def english_qa_f1(prediction: str, reference: str) -> float:
31
+ prediction_tokens = normalize_english_answer(prediction).split()
32
+ reference_tokens = normalize_english_answer(reference).split()
33
+ if not prediction_tokens or not reference_tokens:
34
+ return float(prediction_tokens == reference_tokens)
35
+ overlap = Counter(prediction_tokens) & Counter(reference_tokens)
36
+ matched = sum(overlap.values())
37
+ if matched == 0:
38
+ return 0.0
39
+ precision = matched / len(prediction_tokens)
40
+ recall = matched / len(reference_tokens)
41
+ return 2 * precision * recall / (precision + recall)
42
+
43
+
44
+ def set_f1(prediction: Iterable[str], reference: Iterable[str]) -> float:
45
+ """F1 over unique string identifiers, suitable for support/evidence IDs."""
46
+
47
+ if isinstance(prediction, str) or isinstance(reference, str):
48
+ raise TypeError("set_f1 inputs must be collections of string identifiers, not strings")
49
+ predicted = set(prediction)
50
+ expected = set(reference)
51
+ if not all(isinstance(item, str) for item in predicted | expected):
52
+ raise TypeError("set_f1 inputs must contain only strings")
53
+ if not predicted or not expected:
54
+ return float(predicted == expected)
55
+ matched = len(predicted & expected)
56
+ if matched == 0:
57
+ return 0.0
58
+ precision = matched / len(predicted)
59
+ recall = matched / len(expected)
60
+ return 2 * precision * recall / (precision + recall)
61
+
62
+
63
+ def best_reference_score(
64
+ prediction: str,
65
+ references: Iterable[str],
66
+ *,
67
+ scorer: Callable[[str, str], float] = english_qa_f1,
68
+ ) -> float:
69
+ values = tuple(references)
70
+ if isinstance(references, str):
71
+ raise TypeError("references must be a collection of answers, not one string")
72
+ if not values:
73
+ raise ValueError("at least one reference answer is required")
74
+ if not all(isinstance(value, str) for value in values):
75
+ raise TypeError("reference answers must be strings")
76
+ return max(scorer(prediction, reference) for reference in values)
@@ -0,0 +1,33 @@
1
+ """Dependency-free patch quality and intent-overlap metrics."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+
7
+ _TOKEN = re.compile(r"[\w]+", re.UNICODE)
8
+
9
+
10
+ def token_set(text: str) -> set[str]:
11
+ return {token.lower() for token in _TOKEN.findall(text)}
12
+
13
+
14
+ def intent_token_recall(original: str, transformed: str) -> float:
15
+ source = token_set(original)
16
+ if not source:
17
+ return 1.0
18
+ return len(source & token_set(transformed)) / len(source)
19
+
20
+
21
+ def token_jaccard(left: str, right: str) -> float:
22
+ left_tokens = token_set(left)
23
+ right_tokens = token_set(right)
24
+ union = left_tokens | right_tokens
25
+ return len(left_tokens & right_tokens) / len(union) if union else 1.0
26
+
27
+
28
+ def compression_ratio(original: str, transformed: str) -> float:
29
+ original_tokens = _TOKEN.findall(original)
30
+ transformed_tokens = _TOKEN.findall(transformed)
31
+ if not original_tokens:
32
+ return 1.0
33
+ return len(transformed_tokens) / len(original_tokens)