agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,5 @@
1
+ """Versioned local audit annotation UI and storage."""
2
+
3
+ from .store import AnnotationStore, AnnotationStoreError, validate_annotation_selection
4
+
5
+ __all__ = ["AnnotationStore", "AnnotationStoreError", "validate_annotation_selection"]
@@ -0,0 +1,428 @@
1
+ """Independent, blinded two-person input review with third-person adjudication."""
2
+
3
+ # ruff: noqa: RUF001 -- Chinese UI text intentionally uses native full-width punctuation.
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import hashlib
8
+ import hmac
9
+ import json
10
+ import os
11
+ import secrets
12
+ import sqlite3
13
+ import threading
14
+ from datetime import datetime, timezone
15
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
16
+ from pathlib import Path
17
+ from urllib.parse import urlsplit
18
+
19
+ from agentinputlab.contracts.serialization import content_hash
20
+ from agentinputlab.experiments.measurement import MeasurementItem, analyze_measurement_annotations
21
+
22
+ FIELDS = {
23
+ "contract_anchor": {
24
+ "entity_anchor_present": (
25
+ "实体锚点",
26
+ "输入中是否明确给出必须保留的人名、地点、产品、对象或标识符?",
27
+ ),
28
+ "numeric_anchor_present": (
29
+ "数值锚点",
30
+ "是否给出必须保留的金额、数量、时间、日期、范围或阈值?",
31
+ ),
32
+ "constraint_anchor_present": (
33
+ "约束锚点",
34
+ "是否明确规定必须/禁止、范围、格式、先后次序或资源限制?",
35
+ ),
36
+ },
37
+ "defect_eligibility": {
38
+ "unclear_or_misaligned_request": (
39
+ "歧义/意图不清",
40
+ "在可见上下文中,请求是否仍有阻碍正确执行的歧义或不一致?",
41
+ ),
42
+ "decomposition_need": ("需任务分解", "是否有多个依赖步骤,显式规划或分解能减少执行遗漏?"),
43
+ "missing_external_evidence": (
44
+ "缺外部证据",
45
+ "执行是否缺少输入和已有上下文之外的事实或检索材料?",
46
+ ),
47
+ "clarification_need": ("需澄清", "是否缺少只能由用户补充、且会改变正确行动的关键条件?"),
48
+ "context_overload_or_noise": (
49
+ "上下文噪声",
50
+ "可见上下文是否有明显冗余、冲突或与任务无关的材料?",
51
+ ),
52
+ "low_defect_state": ("低缺陷输入", "是否没有上述明显缺陷,直接执行较为合理?"),
53
+ "reformulate_eligible": ("允许改写", "保持实体、数值和约束不变时,改写是否适用?"),
54
+ "decompose_eligible": ("允许分解", "不增加目标或更改约束时,分解是否适用?"),
55
+ "acquire_eligible": ("允许补充证据", "是否可通过已有工具/公开材料补充所需信息?"),
56
+ "clarify_eligible": ("允许询问", "向用户询问关键缺失条件是否适用?"),
57
+ "curate_eligible": ("允许整理上下文", "不删除任务必需信息时,整理或去噪是否适用?"),
58
+ },
59
+ }
60
+ OPTIONS = {"yes", "no", "uncertain"}
61
+
62
+
63
+ def utc():
64
+ return datetime.now(timezone.utc).isoformat()
65
+
66
+
67
+ def sha(raw):
68
+ return hashlib.sha256(raw).hexdigest()
69
+
70
+
71
+ def create_json(path, value):
72
+ with Path(path).open("x", encoding="utf8") as stream:
73
+ json.dump(value, stream, ensure_ascii=False, indent=2)
74
+ stream.write("\n")
75
+ os.chmod(path, 0o600)
76
+
77
+
78
+ class ReviewStudy:
79
+ def __init__(self, root):
80
+ self.root = Path(root).resolve()
81
+ self.manifest = json.loads((self.root / "assignment_manifest.json").read_text())
82
+ unhashed = {k: v for k, v in self.manifest.items() if k != "assignment_hash"}
83
+ if self.manifest["assignment_hash"] != content_hash(unhashed):
84
+ raise ValueError("Assignment manifest changed")
85
+ self.secret = json.loads((self.root / "private/reviewer_access.json").read_text())
86
+ self.items = {}
87
+ self.assignment = {}
88
+ self.short = {}
89
+ self.lock = threading.RLock()
90
+ for reviewer, expected in self.manifest["packet_hashes"].items():
91
+ packet = json.loads((self.root / "private" / f"{reviewer}.json").read_text())
92
+ if content_hash(packet) != expected:
93
+ raise ValueError("Reviewer packet changed")
94
+ for entry in packet:
95
+ blind = entry["blind_id"]
96
+ self.items[blind] = entry
97
+ for i, row in enumerate(self.manifest["assignments"], 1):
98
+ blind = row["blind_id"]
99
+ self.assignment[blind] = row["assigned_annotators"]
100
+ self.short[blind] = f"S{i:03d}"
101
+ if set(self.items) != set(self.assignment):
102
+ raise ValueError("Item assignment differs")
103
+ for entry in self.items.values():
104
+ MeasurementItem("validation", entry["group_hash"], entry["kind"], entry["payload"])
105
+ self.db = self.root / "private/reviews.sqlite3"
106
+ with self.connect() as db:
107
+ db.execute(
108
+ "CREATE TABLE IF NOT EXISTS events (id INTEGER PRIMARY KEY, reviewer TEXT NOT NULL, blind TEXT NOT NULL, stage TEXT NOT NULL, action TEXT NOT NULL, labels TEXT NOT NULL, notes TEXT NOT NULL, utc TEXT NOT NULL)"
109
+ )
110
+ db.execute(
111
+ "CREATE UNIQUE INDEX IF NOT EXISTS final_vote ON events(reviewer,blind,stage) WHERE action='submit'"
112
+ )
113
+ db.execute(
114
+ "CREATE TABLE IF NOT EXISTS sessions (token_hash TEXT PRIMARY KEY, reviewer TEXT NOT NULL, created TEXT NOT NULL)"
115
+ )
116
+ os.chmod(self.db, 0o600)
117
+
118
+ def connect(self):
119
+ db = sqlite3.connect(self.db, timeout=15)
120
+ db.row_factory = sqlite3.Row
121
+ return db
122
+
123
+ def login(self, reviewer, code, independent):
124
+ if independent is not True:
125
+ raise ValueError("请确认由你本人独立评审")
126
+ expected = self.secret.get(reviewer, {}).get("code_sha256", "")
127
+ if (
128
+ not isinstance(code, str)
129
+ or not expected
130
+ or not hmac.compare_digest(sha(code.encode()), expected)
131
+ ):
132
+ raise ValueError("评审 ID 或访问码不正确")
133
+ token = secrets.token_urlsafe(32)
134
+ with self.connect() as db:
135
+ db.execute("INSERT INTO sessions VALUES(?,?,?)", (sha(token.encode()), reviewer, utc()))
136
+ return token
137
+
138
+ def reviewer(self, token):
139
+ if not isinstance(token, str):
140
+ raise ValueError("请先登录")
141
+ with self.connect() as db:
142
+ row = db.execute(
143
+ "SELECT reviewer FROM sessions WHERE token_hash=?", (sha(token.encode()),)
144
+ ).fetchone()
145
+ if not row:
146
+ raise ValueError("请先登录")
147
+ return row["reviewer"]
148
+
149
+ def latest(self, reviewer, blind, stage):
150
+ with self.connect() as db:
151
+ return db.execute(
152
+ "SELECT * FROM events WHERE reviewer=? AND blind=? AND stage=? ORDER BY id DESC LIMIT 1",
153
+ (reviewer, blind, stage),
154
+ ).fetchone()
155
+
156
+ def votes(self, blind):
157
+ with self.connect() as db:
158
+ return db.execute(
159
+ "SELECT * FROM events WHERE blind=? AND stage='first_pass' AND action='submit'",
160
+ (blind,),
161
+ ).fetchall()
162
+
163
+ def stage(self, reviewer, blind):
164
+ if blind not in self.assignment:
165
+ raise ValueError("样本不存在")
166
+ if reviewer in self.assignment[blind]:
167
+ return "first_pass"
168
+ votes = self.votes(blind)
169
+ if len(votes) == 2 and votes[0]["labels"] != votes[1]["labels"]:
170
+ return "adjudication"
171
+ raise ValueError("这个样本未分配给你")
172
+
173
+ def status(self, reviewer):
174
+ items = []
175
+ for blind in sorted(self.items, key=lambda b: sha((reviewer + b).encode())):
176
+ try:
177
+ stage = self.stage(reviewer, blind)
178
+ except ValueError:
179
+ continue
180
+ latest = self.latest(reviewer, blind, stage)
181
+ items.append(
182
+ {
183
+ "sample_id": self.short[blind],
184
+ "stage": stage,
185
+ "submitted": bool(latest and latest["action"] == "submit"),
186
+ "draft": bool(latest and latest["action"] == "draft"),
187
+ }
188
+ )
189
+ return {
190
+ "reviewer_id": reviewer,
191
+ "items": items,
192
+ "submitted": sum(r["submitted"] for r in items),
193
+ "total": len(items),
194
+ }
195
+
196
+ def item(self, reviewer, sample_id):
197
+ blind = next((b for b, s in self.short.items() if s == sample_id), None)
198
+ stage = self.stage(reviewer, blind)
199
+ entry = self.items[blind]
200
+ latest = self.latest(reviewer, blind, stage)
201
+ return {
202
+ "sample_id": sample_id,
203
+ "stage": stage,
204
+ "kind": entry["kind"],
205
+ "payload": entry["payload"],
206
+ "fields": [
207
+ {"id": k, "title": v[0], "help": v[1]} for k, v in FIELDS[entry["kind"]].items()
208
+ ],
209
+ "labels": json.loads(latest["labels"]) if latest else {},
210
+ "notes": latest["notes"] if latest else "",
211
+ "submitted": bool(latest and latest["action"] == "submit"),
212
+ }
213
+
214
+ def record(self, reviewer, sample_id, labels, notes, submit):
215
+ blind = next((b for b, s in self.short.items() if s == sample_id), None)
216
+ stage = self.stage(reviewer, blind)
217
+ allowed = FIELDS[self.items[blind]["kind"]]
218
+ if (
219
+ not isinstance(labels, dict)
220
+ or set(labels) - set(allowed)
221
+ or any(v not in OPTIONS for v in labels.values())
222
+ ):
223
+ raise ValueError("标注字段或选项不合法")
224
+ if submit and set(labels) != set(allowed):
225
+ raise ValueError("每项都需选择;无法判断请选择“不确定”")
226
+ if not isinstance(notes, str) or len(notes) > 4000:
227
+ raise ValueError("备注最长 4000 字符")
228
+ if submit and not notes.strip():
229
+ raise ValueError("请提供依据、原文摘录或无法判断的原因")
230
+ with self.lock, self.connect() as db:
231
+ if db.execute(
232
+ "SELECT 1 FROM events WHERE reviewer=? AND blind=? AND stage=? AND action='submit'",
233
+ (reviewer, blind, stage),
234
+ ).fetchone():
235
+ raise ValueError("已提交标注不可覆盖")
236
+ db.execute(
237
+ "INSERT INTO events(reviewer,blind,stage,action,labels,notes,utc) VALUES(?,?,?,?,?,?,?)",
238
+ (
239
+ reviewer,
240
+ blind,
241
+ stage,
242
+ "submit" if submit else "draft",
243
+ json.dumps(labels, sort_keys=True),
244
+ notes,
245
+ utc(),
246
+ ),
247
+ )
248
+ return {"saved": True, "submitted": bool(submit), "saved_at": utc()}
249
+
250
+ def export(self, path):
251
+ with self.connect() as db:
252
+ rows = db.execute("SELECT * FROM events WHERE action='submit' ORDER BY id").fetchall()
253
+ annotations = [
254
+ {
255
+ "blind_id": r["blind"],
256
+ "annotator_id": r["reviewer"],
257
+ "annotation_stage": r["stage"],
258
+ "labels": json.loads(r["labels"]),
259
+ }
260
+ for r in rows
261
+ ]
262
+ output = {
263
+ "schema": "independent-human-input-review-export.v1",
264
+ "assignment_hash": self.manifest["assignment_hash"],
265
+ "annotations": annotations,
266
+ "evidence_notes": [
267
+ {
268
+ "blind_id": r["blind"],
269
+ "annotator_id": r["reviewer"],
270
+ "notes": r["notes"],
271
+ "utc": r["utc"],
272
+ }
273
+ for r in rows
274
+ ],
275
+ "scope": "input defects, eligibility and original-input anchor inventories only; not trajectory preservation or execution success",
276
+ }
277
+ try:
278
+ analysis = analyze_measurement_annotations(self.manifest, annotations)
279
+ # The legacy analyzer maps an all-one-category pair to 1. Report the
280
+ # mathematically undefined chance-corrected coefficients explicitly here.
281
+ kinds = {b: self.items[b]["kind"] for b in self.items}
282
+ for field in analysis["agreement"]:
283
+ values = [
284
+ r["labels"][field["field"]]
285
+ for r in annotations
286
+ if r["annotation_stage"] == "first_pass"
287
+ and kinds[r["blind_id"]] == field["kind"]
288
+ ]
289
+ field["category_counts"] = {v: values.count(v) for v in sorted(set(values))}
290
+ field["uncertain_first_pass_count"] = values.count("uncertain")
291
+ field["coefficient_status"] = "defined"
292
+ if len(set(values)) == 1:
293
+ field["cohen_kappa"] = None
294
+ field["krippendorff_alpha_nominal"] = None
295
+ field["coefficient_status"] = "undefined_single_category"
296
+ analysis.pop("analysis_hash", None)
297
+ analysis["analysis_hash"] = content_hash(analysis)
298
+ output["analysis"] = analysis
299
+ except ValueError as exc:
300
+ output["analysis_complete"] = False
301
+ output["incomplete_reason"] = str(exc)
302
+ create_json(path, output)
303
+ return {
304
+ "submitted_votes": len(rows),
305
+ "analysis_complete": output.get("analysis", {}).get("analysis_complete", False),
306
+ }
307
+
308
+
309
+ def handler(study):
310
+ class Handler(BaseHTTPRequestHandler):
311
+ def log_message(self, *args):
312
+ pass
313
+
314
+ def send(self, status, body, mime="application/json"):
315
+ raw = (
316
+ json.dumps(body, ensure_ascii=False).encode()
317
+ if mime == "application/json"
318
+ else body
319
+ )
320
+ self.send_response(status)
321
+ self.send_header("Content-Type", mime)
322
+ self.send_header("Content-Length", str(len(raw)))
323
+ self.send_header("Cache-Control", "no-store")
324
+ self.send_header("X-Content-Type-Options", "nosniff")
325
+ self.send_header("X-Frame-Options", "DENY")
326
+ self.send_header("Referrer-Policy", "no-referrer")
327
+ self.send_header(
328
+ "Content-Security-Policy",
329
+ "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; frame-ancestors 'none'",
330
+ )
331
+ self.end_headers()
332
+ self.wfile.write(raw)
333
+
334
+ def valid_origin(self):
335
+ host = self.headers.get("Host", "")
336
+ expected = "127.0.0.1:" + str(self.server.server_port)
337
+ if host != expected:
338
+ raise ValueError("Host not allowed")
339
+ origin = self.headers.get("Origin")
340
+ if origin is not None and origin != "http://" + expected:
341
+ raise ValueError("Origin not allowed")
342
+
343
+ def identity(self):
344
+ return study.reviewer(self.headers.get("Authorization", "").removeprefix("Bearer "))
345
+
346
+ def do_GET(self):
347
+ try:
348
+ self.valid_origin()
349
+ path = urlsplit(self.path).path
350
+ if path == "/api/status":
351
+ return self.send(200, study.status(self.identity()))
352
+ if path.startswith("/api/items/"):
353
+ return self.send(200, study.item(self.identity(), path.rsplit("/", 1)[1]))
354
+ name = {
355
+ "/": "review.html",
356
+ "/review.js": "review.js",
357
+ "/review.css": "review.css",
358
+ }.get(path)
359
+ if not name:
360
+ return self.send(404, {"error": "Not found"})
361
+ mime = {
362
+ "review.html": "text/html; charset=utf-8",
363
+ "review.js": "application/javascript; charset=utf-8",
364
+ "review.css": "text/css; charset=utf-8",
365
+ }[name]
366
+ return self.send(200, (Path(__file__).parent / "static" / name).read_bytes(), mime)
367
+ except (ValueError, KeyError, TypeError):
368
+ return self.send(403, {"error": "无权限访问;请检查登录或样本 ID"})
369
+
370
+ def do_POST(self):
371
+ try:
372
+ self.valid_origin()
373
+ if self.headers.get("Content-Type", "").split(";")[0] != "application/json":
374
+ raise ValueError("JSON required")
375
+ size = int(self.headers.get("Content-Length", "0"))
376
+ if not 0 < size <= 32768:
377
+ raise ValueError("Invalid request size")
378
+ value = json.loads(self.rfile.read(size))
379
+ path = urlsplit(self.path).path
380
+ if path == "/api/login":
381
+ return self.send(
382
+ 200,
383
+ {
384
+ "token": study.login(
385
+ value["reviewer_id"], value["code"], value.get("independent")
386
+ )
387
+ },
388
+ )
389
+ if path == "/api/review":
390
+ return self.send(
391
+ 200,
392
+ study.record(
393
+ self.identity(),
394
+ value["sample_id"],
395
+ value["labels"],
396
+ value.get("notes", ""),
397
+ value.get("submit") is True,
398
+ ),
399
+ )
400
+ return self.send(404, {"error": "Not found"})
401
+ except (ValueError, KeyError, TypeError, sqlite3.IntegrityError) as exc:
402
+ return self.send(400, {"error": str(exc)})
403
+
404
+ return Handler
405
+
406
+
407
+ def main():
408
+ p = argparse.ArgumentParser()
409
+ p.add_argument("--study", type=Path, required=True)
410
+ p.add_argument("--port", type=int, default=7864)
411
+ p.add_argument("--export", type=Path)
412
+ a = p.parse_args()
413
+ study = ReviewStudy(a.study)
414
+ if a.export:
415
+ print(json.dumps(study.export(a.export)))
416
+ return
417
+ server = ThreadingHTTPServer(("127.0.0.1", a.port), handler(study))
418
+ print(f"Independent input review: http://127.0.0.1:{server.server_port}", flush=True)
419
+ try:
420
+ server.serve_forever()
421
+ except KeyboardInterrupt:
422
+ pass
423
+ finally:
424
+ server.server_close()
425
+
426
+
427
+ if __name__ == "__main__":
428
+ main()
@@ -0,0 +1,233 @@
1
+ """Loopback-only annotation web server with automatic versioned persistence."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import mimetypes
8
+ import socket
9
+ import sys
10
+ from http import HTTPStatus
11
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
12
+ from pathlib import Path
13
+ from typing import Any
14
+ from urllib.parse import urlsplit
15
+
16
+ from .store import ANNOTATION_KINDS, AnnotationStore, AnnotationStoreError
17
+
18
+ LOOPBACK_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"})
19
+ MAX_REQUEST_BYTES = 32_768
20
+ STATIC_DIR = Path(__file__).resolve().parent / "static"
21
+ STATIC_ROUTES = {"/": "index.html", "/app.js": "app.js", "/styles.css": "styles.css"}
22
+
23
+
24
+ class AnnotationHTTPServer(ThreadingHTTPServer):
25
+ daemon_threads = True
26
+ allow_reuse_address = True
27
+
28
+ def __init__(self, address: tuple[str, int], *, store: AnnotationStore) -> None:
29
+ self.store = store
30
+ self.address_family = socket.AF_INET6 if ":" in address[0] else socket.AF_INET
31
+ super().__init__(address, AnnotationHandler)
32
+
33
+
34
+ class AnnotationHandler(BaseHTTPRequestHandler):
35
+ server: AnnotationHTTPServer
36
+ protocol_version = "HTTP/1.1"
37
+
38
+ def do_GET(self) -> None:
39
+ if not self._accept_host():
40
+ return
41
+ path = urlsplit(self.path).path
42
+ try:
43
+ if path == "/api/status":
44
+ self._json(self.server.store.status())
45
+ return
46
+ if path.startswith("/api/versions/"):
47
+ version_id = path.removeprefix("/api/versions/")
48
+ self._json(self.server.store.payload(version_id))
49
+ return
50
+ if path in STATIC_ROUTES:
51
+ self._static(STATIC_ROUTES[path])
52
+ return
53
+ self._error(HTTPStatus.NOT_FOUND, "not_found", "route does not exist")
54
+ except AnnotationStoreError as exc:
55
+ self._error(HTTPStatus.UNPROCESSABLE_ENTITY, "annotation_invalid", str(exc))
56
+
57
+ def do_POST(self) -> None:
58
+ if not self._accept_host():
59
+ return
60
+ path = urlsplit(self.path).path
61
+ try:
62
+ payload = self._read_json()
63
+ if path == "/api/versions":
64
+ if payload.get("action") != "new":
65
+ raise AnnotationStoreError(
66
+ "choose an existing completed version in the UI or request action=new"
67
+ )
68
+ self._json(
69
+ self.server.store.create_version(
70
+ annotator_id=str(payload.get("annotator_id", ""))
71
+ ),
72
+ status=HTTPStatus.CREATED,
73
+ )
74
+ return
75
+ if path == "/api/selection":
76
+ self._json(self.server.store.select(str(payload.get("version_id", ""))))
77
+ return
78
+ if path.startswith("/api/versions/") and path.endswith("/annotations"):
79
+ version_id = path.removeprefix("/api/versions/").removesuffix("/annotations")
80
+ self._json(self.server.store.save(version_id, payload))
81
+ return
82
+ if path.startswith("/api/versions/") and path.endswith("/complete"):
83
+ version_id = path.removeprefix("/api/versions/").removesuffix("/complete")
84
+ self._json(self.server.store.complete(version_id))
85
+ return
86
+ self._error(HTTPStatus.NOT_FOUND, "not_found", "route does not exist")
87
+ except _HTTPInputError as exc:
88
+ self._error(exc.status, exc.code, exc.message)
89
+ except AnnotationStoreError as exc:
90
+ self._error(HTTPStatus.UNPROCESSABLE_ENTITY, "annotation_invalid", str(exc))
91
+
92
+ def _read_json(self) -> dict[str, Any]:
93
+ if self.headers.get("Content-Type", "").partition(";")[0].strip() != "application/json":
94
+ raise _HTTPInputError(
95
+ HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
96
+ "content_type",
97
+ "Content-Type must be application/json",
98
+ )
99
+ try:
100
+ length = int(self.headers.get("Content-Length", "0"))
101
+ except ValueError as exc:
102
+ raise _HTTPInputError(
103
+ HTTPStatus.BAD_REQUEST, "content_length", "invalid length"
104
+ ) from exc
105
+ if length < 1 or length > MAX_REQUEST_BYTES:
106
+ raise _HTTPInputError(
107
+ HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
108
+ "body_size",
109
+ "JSON request size is invalid",
110
+ )
111
+ try:
112
+ value = json.loads(self.rfile.read(length).decode("utf-8"))
113
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
114
+ raise _HTTPInputError(HTTPStatus.BAD_REQUEST, "invalid_json", "invalid JSON") from exc
115
+ if not isinstance(value, dict):
116
+ raise _HTTPInputError(HTTPStatus.BAD_REQUEST, "json_type", "JSON must be an object")
117
+ return value
118
+
119
+ def _accept_host(self) -> bool:
120
+ raw = self.headers.get("Host", "")
121
+ try:
122
+ parsed = urlsplit(f"//{raw}")
123
+ _ = parsed.port
124
+ except ValueError:
125
+ parsed = None
126
+ if parsed is None or parsed.hostname not in LOOPBACK_HOSTS:
127
+ self._error(HTTPStatus.MISDIRECTED_REQUEST, "host_rejected", "loopback Host required")
128
+ return False
129
+ return True
130
+
131
+ def _security_headers(self) -> None:
132
+ self.send_header("X-Content-Type-Options", "nosniff")
133
+ self.send_header("X-Frame-Options", "DENY")
134
+ self.send_header("Referrer-Policy", "no-referrer")
135
+ self.send_header(
136
+ "Content-Security-Policy",
137
+ "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; "
138
+ "img-src 'self' data:; object-src 'none'; base-uri 'none'; frame-ancestors 'none'",
139
+ )
140
+
141
+ def _json(self, value: Any, *, status: HTTPStatus = HTTPStatus.OK) -> None:
142
+ data = json.dumps(value, ensure_ascii=False, sort_keys=True).encode()
143
+ self.send_response(status)
144
+ self.send_header("Content-Type", "application/json; charset=utf-8")
145
+ self.send_header("Content-Length", str(len(data)))
146
+ self.send_header("Cache-Control", "no-store")
147
+ self._security_headers()
148
+ self.end_headers()
149
+ self.wfile.write(data)
150
+
151
+ def _error(self, status: HTTPStatus, code: str, message: str) -> None:
152
+ self._json({"error": {"code": code, "message": message}}, status=status)
153
+
154
+ def _static(self, filename: str) -> None:
155
+ data = (STATIC_DIR / filename).read_bytes()
156
+ content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
157
+ if content_type.startswith("text/") or content_type == "application/javascript":
158
+ content_type += "; charset=utf-8"
159
+ self.send_response(HTTPStatus.OK)
160
+ self.send_header("Content-Type", content_type)
161
+ self.send_header("Content-Length", str(len(data)))
162
+ self.send_header("Cache-Control", "no-cache")
163
+ self._security_headers()
164
+ self.end_headers()
165
+ self.wfile.write(data)
166
+
167
+ def log_message(self, format: str, *args: Any) -> None:
168
+ sys.stderr.write(
169
+ f"annotation {self.client_address[0]} {self.command} "
170
+ f"{urlsplit(self.path).path} — {format % args}\n"
171
+ )
172
+
173
+
174
+ class _HTTPInputError(Exception):
175
+ def __init__(self, status: HTTPStatus, code: str, message: str) -> None:
176
+ super().__init__(message)
177
+ self.status = status
178
+ self.code = code
179
+ self.message = message
180
+
181
+
182
+ def create_server(
183
+ host: str,
184
+ port: int,
185
+ *,
186
+ items_path: str | Path,
187
+ output_root: str | Path,
188
+ kind: str,
189
+ ) -> AnnotationHTTPServer:
190
+ if host not in LOOPBACK_HOSTS:
191
+ raise ValueError("annotation UI is loopback-only; use SSH port forwarding on L20")
192
+ return AnnotationHTTPServer(
193
+ (host, port),
194
+ store=AnnotationStore(items_path=items_path, root=output_root, kind=kind),
195
+ )
196
+
197
+
198
+ def main(argv: list[str] | None = None) -> int:
199
+ parser = argparse.ArgumentParser(description=__doc__)
200
+ parser.add_argument("--items", type=Path, required=True)
201
+ parser.add_argument("--output-root", type=Path, required=True)
202
+ parser.add_argument("--kind", choices=sorted(ANNOTATION_KINDS), required=True)
203
+ parser.add_argument("--host", default="127.0.0.1")
204
+ parser.add_argument("--port", type=int, default=7862)
205
+ args = parser.parse_args(argv)
206
+ if args.host not in LOOPBACK_HOSTS:
207
+ print("annotation UI is loopback-only", file=sys.stderr)
208
+ return 2
209
+ server = create_server(
210
+ args.host,
211
+ args.port,
212
+ items_path=args.items,
213
+ output_root=args.output_root,
214
+ kind=args.kind,
215
+ )
216
+ host, port = server.server_address[:2]
217
+ print(f"AgentInputLab annotation UI listening on http://{host}:{port}")
218
+ if server.store.status()["completed_version_exists"]:
219
+ print("Completed annotation found: choose reuse or create a named new version in the UI.")
220
+ try:
221
+ server.serve_forever(poll_interval=0.25)
222
+ except KeyboardInterrupt:
223
+ print("\nStopping annotation UI")
224
+ finally:
225
+ server.server_close()
226
+ return 0
227
+
228
+
229
+ if __name__ == "__main__":
230
+ raise SystemExit(main())
231
+
232
+
233
+ __all__ = ["AnnotationHTTPServer", "create_server", "main"]