agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. agentinputlab/__init__.py +28 -0
  2. agentinputlab/algorithms/__init__.py +6 -0
  3. agentinputlab/algorithms/acquire/__init__.py +145 -0
  4. agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
  5. agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
  6. agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
  7. agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
  8. agentinputlab/algorithms/acquire/hyde.py +296 -0
  9. agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
  10. agentinputlab/algorithms/acquire/hyde_external.py +148 -0
  11. agentinputlab/algorithms/acquire/ircot.py +28 -0
  12. agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
  13. agentinputlab/algorithms/acquire/ircot_native.py +355 -0
  14. agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
  15. agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
  16. agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
  17. agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
  18. agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
  19. agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
  20. agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
  21. agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
  22. agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
  23. agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
  24. agentinputlab/algorithms/clarify/__init__.py +61 -0
  25. agentinputlab/algorithms/clarify/_structured.py +64 -0
  26. agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
  27. agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
  28. agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
  29. agentinputlab/algorithms/clarify/future_turns.py +379 -0
  30. agentinputlab/algorithms/clarify/udcs.py +255 -0
  31. agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
  32. agentinputlab/algorithms/curate/__init__.py +65 -0
  33. agentinputlab/algorithms/curate/acon/__init__.py +27 -0
  34. agentinputlab/algorithms/curate/acon/common.py +284 -0
  35. agentinputlab/algorithms/curate/acon/common_support.py +136 -0
  36. agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
  37. agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
  38. agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
  39. agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
  40. agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
  41. agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
  42. agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
  43. agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
  44. agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
  45. agentinputlab/algorithms/curate/recomp/external.py +1129 -0
  46. agentinputlab/algorithms/curate/recomp/selective.py +304 -0
  47. agentinputlab/algorithms/decompose/__init__.py +107 -0
  48. agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
  49. agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
  50. agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
  51. agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
  52. agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
  53. agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
  54. agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
  55. agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
  56. agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
  57. agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
  58. agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
  59. agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
  60. agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
  61. agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
  62. agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
  63. agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
  64. agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
  65. agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
  66. agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
  67. agentinputlab/algorithms/reformulate/__init__.py +85 -0
  68. agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
  69. agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
  70. agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
  71. agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
  72. agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
  73. agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
  74. agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
  75. agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
  76. agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
  77. agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
  78. agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
  79. agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
  80. agentinputlab/algorithms/reformulate/rrr.py +696 -0
  81. agentinputlab/annotation/__init__.py +5 -0
  82. agentinputlab/annotation/independent.py +428 -0
  83. agentinputlab/annotation/server.py +233 -0
  84. agentinputlab/annotation/static/app.js +121 -0
  85. agentinputlab/annotation/static/index.html +60 -0
  86. agentinputlab/annotation/static/review.css +2 -0
  87. agentinputlab/annotation/static/review.html +10 -0
  88. agentinputlab/annotation/static/review.js +11 -0
  89. agentinputlab/annotation/static/styles.css +55 -0
  90. agentinputlab/annotation/store.py +444 -0
  91. agentinputlab/api.py +368 -0
  92. agentinputlab/artifacts/__init__.py +15 -0
  93. agentinputlab/artifacts/json_file.py +104 -0
  94. agentinputlab/artifacts/manifest.py +98 -0
  95. agentinputlab/artifacts/store.py +157 -0
  96. agentinputlab/backends/__init__.py +98 -0
  97. agentinputlab/backends/embedding/__init__.py +25 -0
  98. agentinputlab/backends/embedding/base.py +145 -0
  99. agentinputlab/backends/embedding/callable.py +75 -0
  100. agentinputlab/backends/embedding/replay.py +129 -0
  101. agentinputlab/backends/llm/__init__.py +31 -0
  102. agentinputlab/backends/llm/base.py +149 -0
  103. agentinputlab/backends/llm/callable.py +108 -0
  104. agentinputlab/backends/llm/decomp_completion.py +88 -0
  105. agentinputlab/backends/llm/final_content_stop.py +72 -0
  106. agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
  107. agentinputlab/backends/llm/openai_compatible.py +332 -0
  108. agentinputlab/backends/llm/transformers.py +171 -0
  109. agentinputlab/backends/retrieval/__init__.py +35 -0
  110. agentinputlab/backends/retrieval/base.py +50 -0
  111. agentinputlab/backends/retrieval/dense.py +777 -0
  112. agentinputlab/backends/retrieval/memory.py +119 -0
  113. agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
  114. agentinputlab/backends/user/__init__.py +15 -0
  115. agentinputlab/backends/user/base.py +66 -0
  116. agentinputlab/backends/user/callable.py +54 -0
  117. agentinputlab/backends/user/replay.py +89 -0
  118. agentinputlab/backends/vector.py +38 -0
  119. agentinputlab/benchmarks/__init__.py +246 -0
  120. agentinputlab/benchmarks/acquisition.py +106 -0
  121. agentinputlab/benchmarks/ale.py +540 -0
  122. agentinputlab/benchmarks/appworld_ul.py +669 -0
  123. agentinputlab/benchmarks/assets.py +226 -0
  124. agentinputlab/benchmarks/base.py +68 -0
  125. agentinputlab/benchmarks/beir_archive.py +235 -0
  126. agentinputlab/benchmarks/beir_sample.py +269 -0
  127. agentinputlab/benchmarks/bfcl_native.py +245 -0
  128. agentinputlab/benchmarks/bfcl_v3.py +607 -0
  129. agentinputlab/benchmarks/cache.py +218 -0
  130. agentinputlab/benchmarks/clariq.py +203 -0
  131. agentinputlab/benchmarks/data/registry.json +321 -0
  132. agentinputlab/benchmarks/data/smoke_assets.json +511 -0
  133. agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
  134. agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
  135. agentinputlab/benchmarks/execution.py +1111 -0
  136. agentinputlab/benchmarks/interactive.py +257 -0
  137. agentinputlab/benchmarks/ir_query.py +204 -0
  138. agentinputlab/benchmarks/longbench.py +203 -0
  139. agentinputlab/benchmarks/paired_replay.py +479 -0
  140. agentinputlab/benchmarks/preprocess.py +2313 -0
  141. agentinputlab/benchmarks/production.py +236 -0
  142. agentinputlab/benchmarks/qa.py +963 -0
  143. agentinputlab/benchmarks/registry.py +110 -0
  144. agentinputlab/benchmarks/retrieval.py +459 -0
  145. agentinputlab/benchmarks/runtime.py +509 -0
  146. agentinputlab/benchmarks/statebench.py +938 -0
  147. agentinputlab/benchmarks/tau3.py +725 -0
  148. agentinputlab/benchmarks/toolsandbox.py +626 -0
  149. agentinputlab/benchmarks/trec_dl_sample.py +263 -0
  150. agentinputlab/campaign.py +654 -0
  151. agentinputlab/campaign_runner.py +953 -0
  152. agentinputlab/cli/__init__.py +6 -0
  153. agentinputlab/cli/main.py +933 -0
  154. agentinputlab/connectors/__init__.py +46 -0
  155. agentinputlab/connectors/knowledge/__init__.py +4 -0
  156. agentinputlab/connectors/tools/__init__.py +30 -0
  157. agentinputlab/connectors/tools/base.py +57 -0
  158. agentinputlab/connectors/tools/mcp.py +222 -0
  159. agentinputlab/connectors/tools/memory.py +41 -0
  160. agentinputlab/connectors/tools/openapi.py +695 -0
  161. agentinputlab/connectors/tools/schema.py +102 -0
  162. agentinputlab/contracts/__init__.py +91 -0
  163. agentinputlab/contracts/availability.py +24 -0
  164. agentinputlab/contracts/dependency.py +58 -0
  165. agentinputlab/contracts/enums.py +82 -0
  166. agentinputlab/contracts/errors.py +63 -0
  167. agentinputlab/contracts/evidence.py +43 -0
  168. agentinputlab/contracts/intent.py +110 -0
  169. agentinputlab/contracts/lifecycle.py +164 -0
  170. agentinputlab/contracts/patch.py +167 -0
  171. agentinputlab/contracts/serialization.py +93 -0
  172. agentinputlab/contracts/state.py +127 -0
  173. agentinputlab/contracts/trace.py +59 -0
  174. agentinputlab/contracts/worker.py +42 -0
  175. agentinputlab/core/__init__.py +7 -0
  176. agentinputlab/data/__init__.py +32 -0
  177. agentinputlab/data/checksums.py +9 -0
  178. agentinputlab/data/downloader.py +235 -0
  179. agentinputlab/data/http.py +171 -0
  180. agentinputlab/data/index.py +21 -0
  181. agentinputlab/data/overlap.py +28 -0
  182. agentinputlab/data/preprocess.py +32 -0
  183. agentinputlab/data/storage.py +57 -0
  184. agentinputlab/data/trec_sample.py +427 -0
  185. agentinputlab/evaluation/__init__.py +5 -0
  186. agentinputlab/evaluation/outcomes.py +32 -0
  187. agentinputlab/experiments/__init__.py +278 -0
  188. agentinputlab/experiments/analysis.py +981 -0
  189. agentinputlab/experiments/artifacts.py +687 -0
  190. agentinputlab/experiments/automatic_measurement.py +678 -0
  191. agentinputlab/experiments/auxiliary_matrix.py +203 -0
  192. agentinputlab/experiments/bindings.py +430 -0
  193. agentinputlab/experiments/blueprints.py +147 -0
  194. agentinputlab/experiments/builders/__init__.py +45 -0
  195. agentinputlab/experiments/builders/executable_external_payload.py +423 -0
  196. agentinputlab/experiments/builders/external_process_arena.py +1018 -0
  197. agentinputlab/experiments/builders/registry_intervention.py +463 -0
  198. agentinputlab/experiments/cli.py +158 -0
  199. agentinputlab/experiments/controls.py +340 -0
  200. agentinputlab/experiments/driver_guard.py +205 -0
  201. agentinputlab/experiments/drivers/__init__.py +89 -0
  202. agentinputlab/experiments/drivers/external_arena.py +2908 -0
  203. agentinputlab/experiments/drivers/method_process.py +662 -0
  204. agentinputlab/experiments/drivers/parity_process.py +1484 -0
  205. agentinputlab/experiments/drivers/static_production.py +1574 -0
  206. agentinputlab/experiments/executor.py +448 -0
  207. agentinputlab/experiments/loader.py +605 -0
  208. agentinputlab/experiments/measurement.py +675 -0
  209. agentinputlab/experiments/paper_analysis.py +1818 -0
  210. agentinputlab/experiments/parity_identity.py +605 -0
  211. agentinputlab/experiments/plan.py +277 -0
  212. agentinputlab/experiments/production_builders.py +3739 -0
  213. agentinputlab/experiments/readiness.py +432 -0
  214. agentinputlab/experiments/schema.py +514 -0
  215. agentinputlab/experiments/secrets.py +177 -0
  216. agentinputlab/experiments/stateful_analysis.py +1738 -0
  217. agentinputlab/experiments/transfer_analysis.py +1316 -0
  218. agentinputlab/experiments/unseen_analysis.py +1025 -0
  219. agentinputlab/integrations/__init__.py +1 -0
  220. agentinputlab/integrations/action_space/__init__.py +1 -0
  221. agentinputlab/integrations/external_workers/__init__.py +76 -0
  222. agentinputlab/integrations/external_workers/ace.py +431 -0
  223. agentinputlab/integrations/external_workers/base.py +181 -0
  224. agentinputlab/integrations/external_workers/decomp_live.py +249 -0
  225. agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
  226. agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
  227. agentinputlab/integrations/external_workers/ircot.py +576 -0
  228. agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
  229. agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
  230. agentinputlab/integrations/external_workers/method.py +314 -0
  231. agentinputlab/integrations/external_workers/process.py +678 -0
  232. agentinputlab/integrations/external_workers/supervisor.py +1071 -0
  233. agentinputlab/integrations/full_architecture/__init__.py +1 -0
  234. agentinputlab/integrations/full_architecture/hyde.py +212 -0
  235. agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
  236. agentinputlab/integrations/live_history.py +222 -0
  237. agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
  238. agentinputlab/integrations/training/__init__.py +1 -0
  239. agentinputlab/metrics/__init__.py +50 -0
  240. agentinputlab/metrics/catalog.py +313 -0
  241. agentinputlab/metrics/harm.py +95 -0
  242. agentinputlab/metrics/paired.py +40 -0
  243. agentinputlab/metrics/qa.py +76 -0
  244. agentinputlab/metrics/text.py +33 -0
  245. agentinputlab/metrics/third_party.py +161 -0
  246. agentinputlab/models/__init__.py +4 -0
  247. agentinputlab/observability/__init__.py +56 -0
  248. agentinputlab/observability/budget.py +316 -0
  249. agentinputlab/observability/instrumentation.py +754 -0
  250. agentinputlab/observability/models.py +403 -0
  251. agentinputlab/observability/monitor.py +380 -0
  252. agentinputlab/optimizers/__init__.py +1 -0
  253. agentinputlab/optimizers/clarify/__init__.py +12 -0
  254. agentinputlab/optimizers/context/__init__.py +22 -0
  255. agentinputlab/optimizers/decompose/__init__.py +64 -0
  256. agentinputlab/optimizers/evidence/__init__.py +22 -0
  257. agentinputlab/optimizers/rewrite/__init__.py +30 -0
  258. agentinputlab/parity/__init__.py +5 -0
  259. agentinputlab/parity/base.py +206 -0
  260. agentinputlab/production_campaign.py +763 -0
  261. agentinputlab/py.typed +1 -0
  262. agentinputlab/recipes/__init__.py +333 -0
  263. agentinputlab/recipes/ace.py +418 -0
  264. agentinputlab/recipes/ace_bfcl.py +273 -0
  265. agentinputlab/recipes/ace_common_support.py +169 -0
  266. agentinputlab/recipes/ace_native_plan.py +322 -0
  267. agentinputlab/recipes/ace_online.py +711 -0
  268. agentinputlab/recipes/ace_retrieval.py +330 -0
  269. agentinputlab/recipes/ace_statebench.py +382 -0
  270. agentinputlab/recipes/ace_tau.py +463 -0
  271. agentinputlab/recipes/ace_toolsandbox.py +432 -0
  272. agentinputlab/recipes/acon_guideline.py +22 -0
  273. agentinputlab/recipes/adaptive_rag.py +822 -0
  274. agentinputlab/recipes/llmlingua2_inference.py +146 -0
  275. agentinputlab/recipes/llmlingua2_training.py +796 -0
  276. agentinputlab/recipes/mce.py +1486 -0
  277. agentinputlab/recipes/mce_final.py +335 -0
  278. agentinputlab/recipes/mce_text_fit.py +499 -0
  279. agentinputlab/recipes/mce_text_only.py +298 -0
  280. agentinputlab/recipes/qrag.py +729 -0
  281. agentinputlab/recipes/qrag_inference.py +364 -0
  282. agentinputlab/recipes/r2ag_features.py +236 -0
  283. agentinputlab/recipes/r2ag_inference.py +503 -0
  284. agentinputlab/recipes/r2ag_training.py +948 -0
  285. agentinputlab/recipes/recomp_abstractive.py +592 -0
  286. agentinputlab/recipes/recomp_extractive.py +525 -0
  287. agentinputlab/recipes/reformer.py +17 -0
  288. agentinputlab/recipes/rqrag.py +1516 -0
  289. agentinputlab/recipes/rqrag_common_support.py +193 -0
  290. agentinputlab/recipes/rqrag_completion.py +147 -0
  291. agentinputlab/recipes/skillopt.py +462 -0
  292. agentinputlab/recipes/skillopt_common_support.py +154 -0
  293. agentinputlab/recipes/skillopt_environment.py +165 -0
  294. agentinputlab/recipes/skillopt_environment_cli.py +5 -0
  295. agentinputlab/recipes/skillopt_retrieval.py +188 -0
  296. agentinputlab/recipes/smartsearch.py +897 -0
  297. agentinputlab/recipes/smartsearch_curriculum.py +283 -0
  298. agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
  299. agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
  300. agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
  301. agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
  302. agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
  303. agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
  304. agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
  305. agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
  306. agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
  307. agentinputlab/recipes/smartsearch_inference.py +524 -0
  308. agentinputlab/recipes/smartsearch_native_data.py +172 -0
  309. agentinputlab/recipes/smartsearch_policy.py +616 -0
  310. agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
  311. agentinputlab/recipes/smartsearch_sampling.py +232 -0
  312. agentinputlab/recipes/smartsearch_teacher.py +203 -0
  313. agentinputlab/recipes/smartsearch_training.py +265 -0
  314. agentinputlab/registry/__init__.py +46 -0
  315. agentinputlab/registry/beta.py +727 -0
  316. agentinputlab/registry/data/algorithms.json +1959 -0
  317. agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
  318. agentinputlab/registry/loader.py +100 -0
  319. agentinputlab/registry/result_manifest.py +184 -0
  320. agentinputlab/registry/runtime.py +195 -0
  321. agentinputlab/registry/schema.py +449 -0
  322. agentinputlab/runner/__init__.py +30 -0
  323. agentinputlab/runner/balance.py +248 -0
  324. agentinputlab/runner/cache.py +135 -0
  325. agentinputlab/runner/composite.py +130 -0
  326. agentinputlab/runner/engine.py +285 -0
  327. agentinputlab/runner/runtime_guard.py +268 -0
  328. agentinputlab/runner/worker.py +80 -0
  329. agentinputlab/runtime_bootstrap.py +288 -0
  330. agentinputlab/selectors/__init__.py +196 -0
  331. agentinputlab/selectors/advanced.py +1514 -0
  332. agentinputlab/selectors/artifacts.py +1087 -0
  333. agentinputlab/selectors/base.py +112 -0
  334. agentinputlab/selectors/campaigns.py +1893 -0
  335. agentinputlab/selectors/catalog.py +86 -0
  336. agentinputlab/selectors/discovery.py +1860 -0
  337. agentinputlab/selectors/evaluation.py +1348 -0
  338. agentinputlab/selectors/features.py +383 -0
  339. agentinputlab/selectors/learned.py +502 -0
  340. agentinputlab/selectors/no_op.py +28 -0
  341. agentinputlab/selectors/outcome_table.py +475 -0
  342. agentinputlab/selectors/producer.py +853 -0
  343. agentinputlab/selectors/rules.py +109 -0
  344. agentinputlab/selectors/runtime.py +420 -0
  345. agentinputlab/studies/__init__.py +16 -0
  346. agentinputlab/studies/design.py +165 -0
  347. agentinputlab/studies/paired.py +322 -0
  348. agentinputlab/studio/__init__.py +6 -0
  349. agentinputlab/studio/campaigns.py +819 -0
  350. agentinputlab/studio/engine.py +1408 -0
  351. agentinputlab/studio/model_backends.py +425 -0
  352. agentinputlab/studio/prepared.py +410 -0
  353. agentinputlab/studio/server.py +613 -0
  354. agentinputlab/studio/static/app.js +397 -0
  355. agentinputlab/studio/static/campaigns.html +175 -0
  356. agentinputlab/studio/static/campaigns.js +426 -0
  357. agentinputlab/studio/static/index.html +192 -0
  358. agentinputlab/studio/static/result.html +103 -0
  359. agentinputlab/studio/static/result.js +273 -0
  360. agentinputlab/studio/static/styles.css +467 -0
  361. agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
  362. agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
  363. agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
  364. agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
  365. agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
  366. agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
  367. agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
  368. agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
  369. agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
  370. workers/README.md +94 -0
  371. workers/ace/README.md +261 -0
  372. workers/ace/__init__.py +1 -0
  373. workers/ace/compatibility_manifest.json +39 -0
  374. workers/ace/driver.py +1682 -0
  375. workers/acon/README.md +215 -0
  376. workers/acon/__init__.py +5 -0
  377. workers/acon/compatibility_manifest.json +93 -0
  378. workers/acon/driver.py +1859 -0
  379. workers/acon/fitted.py +395 -0
  380. workers/acon/guideline.py +470 -0
  381. workers/acon/guideline_source_manifest.json +20 -0
  382. workers/adaptive_rag/README.md +96 -0
  383. workers/adaptive_rag/__init__.py +15 -0
  384. workers/adaptive_rag/compatibility_manifest.json +61 -0
  385. workers/adaptive_rag/driver.py +1470 -0
  386. workers/adaptive_rag/fixture_runtime.py +163 -0
  387. workers/adaptive_rag/runtime.py +252 -0
  388. workers/ask_or_assume/README.md +124 -0
  389. workers/ask_or_assume/__init__.py +1 -0
  390. workers/ask_or_assume/compatibility_manifest.json +36 -0
  391. workers/ask_or_assume/driver.py +1227 -0
  392. workers/decomp/README.md +133 -0
  393. workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
  394. workers/decomp/__init__.py +5 -0
  395. workers/decomp/compatibility_manifest.json +205 -0
  396. workers/decomp/driver.py +1015 -0
  397. workers/decomp/live.py +137 -0
  398. workers/decomp/runtime.py +231 -0
  399. workers/infocqr/README.md +106 -0
  400. workers/infocqr/__init__.py +1 -0
  401. workers/infocqr/compatibility_manifest.json +84 -0
  402. workers/infocqr/driver.py +1766 -0
  403. workers/ircot/README.md +167 -0
  404. workers/ircot/__init__.py +7 -0
  405. workers/ircot/compatibility_manifest.json +121 -0
  406. workers/ircot/controlled.py +457 -0
  407. workers/ircot/driver.py +1373 -0
  408. workers/ircot/fixture_bindings.json +39 -0
  409. workers/ircot/fixture_config.json +48 -0
  410. workers/ircot/fixture_prompt.txt +2 -0
  411. workers/ircot/native_qa.py +398 -0
  412. workers/ircot/qa_compatibility_manifest.json +57 -0
  413. workers/ircot/runtime.py +97 -0
  414. workers/llmcompiler/README.md +160 -0
  415. workers/llmcompiler/__init__.py +1 -0
  416. workers/llmcompiler/compatibility_manifest.json +45 -0
  417. workers/llmcompiler/driver.py +1514 -0
  418. workers/llmcompiler/export_prompts.py +74 -0
  419. workers/llmcompiler/live.py +174 -0
  420. workers/llmcompiler/patch_manifest.json +147 -0
  421. workers/mce/README.md +198 -0
  422. workers/mce/__init__.py +29 -0
  423. workers/mce/compatibility_manifest.json +115 -0
  424. workers/mce/driver.py +2585 -0
  425. workers/mce/final_eval.py +173 -0
  426. workers/official_arenas/README.md +25 -0
  427. workers/official_arenas/__init__.py +1 -0
  428. workers/official_arenas/capture_records.py +110 -0
  429. workers/official_arenas/driver.py +619 -0
  430. workers/official_arenas/protocol.py +334 -0
  431. workers/official_arenas/providers/__init__.py +1 -0
  432. workers/official_arenas/providers/_live_history.py +228 -0
  433. workers/official_arenas/providers/_message_compat.py +77 -0
  434. workers/official_arenas/providers/appworld_ul.py +196 -0
  435. workers/official_arenas/providers/fixture.py +112 -0
  436. workers/official_arenas/providers/statebench.py +960 -0
  437. workers/official_arenas/providers/tau3.py +1672 -0
  438. workers/official_arenas/providers/tau_retail.py +586 -0
  439. workers/official_arenas/providers/toolsandbox.py +1321 -0
  440. workers/official_arenas/providers/tooltalk.py +699 -0
  441. workers/official_arenas/retail_capture.py +188 -0
  442. workers/official_arenas/retail_driver.py +275 -0
  443. workers/qrag/README.md +172 -0
  444. workers/qrag/__init__.py +1 -0
  445. workers/qrag/compatibility_manifest.json +64 -0
  446. workers/qrag/driver.py +673 -0
  447. workers/r2ag/README.md +154 -0
  448. workers/r2ag/__init__.py +5 -0
  449. workers/r2ag/compatibility_manifest.json +55 -0
  450. workers/r2ag/driver.py +1287 -0
  451. workers/r2ag/feature_schema.json +8 -0
  452. workers/r2ag/runtime.py +589 -0
  453. workers/recomp/README.md +120 -0
  454. workers/recomp/__init__.py +1 -0
  455. workers/recomp/compatibility_manifest.json +43 -0
  456. workers/recomp/driver.py +1653 -0
  457. workers/reformer/README.md +125 -0
  458. workers/reformer/__init__.py +1 -0
  459. workers/reformer/compatibility_manifest.json +68 -0
  460. workers/reformer/driver.py +1459 -0
  461. workers/reformer/induction.py +457 -0
  462. workers/rrr/README.md +126 -0
  463. workers/rrr/__init__.py +1 -0
  464. workers/rrr/compatibility_manifest.json +57 -0
  465. workers/rrr/driver.py +1133 -0
  466. workers/smartsearch/README.md +331 -0
  467. workers/smartsearch/__init__.py +1 -0
  468. workers/smartsearch/compatibility_manifest.json +41 -0
  469. workers/smartsearch/driver.py +1643 -0
@@ -0,0 +1,1484 @@
1
+ """Exact-SHA, two-process driver for upstream-versus-candidate parity checks.
2
+
3
+ The host, rather than either worker, establishes implementation identity. Each
4
+ arm is started in a fresh bounded subprocess with a checkout-specific working
5
+ directory and ``PYTHONPATH``. The host independently verifies the checkout,
6
+ executable, entrypoint, and command before and after every invocation and emits
7
+ an execution receipt that the guarded experiment driver validates.
8
+
9
+ This driver provides execution evidence only. A successful run is not, by
10
+ itself, a paper-result or official-parity claim.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import hashlib
16
+ import json
17
+ import os
18
+ import re
19
+ import stat
20
+ import subprocess
21
+ from collections.abc import Mapping, Sequence
22
+ from dataclasses import dataclass
23
+ from pathlib import Path, PurePosixPath
24
+ from typing import Any
25
+
26
+ from agentinputlab.contracts.errors import (
27
+ AlgorithmOutputError,
28
+ ArtifactError,
29
+ ExternalDependencyError,
30
+ ValidationError,
31
+ )
32
+ from agentinputlab.contracts.serialization import (
33
+ canonical_json,
34
+ content_hash,
35
+ freeze_mapping,
36
+ to_jsonable,
37
+ )
38
+ from agentinputlab.experiments.bindings import resolve_frozen_binding_receipt
39
+ from agentinputlab.experiments.executor import ExperimentCellResult
40
+ from agentinputlab.experiments.parity_identity import (
41
+ HOST_EXECUTION_RECEIPT_PROTOCOL,
42
+ build_parity_implementation_contract,
43
+ )
44
+ from agentinputlab.experiments.plan import ExperimentCell
45
+ from agentinputlab.experiments.schema import ExperimentManifest, ExperimentStage, require_sha256
46
+ from agentinputlab.experiments.secrets import (
47
+ environment_secret_values,
48
+ redact_secret_text,
49
+ require_secret_free,
50
+ require_secret_values_absent,
51
+ )
52
+ from agentinputlab.integrations.external_workers.supervisor import (
53
+ DockerWorkerSupervisor,
54
+ LocalSubprocessSupervisor,
55
+ SupervisedProcessResult,
56
+ SupervisedWorkerError,
57
+ WorkerSupervisor,
58
+ worker_supervisor_from_environment,
59
+ )
60
+ from agentinputlab.observability import ResourceLimits, ResourceReceipt, ResourceUsage
61
+
62
+ CONFIG_PATH_ENV = "AILAB_PARITY_PROCESS_CONFIG"
63
+ CONFIG_SHA256_ENV = "AILAB_PARITY_PROCESS_CONFIG_SHA256"
64
+ UNIT_INVENTORY_ENV = "AILAB_PARITY_UNIT_INVENTORY"
65
+ UNIT_INVENTORY_SHA256_ENV = "AILAB_PARITY_UNIT_INVENTORY_SHA256"
66
+ REQUEST_PROTOCOL = "agentinputlab.parity-process-request.v1"
67
+ RESPONSE_PROTOCOL = "agentinputlab.parity-process-response.v1"
68
+ DRIVER_PROTOCOL = "agentinputlab.parity-process-driver.v1"
69
+
70
+ _UPSTREAM_ARM = "upstream_reference"
71
+ _CANDIDATE_ARM = "agentinputlab_candidate"
72
+ _ARM_IDS = (_UPSTREAM_ARM, _CANDIDATE_ARM)
73
+ _CHECKOUT_ENVIRONMENTS = {
74
+ _UPSTREAM_ARM: "AILAB_UPSTREAM_CHECKOUT",
75
+ _CANDIDATE_ARM: "AILAB_CANDIDATE_CHECKOUT",
76
+ }
77
+ _BASE_PASSTHROUGH_ENVIRONMENTS = (
78
+ "AILAB_PARITY_ASSET_ROOT",
79
+ "AILAB_MODEL_CONFIG",
80
+ "AILAB_RETRIEVER_CONFIG",
81
+ )
82
+ _RESERVED_WORKER_ENVIRONMENTS = frozenset(
83
+ {
84
+ "AILAB_PARITY_ARM_ID",
85
+ "AILAB_PARITY_PROTOCOL",
86
+ "AILAB_PROCESS_ARM_ID",
87
+ "AILAB_PROCESS_PROTOCOL",
88
+ "LANG",
89
+ "LC_ALL",
90
+ "PWD",
91
+ "PYTHONPATH",
92
+ "PYTHONDONTWRITEBYTECODE",
93
+ "PYTHONIOENCODING",
94
+ "PYTHONNOUSERSITE",
95
+ "TZ",
96
+ }
97
+ )
98
+ _ENVIRONMENT_NAME = re.compile(r"^[A-Z_][A-Z0-9_]*$")
99
+ _GIT_OBJECT_ID = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
100
+ _USAGE_FIELDS = frozenset(
101
+ {
102
+ "model_calls",
103
+ "retrieval_calls",
104
+ "tool_calls",
105
+ "embedding_calls",
106
+ "unpriced_model_calls",
107
+ "input_tokens",
108
+ "output_tokens",
109
+ "cpu_seconds",
110
+ "accelerator_seconds",
111
+ "cost_usd",
112
+ "compute_cost_usd",
113
+ }
114
+ )
115
+ _PIPE_FIELDS = frozenset(
116
+ {
117
+ "max_input_bytes",
118
+ "max_stdout_bytes",
119
+ "max_stderr_bytes",
120
+ }
121
+ )
122
+ _WORKER_FIELDS = frozenset(
123
+ {
124
+ "checkout_environment",
125
+ "expected_checkout_revision",
126
+ "expected_checkout_tree",
127
+ "adapter_checkout_environment",
128
+ "expected_adapter_checkout_revision",
129
+ "expected_adapter_checkout_tree",
130
+ "executable_path",
131
+ "executable_sha256",
132
+ "entrypoint",
133
+ "entrypoint_sha256",
134
+ "arguments",
135
+ "pass_environment",
136
+ "request_parameters",
137
+ }
138
+ )
139
+ _CONFIG_FIELDS = frozenset(
140
+ {
141
+ "schema_version",
142
+ "protocol",
143
+ "method_id",
144
+ "candidate_release_id",
145
+ "workers",
146
+ "resource_limits",
147
+ "pipe_limits",
148
+ "shared_request_parameters",
149
+ }
150
+ )
151
+ _RESPONSE_FIELDS = frozenset(
152
+ {
153
+ "schema_version",
154
+ "protocol",
155
+ "request_hash",
156
+ "status",
157
+ "metrics",
158
+ "usage",
159
+ "output",
160
+ "failure",
161
+ }
162
+ )
163
+ _RESPONSE_FORBIDDEN_KEYS = frozenset(
164
+ {
165
+ "implementation_identity",
166
+ "host_execution_receipt",
167
+ "resource_receipt",
168
+ }
169
+ )
170
+
171
+
172
+ @dataclass(frozen=True, slots=True)
173
+ class _CheckoutIdentity:
174
+ resolved_path_sha256: str
175
+ git_head: str
176
+ git_tree: str
177
+
178
+ def to_dict(self) -> dict[str, Any]:
179
+ return to_jsonable(self)
180
+
181
+
182
+ @dataclass(frozen=True, slots=True)
183
+ class _CommandIdentity:
184
+ executable_resolved_path_sha256: str
185
+ executable_content_sha256: str
186
+ entrypoint_relative_path: str
187
+ entrypoint_resolved_path_sha256: str
188
+ entrypoint_content_sha256: str
189
+ argv_sha256: str
190
+ cwd_resolved_path_sha256: str
191
+ pythonpath_resolved_path_sha256: tuple[str, ...]
192
+
193
+ def to_dict(self) -> dict[str, Any]:
194
+ return to_jsonable(self)
195
+
196
+
197
+ @dataclass(frozen=True, slots=True)
198
+ class _WorkerSpec:
199
+ arm_id: str
200
+ checkout_environment: str
201
+ checkout: Path
202
+ expected_checkout_revision: str
203
+ expected_checkout_tree: str
204
+ adapter_checkout_environment: str
205
+ adapter_checkout: Path
206
+ expected_adapter_checkout_revision: str
207
+ expected_adapter_checkout_tree: str
208
+ executable: Path
209
+ executable_sha256: str
210
+ entrypoint_relative_path: str
211
+ entrypoint: Path
212
+ entrypoint_sha256: str
213
+ arguments: tuple[str, ...]
214
+ pass_environment: tuple[str, ...]
215
+ request_parameters: Mapping[str, Any]
216
+ checkout_identity: _CheckoutIdentity
217
+ adapter_checkout_identity: _CheckoutIdentity
218
+ command_identity: _CommandIdentity
219
+
220
+ @property
221
+ def argv(self) -> tuple[str, ...]:
222
+ return (str(self.executable), str(self.entrypoint), *self.arguments)
223
+
224
+
225
+ @dataclass(frozen=True, slots=True)
226
+ class _DriverConfiguration:
227
+ method_id: str
228
+ candidate_release_id: str
229
+ workers: Mapping[str, _WorkerSpec]
230
+ resource_limits: ResourceLimits
231
+ pipe_limits: Mapping[str, int]
232
+ shared_request_parameters: Mapping[str, Any]
233
+ config_sha256: str
234
+
235
+
236
+ class ParityProcessProtocolError(AlgorithmOutputError):
237
+ """A worker violated the parity-process protocol after it was started."""
238
+
239
+ def __init__(self, message: str, *, resource_receipt: ResourceReceipt) -> None:
240
+ super().__init__(message)
241
+ self.resource_receipt = resource_receipt
242
+
243
+
244
+ class ParityProcessExperimentDriver:
245
+ """Run exact upstream and candidate adapters as independent JSON workers."""
246
+
247
+ def __init__(
248
+ self,
249
+ configuration: _DriverConfiguration,
250
+ *,
251
+ environment: Mapping[str, str],
252
+ supervisor: WorkerSupervisor | None = None,
253
+ ) -> None:
254
+ self._configuration = configuration
255
+ self._environment = dict(environment)
256
+ self._supervisor = supervisor or LocalSubprocessSupervisor()
257
+ if not isinstance(self._supervisor, WorkerSupervisor):
258
+ raise TypeError("parity-process supervisor must implement WorkerSupervisor")
259
+ self._manifest_hash: str | None = None
260
+ self._binding_receipt_hash: str | None = None
261
+ self._contract_hash: str | None = None
262
+ self._preflight_worker_state: Mapping[str, Mapping[str, Any]] = freeze_mapping({})
263
+ self._fixture_unit_ids: frozenset[str] = frozenset()
264
+ self._checkout_binding_hashes: Mapping[str, str] = freeze_mapping({})
265
+ self._adapter_checkout_binding_hashes: Mapping[str, str] = freeze_mapping({})
266
+ self._fingerprint = freeze_mapping(self._build_fingerprint())
267
+
268
+ @classmethod
269
+ def from_config(
270
+ cls,
271
+ path: str | Path,
272
+ *,
273
+ expected_sha256: str,
274
+ environment: Mapping[str, str] | None = None,
275
+ supervisor: WorkerSupervisor | None = None,
276
+ ) -> ParityProcessExperimentDriver:
277
+ env = dict(os.environ if environment is None else environment)
278
+ configuration = _load_configuration(
279
+ Path(path),
280
+ expected_sha256=expected_sha256,
281
+ environment=env,
282
+ )
283
+ return cls(
284
+ configuration,
285
+ environment=env,
286
+ supervisor=supervisor,
287
+ )
288
+
289
+ def fingerprint(self) -> Mapping[str, Any]:
290
+ return self._fingerprint
291
+
292
+ def preflight(self, manifest: ExperimentManifest) -> Sequence[str]:
293
+ blockers: list[str] = []
294
+ if manifest.stage is not ExperimentStage.PARITY:
295
+ blockers.append("parity-process driver requires stage=parity")
296
+ if manifest.metadata.get("method_id") != self._configuration.method_id:
297
+ blockers.append("parity-process config method_id differs from manifest")
298
+ if (
299
+ manifest.metadata.get("candidate_release_id")
300
+ != self._configuration.candidate_release_id
301
+ ):
302
+ blockers.append("parity-process candidate release differs from manifest")
303
+ if set(arm.arm_id for arm in manifest.arms) != set(_ARM_IDS):
304
+ blockers.append("parity-process driver requires exact upstream and candidate arms")
305
+ upstream_commit = manifest.metadata.get("upstream_commit")
306
+ if upstream_commit != self._configuration.workers[_UPSTREAM_ARM].expected_checkout_revision:
307
+ blockers.append("parity-process upstream checkout revision differs from manifest")
308
+ if isinstance(self._supervisor, LocalSubprocessSupervisor) and any(
309
+ value is not None
310
+ for value in (
311
+ self._configuration.resource_limits.memory_bytes,
312
+ self._configuration.resource_limits.cpus,
313
+ self._configuration.resource_limits.pids_limit,
314
+ )
315
+ ):
316
+ blockers.append(
317
+ "local parity-process supervision supports hard wall/pipe limits only; "
318
+ "use a container supervisor for memory/CPU/PID limits"
319
+ )
320
+ for worker in self._configuration.workers.values():
321
+ missing = sorted(
322
+ name
323
+ for name in (
324
+ *_BASE_PASSTHROUGH_ENVIRONMENTS,
325
+ *worker.pass_environment,
326
+ )
327
+ if not self._environment.get(name)
328
+ )
329
+ if missing:
330
+ blockers.append(
331
+ f"parity-process {worker.arm_id} missing worker environments: "
332
+ + ", ".join(missing)
333
+ )
334
+ try:
335
+ fixture_units = _load_fixture_inventory(self._environment)
336
+ except (ArtifactError, ValidationError) as exc:
337
+ blockers.append(redact_secret_text(str(exc)))
338
+ fixture_units = ()
339
+ if blockers:
340
+ return tuple(blockers)
341
+ try:
342
+ state = {
343
+ arm_id: _attest_worker(worker).to_dict()
344
+ for arm_id, worker in self._configuration.workers.items()
345
+ }
346
+ receipt = resolve_frozen_binding_receipt(
347
+ manifest,
348
+ environment=self._environment,
349
+ )
350
+ contract = build_parity_implementation_contract(
351
+ manifest,
352
+ receipt,
353
+ self._fingerprint,
354
+ )
355
+ except (ArtifactError, ExternalDependencyError, ValidationError) as exc:
356
+ return (redact_secret_text(str(exc)),)
357
+ self._manifest_hash = manifest.manifest_hash
358
+ self._binding_receipt_hash = receipt.receipt_hash
359
+ self._contract_hash = contract.contract_hash
360
+ self._preflight_worker_state = freeze_mapping(state)
361
+ self._fixture_unit_ids = frozenset(fixture_units)
362
+ self._checkout_binding_hashes = freeze_mapping(
363
+ {
364
+ arm_id: receipt.binding(
365
+ worker.checkout_environment,
366
+ kind="directory_env",
367
+ ).binding_hash
368
+ for arm_id, worker in self._configuration.workers.items()
369
+ }
370
+ )
371
+ self._adapter_checkout_binding_hashes = freeze_mapping(
372
+ {
373
+ arm_id: receipt.binding(
374
+ worker.adapter_checkout_environment,
375
+ kind="directory_env",
376
+ ).binding_hash
377
+ for arm_id, worker in self._configuration.workers.items()
378
+ }
379
+ )
380
+ return ()
381
+
382
+ def preflight_units(
383
+ self,
384
+ manifest: ExperimentManifest,
385
+ resolved_units: Mapping[str, Sequence[str]],
386
+ ) -> Sequence[str]:
387
+ """Prove the plan uses exactly the checksum-locked fixture inventory."""
388
+
389
+ if self._manifest_hash != manifest.manifest_hash or not self._fixture_unit_ids:
390
+ return ("unit inventory validation requires a successful driver preflight",)
391
+ expected_keys = {f"{target.cell_id}:{target.split}" for target in manifest.benchmarks}
392
+ if set(resolved_units) != expected_keys:
393
+ return ("resolved unit inventories differ from parity manifest targets",)
394
+ blockers: list[str] = []
395
+ observed: set[str] = set()
396
+ for key in sorted(expected_keys):
397
+ values = tuple(resolved_units[key])
398
+ if len(values) != len(set(values)):
399
+ blockers.append(f"{key}: resolved parity units contain duplicates")
400
+ continue
401
+ observed.update(values)
402
+ missing = sorted(self._fixture_unit_ids - observed)
403
+ extra = sorted(observed - self._fixture_unit_ids)
404
+ if missing or extra:
405
+ blockers.append(
406
+ "resolved parity units differ from the checksum-locked fixture inventory; "
407
+ f"missing={missing[:5]}, extra={extra[:5]}"
408
+ )
409
+ return tuple(blockers)
410
+
411
+ def execute_cell(
412
+ self,
413
+ manifest: ExperimentManifest,
414
+ cell: ExperimentCell,
415
+ ) -> ExperimentCellResult:
416
+ self._require_preflight(manifest, cell)
417
+ arm_outcomes: list[dict[str, Any]] = []
418
+ resource_receipts: list[ResourceReceipt] = []
419
+ for arm_id in cell.arm_ids:
420
+ worker = self._configuration.workers[arm_id]
421
+ response, host_receipt, resource_receipt = self._execute_arm(
422
+ manifest,
423
+ cell,
424
+ worker,
425
+ )
426
+ resource_receipts.append(resource_receipt)
427
+ outcome: dict[str, Any] = {
428
+ "arm_id": arm_id,
429
+ "status": response["status"],
430
+ "request_hash": response["request_hash"],
431
+ "usage": response["usage"],
432
+ "output": response["output"],
433
+ "host_execution_receipt": host_receipt,
434
+ }
435
+ if response["status"] == "succeeded":
436
+ outcome["metrics"] = response["metrics"]
437
+ else:
438
+ outcome["failure"] = response["failure"]
439
+ arm_outcomes.append(outcome)
440
+ return ExperimentCellResult(
441
+ cell_id=cell.cell_id,
442
+ status="succeeded",
443
+ payload={
444
+ "frozen_binding_receipt_hash": self._binding_receipt_hash,
445
+ "parity_contract_hash": self._contract_hash,
446
+ "host_execution_receipt_protocol": HOST_EXECUTION_RECEIPT_PROTOCOL,
447
+ "arms": arm_outcomes,
448
+ },
449
+ resource_receipts=tuple(resource_receipts),
450
+ )
451
+
452
+ def _build_fingerprint(self) -> dict[str, Any]:
453
+ declarations: dict[str, Any] = {}
454
+ bindings: dict[str, Any] = {}
455
+ for arm_id, worker in self._configuration.workers.items():
456
+ release_id = (
457
+ f"upstream:{self._configuration.method_id}"
458
+ if arm_id == _UPSTREAM_ARM
459
+ else self._configuration.candidate_release_id
460
+ )
461
+ runtime_entrypoint = (
462
+ f"{worker.adapter_checkout_environment}:{worker.entrypoint_relative_path}"
463
+ )
464
+ declarations[arm_id] = {
465
+ "implementation_kind": (
466
+ "upstream_reference" if arm_id == _UPSTREAM_ARM else "agentinputlab_candidate"
467
+ ),
468
+ "implementation_id": (
469
+ f"host-process:{content_hash({'arm_id': arm_id, 'command': worker.command_identity})}"
470
+ ),
471
+ "runtime_entrypoint": runtime_entrypoint,
472
+ "source_revision": worker.expected_checkout_revision,
473
+ "source_tree": worker.expected_checkout_tree,
474
+ "release_id": release_id,
475
+ }
476
+ bindings[arm_id] = {
477
+ "checkout_environment": worker.checkout_environment,
478
+ "checkout": worker.checkout_identity.to_dict(),
479
+ "adapter_checkout_environment": worker.adapter_checkout_environment,
480
+ "adapter_checkout": worker.adapter_checkout_identity.to_dict(),
481
+ "command": worker.command_identity.to_dict(),
482
+ }
483
+ return {
484
+ "schema_version": 1,
485
+ "driver": DRIVER_PROTOCOL,
486
+ "config_sha256": self._configuration.config_sha256,
487
+ "method_id": self._configuration.method_id,
488
+ "candidate_release_id": self._configuration.candidate_release_id,
489
+ "execution_receipt_protocol": HOST_EXECUTION_RECEIPT_PROTOCOL,
490
+ "parity_implementations": declarations,
491
+ "host_process_bindings": bindings,
492
+ "resource_limits": self._configuration.resource_limits.as_dict(),
493
+ "pipe_limits": dict(self._configuration.pipe_limits),
494
+ "supervisor": dict(self._supervisor.fingerprint()),
495
+ "fixture_inventory_sha256": self._environment.get(UNIT_INVENTORY_SHA256_ENV),
496
+ }
497
+
498
+ def _require_preflight(
499
+ self,
500
+ manifest: ExperimentManifest,
501
+ cell: ExperimentCell,
502
+ ) -> None:
503
+ if (
504
+ self._manifest_hash is None
505
+ or self._binding_receipt_hash is None
506
+ or self._contract_hash is None
507
+ ):
508
+ raise ArtifactError("parity-process execute_cell requires successful preflight")
509
+ if manifest.manifest_hash != self._manifest_hash:
510
+ raise ArtifactError("parity-process driver received another manifest")
511
+ if cell.manifest_hash != manifest.manifest_hash:
512
+ raise ValidationError("parity-process cell belongs to another manifest")
513
+ if set(cell.arm_ids) != set(_ARM_IDS):
514
+ raise ValidationError("parity-process cell requires exact implementation arms")
515
+ if cell.unit_id not in self._fixture_unit_ids:
516
+ raise ArtifactError("parity-process cell is outside the preflighted fixture inventory")
517
+
518
+ def _execute_arm(
519
+ self,
520
+ manifest: ExperimentManifest,
521
+ cell: ExperimentCell,
522
+ worker: _WorkerSpec,
523
+ ) -> tuple[Mapping[str, Any], Mapping[str, Any], ResourceReceipt]:
524
+ expected_state = self._preflight_worker_state.get(worker.arm_id)
525
+ request = _build_request(
526
+ configuration=self._configuration,
527
+ manifest=manifest,
528
+ cell=cell,
529
+ worker=worker,
530
+ )
531
+ input_text = canonical_json(request) + "\n"
532
+ environment = self._worker_environment(worker)
533
+ passed_values = environment_secret_values(
534
+ self._environment,
535
+ (
536
+ *_BASE_PASSTHROUGH_ENVIRONMENTS,
537
+ *worker.pass_environment,
538
+ ),
539
+ )
540
+ process = execute_locked_process_worker(
541
+ worker,
542
+ supervisor=self._supervisor,
543
+ input_text=input_text,
544
+ limits=self._configuration.resource_limits,
545
+ pipe_limits=self._configuration.pipe_limits,
546
+ environment=environment,
547
+ process_name=f"parity-process-{worker.arm_id}",
548
+ config_sha256=self._configuration.config_sha256,
549
+ expected_worker_state=expected_state,
550
+ )
551
+ if process.returncode != 0:
552
+ message = redact_secret_text(
553
+ process.stderr.strip() or "worker exited non-zero",
554
+ secret_values=passed_values,
555
+ )
556
+ raise SupervisedWorkerError(
557
+ f"parity-process {worker.arm_id} failed: {message}",
558
+ resource_receipt=process.resource_receipt,
559
+ )
560
+ response = _parse_response(
561
+ process,
562
+ expected_request_hash=request["request_hash"],
563
+ secret_values=passed_values,
564
+ )
565
+ host_receipt = _host_execution_receipt(
566
+ config_sha256=self._configuration.config_sha256,
567
+ manifest=manifest,
568
+ cell=cell,
569
+ worker=worker,
570
+ request=request,
571
+ process=process,
572
+ checkout_binding_hash=self._checkout_binding_hashes[worker.arm_id],
573
+ adapter_checkout_binding_hash=self._adapter_checkout_binding_hashes[worker.arm_id],
574
+ )
575
+ return response, host_receipt, process.resource_receipt
576
+
577
+ def _worker_environment(self, worker: _WorkerSpec) -> Mapping[str, str]:
578
+ names = (
579
+ *_BASE_PASSTHROUGH_ENVIRONMENTS,
580
+ *worker.pass_environment,
581
+ )
582
+ environment = dict(
583
+ locked_process_worker_environment(
584
+ worker,
585
+ environment=self._environment,
586
+ pass_environment=names,
587
+ request_protocol=REQUEST_PROTOCOL,
588
+ )
589
+ )
590
+ environment.update(
591
+ {
592
+ "AILAB_PARITY_ARM_ID": worker.arm_id,
593
+ "AILAB_PARITY_PROTOCOL": REQUEST_PROTOCOL,
594
+ }
595
+ )
596
+ return freeze_mapping(environment)
597
+
598
+
599
+ def factory() -> ParityProcessExperimentDriver:
600
+ """Load the checksum-locked parity-process configuration from the environment."""
601
+
602
+ path = os.environ.get(CONFIG_PATH_ENV)
603
+ expected = os.environ.get(CONFIG_SHA256_ENV)
604
+ if not path:
605
+ raise ArtifactError(f"{CONFIG_PATH_ENV} is required")
606
+ if not expected:
607
+ raise ArtifactError(f"{CONFIG_SHA256_ENV} is required")
608
+ return ParityProcessExperimentDriver.from_config(
609
+ path,
610
+ expected_sha256=expected,
611
+ supervisor=worker_supervisor_from_environment(),
612
+ )
613
+
614
+
615
+ LockedProcessWorker = _WorkerSpec
616
+
617
+
618
+ def load_locked_process_worker(
619
+ arm_id: str,
620
+ value: Any,
621
+ *,
622
+ environment: Mapping[str, str],
623
+ checkout_environment: str,
624
+ adapter_checkout_environment: str | None = None,
625
+ ) -> LockedProcessWorker:
626
+ """Resolve one exact-SHA worker for another host-process experiment."""
627
+
628
+ return _load_worker(
629
+ arm_id,
630
+ value,
631
+ environment=environment,
632
+ expected_checkout_environment=checkout_environment,
633
+ expected_adapter_checkout_environment=(
634
+ checkout_environment
635
+ if adapter_checkout_environment is None
636
+ else adapter_checkout_environment
637
+ ),
638
+ )
639
+
640
+
641
+ def attest_locked_process_worker(
642
+ worker: LockedProcessWorker,
643
+ ) -> Mapping[str, Any]:
644
+ """Independently re-attest a worker checkout and command."""
645
+
646
+ return freeze_mapping(_attest_worker(worker).to_dict())
647
+
648
+
649
+ def locked_process_worker_environment(
650
+ worker: LockedProcessWorker,
651
+ *,
652
+ environment: Mapping[str, str],
653
+ pass_environment: Sequence[str],
654
+ request_protocol: str,
655
+ ) -> Mapping[str, str]:
656
+ """Build a minimal environment with checkout-confined Python imports."""
657
+
658
+ missing = sorted(name for name in pass_environment if not environment.get(name))
659
+ if missing:
660
+ raise ArtifactError(
661
+ f"locked process {worker.arm_id} missing environments: " + ", ".join(missing)
662
+ )
663
+ result = {name: environment[name] for name in pass_environment}
664
+ python_paths = [str(worker.checkout)]
665
+ source_root = worker.checkout / "src"
666
+ if source_root.is_dir():
667
+ python_paths.append(str(source_root))
668
+ result.update(
669
+ {
670
+ "AILAB_PROCESS_ARM_ID": worker.arm_id,
671
+ "AILAB_PROCESS_PROTOCOL": request_protocol,
672
+ "LANG": "C",
673
+ "LC_ALL": "C",
674
+ "PWD": str(worker.checkout),
675
+ "PYTHONPATH": os.pathsep.join(python_paths),
676
+ "PYTHONDONTWRITEBYTECODE": "1",
677
+ "PYTHONIOENCODING": "utf-8",
678
+ "PYTHONNOUSERSITE": "1",
679
+ "TZ": "UTC",
680
+ }
681
+ )
682
+ return freeze_mapping(result)
683
+
684
+
685
+ def execute_locked_process_worker(
686
+ worker: LockedProcessWorker,
687
+ *,
688
+ supervisor: WorkerSupervisor,
689
+ input_text: str,
690
+ limits: ResourceLimits,
691
+ pipe_limits: Mapping[str, int],
692
+ environment: Mapping[str, str],
693
+ process_name: str,
694
+ config_sha256: str,
695
+ expected_worker_state: Mapping[str, Any],
696
+ ) -> SupervisedProcessResult:
697
+ """Run one fresh worker and fail if its identity changes around the call."""
698
+
699
+ before = attest_locked_process_worker(worker)
700
+ if before != freeze_mapping(expected_worker_state):
701
+ raise ArtifactError(
702
+ f"locked process {worker.arm_id} command or checkout drifted after preflight"
703
+ )
704
+ try:
705
+ identity = {
706
+ "kind": "content_attested_checkout_process",
707
+ "arm_id": worker.arm_id,
708
+ "checkout": worker.checkout_identity.to_dict(),
709
+ "adapter_checkout": worker.adapter_checkout_identity.to_dict(),
710
+ "command": worker.command_identity.to_dict(),
711
+ "config_sha256": config_sha256,
712
+ }
713
+ if isinstance(
714
+ supervisor,
715
+ (DockerWorkerSupervisor, LocalSubprocessSupervisor),
716
+ ):
717
+ process = supervisor.execute_locked_worker(
718
+ worker.argv,
719
+ checkout=worker.checkout,
720
+ adapter_checkout=worker.adapter_checkout,
721
+ entrypoint_relative_path=str(worker.entrypoint_relative_path),
722
+ input_text=input_text,
723
+ limits=limits,
724
+ max_input_bytes=pipe_limits["max_input_bytes"],
725
+ max_stdout_bytes=pipe_limits["max_stdout_bytes"],
726
+ max_stderr_bytes=pipe_limits["max_stderr_bytes"],
727
+ environment=environment,
728
+ process_name=process_name,
729
+ image_identity=identity,
730
+ )
731
+ else:
732
+ process = supervisor.execute(
733
+ worker.argv,
734
+ input_text=input_text,
735
+ limits=limits,
736
+ max_input_bytes=pipe_limits["max_input_bytes"],
737
+ max_stdout_bytes=pipe_limits["max_stdout_bytes"],
738
+ max_stderr_bytes=pipe_limits["max_stderr_bytes"],
739
+ cwd=str(worker.checkout),
740
+ environment=environment,
741
+ process_name=process_name,
742
+ image_identity=identity,
743
+ )
744
+ except SupervisedWorkerError as exc:
745
+ try:
746
+ after = attest_locked_process_worker(worker)
747
+ except (ArtifactError, ExternalDependencyError, ValidationError) as drift:
748
+ error = ArtifactError(
749
+ f"locked process {worker.arm_id} identity changed during failed execution"
750
+ )
751
+ error.resource_receipt = exc.resource_receipt # type: ignore[attr-defined]
752
+ raise error from drift
753
+ if after != freeze_mapping(expected_worker_state):
754
+ error = ArtifactError(
755
+ f"locked process {worker.arm_id} identity drifted during failed execution"
756
+ )
757
+ error.resource_receipt = exc.resource_receipt # type: ignore[attr-defined]
758
+ raise error from exc
759
+ raise
760
+ try:
761
+ after = attest_locked_process_worker(worker)
762
+ except (ArtifactError, ExternalDependencyError, ValidationError) as exc:
763
+ error = ArtifactError(f"locked process {worker.arm_id} identity changed during execution")
764
+ error.resource_receipt = process.resource_receipt # type: ignore[attr-defined]
765
+ raise error from exc
766
+ if after != freeze_mapping(expected_worker_state):
767
+ error = ArtifactError(f"locked process {worker.arm_id} identity drifted during execution")
768
+ error.resource_receipt = process.resource_receipt # type: ignore[attr-defined]
769
+ raise error
770
+ return process
771
+
772
+
773
+ def parse_locked_process_response(
774
+ process: SupervisedProcessResult,
775
+ *,
776
+ expected_request_hash: str,
777
+ secret_values: Sequence[str] = (),
778
+ ) -> Mapping[str, Any]:
779
+ """Parse the shared strict response schema and reject identity echoes."""
780
+
781
+ return _parse_response(
782
+ process,
783
+ expected_request_hash=expected_request_hash,
784
+ secret_values=secret_values,
785
+ )
786
+
787
+
788
+ def build_host_execution_receipt(
789
+ *,
790
+ config_sha256: str,
791
+ manifest: ExperimentManifest,
792
+ cell: ExperimentCell,
793
+ worker: LockedProcessWorker,
794
+ request: Mapping[str, Any],
795
+ process: SupervisedProcessResult,
796
+ checkout_binding_hash: str,
797
+ adapter_checkout_binding_hash: str,
798
+ ) -> Mapping[str, Any]:
799
+ """Build the host-owned receipt consumed by the guarded validator."""
800
+
801
+ return _host_execution_receipt(
802
+ config_sha256=config_sha256,
803
+ manifest=manifest,
804
+ cell=cell,
805
+ worker=worker,
806
+ request=request,
807
+ process=process,
808
+ checkout_binding_hash=checkout_binding_hash,
809
+ adapter_checkout_binding_hash=adapter_checkout_binding_hash,
810
+ )
811
+
812
+
813
+ def _load_configuration(
814
+ path: Path,
815
+ *,
816
+ expected_sha256: str,
817
+ environment: Mapping[str, str],
818
+ ) -> _DriverConfiguration:
819
+ require_sha256(expected_sha256, label=CONFIG_SHA256_ENV)
820
+ unresolved_source = path.expanduser()
821
+ if unresolved_source.is_symlink():
822
+ raise ArtifactError("parity-process config must be a regular non-symlink file")
823
+ source = unresolved_source.resolve()
824
+ if not source.is_file():
825
+ raise ArtifactError("parity-process config must be a regular non-symlink file")
826
+ actual_sha256 = _sha256_file(source)
827
+ if actual_sha256 != expected_sha256:
828
+ raise ArtifactError("parity-process config checksum differs from its environment lock")
829
+ try:
830
+ raw = json.loads(source.read_text(encoding="utf-8"))
831
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
832
+ raise ArtifactError("parity-process config is not valid UTF-8 JSON") from exc
833
+ data = _strict_mapping(raw, _CONFIG_FIELDS, label="parity-process config")
834
+ require_secret_free(data, label="parity-process config")
835
+ if data["schema_version"] != 1 or data["protocol"] != DRIVER_PROTOCOL:
836
+ raise ValidationError("unsupported parity-process config schema or protocol")
837
+ method_id = _nonempty_text(data["method_id"], "method_id")
838
+ candidate_release_id = _nonempty_text(
839
+ data["candidate_release_id"],
840
+ "candidate_release_id",
841
+ )
842
+ raw_workers = _strict_mapping(
843
+ data["workers"],
844
+ frozenset(_ARM_IDS),
845
+ label="parity-process workers",
846
+ )
847
+ workers = {
848
+ arm_id: _load_worker(
849
+ arm_id,
850
+ raw_workers[arm_id],
851
+ environment=environment,
852
+ )
853
+ for arm_id in _ARM_IDS
854
+ }
855
+ if workers[_UPSTREAM_ARM].checkout == workers[_CANDIDATE_ARM].checkout:
856
+ raise ValidationError("parity-process arms cannot use the same checkout")
857
+ if workers[_UPSTREAM_ARM].entrypoint == workers[_CANDIDATE_ARM].entrypoint:
858
+ raise ValidationError("parity-process arms cannot use the same entrypoint")
859
+ if workers[_UPSTREAM_ARM].argv == workers[_CANDIDATE_ARM].argv:
860
+ raise ValidationError("parity-process arms cannot use the same command")
861
+ limits_data = _strict_mapping(
862
+ data["resource_limits"],
863
+ frozenset(
864
+ {
865
+ "wall_timeout_seconds",
866
+ "memory_bytes",
867
+ "cpus",
868
+ "pids_limit",
869
+ }
870
+ ),
871
+ label="parity-process resource_limits",
872
+ )
873
+ resource_limits = ResourceLimits(**limits_data)
874
+ pipe_data = _strict_mapping(
875
+ data["pipe_limits"],
876
+ _PIPE_FIELDS,
877
+ label="parity-process pipe_limits",
878
+ )
879
+ pipe_limits = {
880
+ name: _positive_integer(pipe_data[name], f"pipe_limits.{name}")
881
+ for name in sorted(_PIPE_FIELDS)
882
+ }
883
+ shared_parameters = _canonical_mapping(
884
+ data["shared_request_parameters"],
885
+ label="shared_request_parameters",
886
+ )
887
+ require_secret_free(shared_parameters, label="parity-process shared parameters")
888
+ return _DriverConfiguration(
889
+ method_id=method_id,
890
+ candidate_release_id=candidate_release_id,
891
+ workers=workers,
892
+ resource_limits=resource_limits,
893
+ pipe_limits=freeze_mapping(pipe_limits),
894
+ shared_request_parameters=shared_parameters,
895
+ config_sha256=actual_sha256,
896
+ )
897
+
898
+
899
+ def _load_worker(
900
+ arm_id: str,
901
+ raw: Any,
902
+ *,
903
+ environment: Mapping[str, str],
904
+ expected_checkout_environment: str | None = None,
905
+ expected_adapter_checkout_environment: str | None = None,
906
+ ) -> _WorkerSpec:
907
+ data = _strict_mapping(raw, _WORKER_FIELDS, label=f"worker {arm_id}")
908
+ checkout_environment = _nonempty_text(
909
+ data["checkout_environment"],
910
+ f"{arm_id}.checkout_environment",
911
+ )
912
+ required_checkout_environment = (
913
+ _CHECKOUT_ENVIRONMENTS[arm_id]
914
+ if expected_checkout_environment is None
915
+ else expected_checkout_environment
916
+ )
917
+ if checkout_environment != required_checkout_environment:
918
+ raise ValidationError(f"{arm_id} must use {required_checkout_environment}")
919
+ checkout_value = environment.get(checkout_environment)
920
+ if not checkout_value:
921
+ raise ArtifactError(f"missing checkout environment {checkout_environment}")
922
+ checkout = Path(checkout_value).expanduser().resolve()
923
+ expected_revision = _git_object_id(
924
+ data["expected_checkout_revision"],
925
+ f"{arm_id}.expected_checkout_revision",
926
+ )
927
+ expected_tree = _git_object_id(
928
+ data["expected_checkout_tree"],
929
+ f"{arm_id}.expected_checkout_tree",
930
+ )
931
+ checkout_identity = _attest_checkout(
932
+ checkout,
933
+ expected_revision=expected_revision,
934
+ expected_tree=expected_tree,
935
+ )
936
+ adapter_checkout_environment = _nonempty_text(
937
+ data["adapter_checkout_environment"],
938
+ f"{arm_id}.adapter_checkout_environment",
939
+ )
940
+ required_adapter_environment = (
941
+ "AILAB_CANDIDATE_CHECKOUT"
942
+ if expected_adapter_checkout_environment is None
943
+ else expected_adapter_checkout_environment
944
+ )
945
+ if adapter_checkout_environment != required_adapter_environment:
946
+ raise ValidationError(f"{arm_id} adapter must use {required_adapter_environment}")
947
+ adapter_checkout_value = environment.get(adapter_checkout_environment)
948
+ if not adapter_checkout_value:
949
+ raise ArtifactError(f"missing adapter checkout environment {adapter_checkout_environment}")
950
+ adapter_checkout = Path(adapter_checkout_value).expanduser().resolve()
951
+ expected_adapter_revision = _git_object_id(
952
+ data["expected_adapter_checkout_revision"],
953
+ f"{arm_id}.expected_adapter_checkout_revision",
954
+ )
955
+ expected_adapter_tree = _git_object_id(
956
+ data["expected_adapter_checkout_tree"],
957
+ f"{arm_id}.expected_adapter_checkout_tree",
958
+ )
959
+ adapter_checkout_identity = _attest_checkout(
960
+ adapter_checkout,
961
+ expected_revision=expected_adapter_revision,
962
+ expected_tree=expected_adapter_tree,
963
+ )
964
+ executable = (
965
+ Path(_nonempty_text(data["executable_path"], f"{arm_id}.executable_path"))
966
+ .expanduser()
967
+ .resolve()
968
+ )
969
+ if not executable.is_file():
970
+ raise ArtifactError(f"{arm_id} executable does not exist")
971
+ executable_sha256 = _sha256_text_field(
972
+ data["executable_sha256"],
973
+ f"{arm_id}.executable_sha256",
974
+ )
975
+ if _sha256_file(executable) != executable_sha256:
976
+ raise ArtifactError(f"{arm_id} executable checksum differs from config")
977
+ entrypoint_relative_path = _relative_entrypoint(
978
+ data["entrypoint"],
979
+ label=f"{arm_id}.entrypoint",
980
+ )
981
+ unresolved_entrypoint = adapter_checkout / entrypoint_relative_path
982
+ if unresolved_entrypoint.is_symlink():
983
+ raise ArtifactError(f"{arm_id} entrypoint must not be a symlink")
984
+ entrypoint = unresolved_entrypoint.resolve()
985
+ if not entrypoint.is_relative_to(adapter_checkout):
986
+ raise ValidationError(f"{arm_id} entrypoint escapes its adapter checkout")
987
+ if not entrypoint.is_file() or not stat.S_ISREG(entrypoint.stat().st_mode):
988
+ raise ArtifactError(f"{arm_id} entrypoint must be a regular non-symlink file")
989
+ _require_tracked_file(adapter_checkout, entrypoint_relative_path)
990
+ entrypoint_sha256 = _sha256_text_field(
991
+ data["entrypoint_sha256"],
992
+ f"{arm_id}.entrypoint_sha256",
993
+ )
994
+ if _sha256_file(entrypoint) != entrypoint_sha256:
995
+ raise ArtifactError(f"{arm_id} entrypoint checksum differs from config")
996
+ arguments = _string_sequence(data["arguments"], label=f"{arm_id}.arguments")
997
+ pass_environment = _environment_names(
998
+ data["pass_environment"],
999
+ label=f"{arm_id}.pass_environment",
1000
+ )
1001
+ reserved = sorted(set(pass_environment) & _RESERVED_WORKER_ENVIRONMENTS)
1002
+ if reserved:
1003
+ raise ValidationError(
1004
+ f"{arm_id} pass_environment contains host-controlled names: " + ", ".join(reserved)
1005
+ )
1006
+ request_parameters = _canonical_mapping(
1007
+ data["request_parameters"],
1008
+ label=f"{arm_id}.request_parameters",
1009
+ )
1010
+ require_secret_free(
1011
+ request_parameters,
1012
+ label=f"parity-process {arm_id} request parameters",
1013
+ )
1014
+ python_paths = [checkout]
1015
+ if (checkout / "src").is_dir():
1016
+ python_paths.append(checkout / "src")
1017
+ command_identity = _CommandIdentity(
1018
+ executable_resolved_path_sha256=_sha256_text(str(executable)),
1019
+ executable_content_sha256=executable_sha256,
1020
+ entrypoint_relative_path=entrypoint_relative_path,
1021
+ entrypoint_resolved_path_sha256=_sha256_text(str(entrypoint)),
1022
+ entrypoint_content_sha256=entrypoint_sha256,
1023
+ argv_sha256=content_hash(
1024
+ {
1025
+ "argv": (
1026
+ str(executable),
1027
+ str(entrypoint),
1028
+ *arguments,
1029
+ )
1030
+ }
1031
+ ),
1032
+ cwd_resolved_path_sha256=checkout_identity.resolved_path_sha256,
1033
+ pythonpath_resolved_path_sha256=tuple(
1034
+ _sha256_text(str(item.resolve())) for item in python_paths
1035
+ ),
1036
+ )
1037
+ return _WorkerSpec(
1038
+ arm_id=arm_id,
1039
+ checkout_environment=checkout_environment,
1040
+ checkout=checkout,
1041
+ expected_checkout_revision=expected_revision,
1042
+ expected_checkout_tree=expected_tree,
1043
+ adapter_checkout_environment=adapter_checkout_environment,
1044
+ adapter_checkout=adapter_checkout,
1045
+ expected_adapter_checkout_revision=expected_adapter_revision,
1046
+ expected_adapter_checkout_tree=expected_adapter_tree,
1047
+ executable=executable,
1048
+ executable_sha256=executable_sha256,
1049
+ entrypoint_relative_path=entrypoint_relative_path,
1050
+ entrypoint=entrypoint,
1051
+ entrypoint_sha256=entrypoint_sha256,
1052
+ arguments=arguments,
1053
+ pass_environment=pass_environment,
1054
+ request_parameters=request_parameters,
1055
+ checkout_identity=checkout_identity,
1056
+ adapter_checkout_identity=adapter_checkout_identity,
1057
+ command_identity=command_identity,
1058
+ )
1059
+
1060
+
1061
+ def _attest_worker(worker: _WorkerSpec) -> _CommandIdentity:
1062
+ checkout = _attest_checkout(
1063
+ worker.checkout,
1064
+ expected_revision=worker.expected_checkout_revision,
1065
+ expected_tree=worker.expected_checkout_tree,
1066
+ )
1067
+ adapter_checkout = _attest_checkout(
1068
+ worker.adapter_checkout,
1069
+ expected_revision=worker.expected_adapter_checkout_revision,
1070
+ expected_tree=worker.expected_adapter_checkout_tree,
1071
+ )
1072
+ if _sha256_file(worker.executable) != worker.executable_sha256:
1073
+ raise ArtifactError(f"{worker.arm_id} executable changed")
1074
+ unresolved_entrypoint = worker.adapter_checkout / worker.entrypoint_relative_path
1075
+ if unresolved_entrypoint.is_symlink():
1076
+ raise ArtifactError(f"{worker.arm_id} entrypoint path changed")
1077
+ entrypoint = unresolved_entrypoint.resolve()
1078
+ if (
1079
+ entrypoint != worker.entrypoint
1080
+ or not entrypoint.is_relative_to(worker.adapter_checkout)
1081
+ or not entrypoint.is_file()
1082
+ ):
1083
+ raise ArtifactError(f"{worker.arm_id} entrypoint path changed")
1084
+ _require_tracked_file(worker.adapter_checkout, worker.entrypoint_relative_path)
1085
+ if _sha256_file(entrypoint) != worker.entrypoint_sha256:
1086
+ raise ArtifactError(f"{worker.arm_id} entrypoint changed")
1087
+ if checkout.to_dict() != worker.checkout_identity.to_dict():
1088
+ raise ArtifactError(f"{worker.arm_id} checkout identity changed")
1089
+ if adapter_checkout.to_dict() != worker.adapter_checkout_identity.to_dict():
1090
+ raise ArtifactError(f"{worker.arm_id} adapter checkout identity changed")
1091
+ return worker.command_identity
1092
+
1093
+
1094
+ def _attest_checkout(
1095
+ checkout: Path,
1096
+ *,
1097
+ expected_revision: str,
1098
+ expected_tree: str,
1099
+ ) -> _CheckoutIdentity:
1100
+ if not checkout.is_dir():
1101
+ raise ArtifactError("parity-process checkout is unavailable")
1102
+ top_level = Path(_git(checkout, "rev-parse", "--show-toplevel")).resolve()
1103
+ if top_level != checkout:
1104
+ raise ArtifactError("parity-process checkout must identify its Git root")
1105
+ head = _git(checkout, "rev-parse", "HEAD")
1106
+ tree = _git(checkout, "rev-parse", "HEAD^{tree}")
1107
+ if head != expected_revision:
1108
+ raise ArtifactError("parity-process checkout HEAD differs from config")
1109
+ if tree != expected_tree:
1110
+ raise ArtifactError("parity-process checkout tree differs from config")
1111
+ status = _git(checkout, "status", "--porcelain=v1", "--untracked-files=all")
1112
+ if status:
1113
+ raise ArtifactError("parity-process checkout must have a clean working tree")
1114
+ return _CheckoutIdentity(
1115
+ resolved_path_sha256=_sha256_text(str(checkout)),
1116
+ git_head=head,
1117
+ git_tree=tree,
1118
+ )
1119
+
1120
+
1121
+ def _require_tracked_file(checkout: Path, relative_path: str) -> None:
1122
+ try:
1123
+ _git(checkout, "ls-files", "--error-unmatch", "--", relative_path)
1124
+ except ArtifactError as exc:
1125
+ raise ArtifactError("parity-process entrypoint must be tracked by Git") from exc
1126
+
1127
+
1128
+ def _build_request(
1129
+ *,
1130
+ configuration: _DriverConfiguration,
1131
+ manifest: ExperimentManifest,
1132
+ cell: ExperimentCell,
1133
+ worker: _WorkerSpec,
1134
+ ) -> Mapping[str, Any]:
1135
+ arm = next(item for item in manifest.arms if item.arm_id == worker.arm_id)
1136
+ payload = {
1137
+ "schema_version": 1,
1138
+ "protocol": REQUEST_PROTOCOL,
1139
+ "manifest_hash": manifest.manifest_hash,
1140
+ "cell": {
1141
+ "cell_id": cell.cell_id,
1142
+ "study_id": cell.study_id,
1143
+ "benchmark_id": cell.benchmark_id,
1144
+ "split": cell.split,
1145
+ "unit_id": cell.unit_id,
1146
+ "model_slot": cell.model_slot,
1147
+ "seed": cell.seed,
1148
+ },
1149
+ "method": {
1150
+ "method_id": configuration.method_id,
1151
+ "candidate_release_id": configuration.candidate_release_id,
1152
+ "profile_kind": arm.profile_kind.value,
1153
+ "family": arm.family.value,
1154
+ },
1155
+ "arm": {
1156
+ "arm_id": worker.arm_id,
1157
+ "release_id": (
1158
+ f"upstream:{configuration.method_id}"
1159
+ if worker.arm_id == _UPSTREAM_ARM
1160
+ else configuration.candidate_release_id
1161
+ ),
1162
+ },
1163
+ "parameters": {
1164
+ "shared": configuration.shared_request_parameters,
1165
+ "worker": worker.request_parameters,
1166
+ },
1167
+ "asset_environments": list(_BASE_PASSTHROUGH_ENVIRONMENTS),
1168
+ }
1169
+ require_secret_free(payload, label="parity-process worker request")
1170
+ request_hash = content_hash(payload)
1171
+ return freeze_mapping({**payload, "request_hash": request_hash})
1172
+
1173
+
1174
+ def _parse_response(
1175
+ process: SupervisedProcessResult,
1176
+ *,
1177
+ expected_request_hash: str,
1178
+ secret_values: Sequence[str] = (),
1179
+ ) -> Mapping[str, Any]:
1180
+ try:
1181
+ raw = json.loads(process.stdout)
1182
+ except json.JSONDecodeError as exc:
1183
+ raise ParityProcessProtocolError(
1184
+ "parity-process worker stdout is not one JSON value",
1185
+ resource_receipt=process.resource_receipt,
1186
+ ) from exc
1187
+ try:
1188
+ require_secret_values_absent(
1189
+ raw,
1190
+ secret_values=secret_values,
1191
+ label="parity-process worker response",
1192
+ )
1193
+ _reject_forbidden_worker_keys(raw)
1194
+ data = _strict_mapping(
1195
+ raw,
1196
+ _RESPONSE_FIELDS,
1197
+ label="parity-process response",
1198
+ )
1199
+ if data["schema_version"] != 1 or data["protocol"] != RESPONSE_PROTOCOL:
1200
+ raise ValidationError("unsupported parity-process worker response")
1201
+ if data["request_hash"] != expected_request_hash:
1202
+ raise ValidationError("parity-process response belongs to another request")
1203
+ status = data["status"]
1204
+ if status not in {"succeeded", "failed"}:
1205
+ raise ValidationError("parity-process response status is invalid")
1206
+ metrics = _metrics(data["metrics"])
1207
+ usage_data = _strict_mapping(
1208
+ data["usage"],
1209
+ _USAGE_FIELDS,
1210
+ label="parity-process usage",
1211
+ )
1212
+ usage = ResourceUsage(**usage_data).as_dict()
1213
+ usage.pop("total_cost_usd")
1214
+ output = _canonical_mapping(data["output"], label="parity-process output")
1215
+ failure = data["failure"]
1216
+ if status == "succeeded":
1217
+ if not metrics:
1218
+ raise ValidationError("successful parity-process response requires metrics")
1219
+ if failure is not None:
1220
+ raise ValidationError("successful parity-process response cannot have failure")
1221
+ else:
1222
+ if metrics or output:
1223
+ raise ValidationError("failed parity-process response cannot have metrics/output")
1224
+ failure = _strict_mapping(
1225
+ failure,
1226
+ frozenset({"scope", "category", "message"}),
1227
+ label="parity-process failure",
1228
+ )
1229
+ if failure["scope"] != "algorithm":
1230
+ raise ValidationError("parity-process arm failure must be algorithm-scoped")
1231
+ _nonempty_text(failure["category"], "failure.category")
1232
+ _nonempty_text(failure["message"], "failure.message")
1233
+ normalized = {
1234
+ "schema_version": 1,
1235
+ "protocol": RESPONSE_PROTOCOL,
1236
+ "request_hash": expected_request_hash,
1237
+ "status": status,
1238
+ "metrics": metrics,
1239
+ "usage": usage,
1240
+ "output": output,
1241
+ "failure": failure,
1242
+ }
1243
+ require_secret_free(normalized, label="parity-process worker response")
1244
+ return freeze_mapping(normalized)
1245
+ except (ArtifactError, TypeError, ValueError, ValidationError) as exc:
1246
+ raise ParityProcessProtocolError(
1247
+ redact_secret_text(str(exc), secret_values=secret_values),
1248
+ resource_receipt=process.resource_receipt,
1249
+ ) from exc
1250
+
1251
+
1252
+ def _host_execution_receipt(
1253
+ *,
1254
+ config_sha256: str,
1255
+ manifest: ExperimentManifest,
1256
+ cell: ExperimentCell,
1257
+ worker: _WorkerSpec,
1258
+ request: Mapping[str, Any],
1259
+ process: SupervisedProcessResult,
1260
+ checkout_binding_hash: str,
1261
+ adapter_checkout_binding_hash: str,
1262
+ ) -> Mapping[str, Any]:
1263
+ receipt_without_hash = {
1264
+ "schema_version": 1,
1265
+ "protocol": HOST_EXECUTION_RECEIPT_PROTOCOL,
1266
+ "arm_id": worker.arm_id,
1267
+ "manifest_hash": manifest.manifest_hash,
1268
+ "cell_id": cell.cell_id,
1269
+ "request_hash": request["request_hash"],
1270
+ "config_sha256": config_sha256,
1271
+ "checkout_environment": worker.checkout_environment,
1272
+ "checkout_binding_hash": checkout_binding_hash,
1273
+ "adapter_checkout_environment": worker.adapter_checkout_environment,
1274
+ "adapter_checkout_binding_hash": adapter_checkout_binding_hash,
1275
+ "checkout": {
1276
+ **worker.checkout_identity.to_dict(),
1277
+ "git_clean_before": True,
1278
+ "git_clean_after": True,
1279
+ },
1280
+ "adapter_checkout": {
1281
+ **worker.adapter_checkout_identity.to_dict(),
1282
+ "git_clean_before": True,
1283
+ "git_clean_after": True,
1284
+ },
1285
+ "command": worker.command_identity.to_dict(),
1286
+ "process": {
1287
+ "resource_receipt_sha256": content_hash(process.resource_receipt.as_dict()),
1288
+ "stdout_sha256": _sha256_text(process.stdout),
1289
+ "stderr_sha256": _sha256_text(process.stderr),
1290
+ },
1291
+ }
1292
+ return freeze_mapping(
1293
+ {
1294
+ **receipt_without_hash,
1295
+ "receipt_hash": content_hash(receipt_without_hash),
1296
+ }
1297
+ )
1298
+
1299
+
1300
+ def _strict_mapping(
1301
+ value: Any,
1302
+ fields: frozenset[str],
1303
+ *,
1304
+ label: str,
1305
+ ) -> dict[str, Any]:
1306
+ if not isinstance(value, Mapping) or set(value) != set(fields):
1307
+ raise ValidationError(f"{label} fields are invalid")
1308
+ return dict(value)
1309
+
1310
+
1311
+ def _load_fixture_inventory(
1312
+ environment: Mapping[str, str],
1313
+ ) -> tuple[str, ...]:
1314
+ raw_path = environment.get(UNIT_INVENTORY_ENV)
1315
+ expected_sha256 = environment.get(UNIT_INVENTORY_SHA256_ENV)
1316
+ if not raw_path:
1317
+ raise ArtifactError(f"{UNIT_INVENTORY_ENV} is required")
1318
+ if not expected_sha256:
1319
+ raise ArtifactError(f"{UNIT_INVENTORY_SHA256_ENV} is required")
1320
+ require_sha256(expected_sha256, label=UNIT_INVENTORY_SHA256_ENV)
1321
+ path = Path(raw_path).expanduser().resolve()
1322
+ if not path.is_file():
1323
+ raise ArtifactError("parity-process fixture inventory is unavailable")
1324
+ if _sha256_file(path) != expected_sha256:
1325
+ raise ArtifactError("parity-process fixture inventory checksum differs")
1326
+ try:
1327
+ raw = json.loads(path.read_text(encoding="utf-8"))
1328
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
1329
+ raise ArtifactError("parity-process fixture inventory is not valid UTF-8 JSON") from exc
1330
+ if (
1331
+ isinstance(raw, (str, bytes))
1332
+ or not isinstance(raw, Sequence)
1333
+ or not raw
1334
+ or any(not isinstance(item, str) or not item.strip() for item in raw)
1335
+ or len(raw) != len(set(raw))
1336
+ ):
1337
+ raise ValidationError(
1338
+ "parity-process fixture inventory must be a non-empty unique string array"
1339
+ )
1340
+ return tuple(raw)
1341
+
1342
+
1343
+ def _canonical_mapping(value: Any, *, label: str) -> Mapping[str, Any]:
1344
+ if not isinstance(value, Mapping):
1345
+ raise ValidationError(f"{label} must be an object")
1346
+ try:
1347
+ return freeze_mapping(value)
1348
+ except (TypeError, ValueError) as exc:
1349
+ raise ValidationError(f"{label} must be canonical JSON") from exc
1350
+
1351
+
1352
+ def _metrics(value: Any) -> Mapping[str, float]:
1353
+ if not isinstance(value, Mapping):
1354
+ raise ValidationError("parity-process metrics must be an object")
1355
+ result: dict[str, float] = {}
1356
+ for name, raw in value.items():
1357
+ if (
1358
+ not isinstance(name, str)
1359
+ or not name
1360
+ or isinstance(raw, bool)
1361
+ or not isinstance(raw, (int, float))
1362
+ ):
1363
+ raise ValidationError("parity-process metrics must be named numerics")
1364
+ numeric = float(raw)
1365
+ if not -float("inf") < numeric < float("inf"):
1366
+ raise ValidationError("parity-process metrics must be finite")
1367
+ result[name] = numeric
1368
+ return freeze_mapping(result)
1369
+
1370
+
1371
+ def _reject_forbidden_worker_keys(value: Any) -> None:
1372
+ if isinstance(value, Mapping):
1373
+ forbidden = sorted(set(value) & _RESPONSE_FORBIDDEN_KEYS)
1374
+ if forbidden:
1375
+ raise ValidationError(
1376
+ "worker may not emit host-owned identity fields: " + ", ".join(forbidden)
1377
+ )
1378
+ for item in value.values():
1379
+ _reject_forbidden_worker_keys(item)
1380
+ elif isinstance(value, Sequence) and not isinstance(
1381
+ value,
1382
+ (str, bytes, bytearray),
1383
+ ):
1384
+ for item in value:
1385
+ _reject_forbidden_worker_keys(item)
1386
+
1387
+
1388
+ def _relative_entrypoint(value: Any, *, label: str) -> str:
1389
+ text = _nonempty_text(value, label)
1390
+ path = PurePosixPath(text)
1391
+ if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
1392
+ raise ValidationError(f"{label} must be a normalized relative POSIX path")
1393
+ return path.as_posix()
1394
+
1395
+
1396
+ def _environment_names(value: Any, *, label: str) -> tuple[str, ...]:
1397
+ names = _string_sequence(value, label=label)
1398
+ if len(names) != len(set(names)):
1399
+ raise ValidationError(f"{label} cannot contain duplicates")
1400
+ if any(_ENVIRONMENT_NAME.fullmatch(name) is None for name in names):
1401
+ raise ValidationError(f"{label} contains an invalid environment name")
1402
+ return names
1403
+
1404
+
1405
+ def _string_sequence(value: Any, *, label: str) -> tuple[str, ...]:
1406
+ if (
1407
+ isinstance(value, (str, bytes))
1408
+ or not isinstance(value, Sequence)
1409
+ or any(not isinstance(item, str) or "\x00" in item for item in value)
1410
+ ):
1411
+ raise ValidationError(f"{label} must be a string array")
1412
+ return tuple(value)
1413
+
1414
+
1415
+ def _nonempty_text(value: Any, label: str) -> str:
1416
+ if not isinstance(value, str) or not value.strip() or "\x00" in value:
1417
+ raise ValidationError(f"{label} must be a non-empty string")
1418
+ return value
1419
+
1420
+
1421
+ def _positive_integer(value: Any, label: str) -> int:
1422
+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:
1423
+ raise ValidationError(f"{label} must be a positive integer")
1424
+ return value
1425
+
1426
+
1427
+ def _sha256_text_field(value: Any, label: str) -> str:
1428
+ if not isinstance(value, str):
1429
+ raise ValidationError(f"{label} must be a SHA-256 digest")
1430
+ require_sha256(value, label=label)
1431
+ return value
1432
+
1433
+
1434
+ def _git_object_id(value: Any, label: str) -> str:
1435
+ if not isinstance(value, str) or _GIT_OBJECT_ID.fullmatch(value) is None:
1436
+ raise ValidationError(f"{label} must be a full lowercase Git object ID")
1437
+ return value
1438
+
1439
+
1440
+ def _git(path: Path, *arguments: str) -> str:
1441
+ try:
1442
+ completed = subprocess.run(
1443
+ ("git", "-C", str(path), *arguments),
1444
+ check=True,
1445
+ capture_output=True,
1446
+ text=True,
1447
+ timeout=30,
1448
+ )
1449
+ except (OSError, subprocess.SubprocessError) as exc:
1450
+ raise ArtifactError("cannot independently attest parity-process Git checkout") from exc
1451
+ return completed.stdout.strip()
1452
+
1453
+
1454
+ def _sha256_file(path: Path) -> str:
1455
+ digest = hashlib.sha256()
1456
+ with path.open("rb") as handle:
1457
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
1458
+ digest.update(block)
1459
+ return digest.hexdigest()
1460
+
1461
+
1462
+ def _sha256_text(value: str) -> str:
1463
+ return hashlib.sha256(value.encode("utf-8")).hexdigest()
1464
+
1465
+
1466
+ __all__ = [
1467
+ "CONFIG_PATH_ENV",
1468
+ "CONFIG_SHA256_ENV",
1469
+ "DRIVER_PROTOCOL",
1470
+ "REQUEST_PROTOCOL",
1471
+ "RESPONSE_PROTOCOL",
1472
+ "UNIT_INVENTORY_ENV",
1473
+ "UNIT_INVENTORY_SHA256_ENV",
1474
+ "LockedProcessWorker",
1475
+ "ParityProcessExperimentDriver",
1476
+ "ParityProcessProtocolError",
1477
+ "attest_locked_process_worker",
1478
+ "build_host_execution_receipt",
1479
+ "execute_locked_process_worker",
1480
+ "factory",
1481
+ "load_locked_process_worker",
1482
+ "locked_process_worker_environment",
1483
+ "parse_locked_process_response",
1484
+ ]