agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,1484 @@
|
|
|
1
|
+
"""Exact-SHA, two-process driver for upstream-versus-candidate parity checks.
|
|
2
|
+
|
|
3
|
+
The host, rather than either worker, establishes implementation identity. Each
|
|
4
|
+
arm is started in a fresh bounded subprocess with a checkout-specific working
|
|
5
|
+
directory and ``PYTHONPATH``. The host independently verifies the checkout,
|
|
6
|
+
executable, entrypoint, and command before and after every invocation and emits
|
|
7
|
+
an execution receipt that the guarded experiment driver validates.
|
|
8
|
+
|
|
9
|
+
This driver provides execution evidence only. A successful run is not, by
|
|
10
|
+
itself, a paper-result or official-parity claim.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import stat
|
|
20
|
+
import subprocess
|
|
21
|
+
from collections.abc import Mapping, Sequence
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import Path, PurePosixPath
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from agentinputlab.contracts.errors import (
|
|
27
|
+
AlgorithmOutputError,
|
|
28
|
+
ArtifactError,
|
|
29
|
+
ExternalDependencyError,
|
|
30
|
+
ValidationError,
|
|
31
|
+
)
|
|
32
|
+
from agentinputlab.contracts.serialization import (
|
|
33
|
+
canonical_json,
|
|
34
|
+
content_hash,
|
|
35
|
+
freeze_mapping,
|
|
36
|
+
to_jsonable,
|
|
37
|
+
)
|
|
38
|
+
from agentinputlab.experiments.bindings import resolve_frozen_binding_receipt
|
|
39
|
+
from agentinputlab.experiments.executor import ExperimentCellResult
|
|
40
|
+
from agentinputlab.experiments.parity_identity import (
|
|
41
|
+
HOST_EXECUTION_RECEIPT_PROTOCOL,
|
|
42
|
+
build_parity_implementation_contract,
|
|
43
|
+
)
|
|
44
|
+
from agentinputlab.experiments.plan import ExperimentCell
|
|
45
|
+
from agentinputlab.experiments.schema import ExperimentManifest, ExperimentStage, require_sha256
|
|
46
|
+
from agentinputlab.experiments.secrets import (
|
|
47
|
+
environment_secret_values,
|
|
48
|
+
redact_secret_text,
|
|
49
|
+
require_secret_free,
|
|
50
|
+
require_secret_values_absent,
|
|
51
|
+
)
|
|
52
|
+
from agentinputlab.integrations.external_workers.supervisor import (
|
|
53
|
+
DockerWorkerSupervisor,
|
|
54
|
+
LocalSubprocessSupervisor,
|
|
55
|
+
SupervisedProcessResult,
|
|
56
|
+
SupervisedWorkerError,
|
|
57
|
+
WorkerSupervisor,
|
|
58
|
+
worker_supervisor_from_environment,
|
|
59
|
+
)
|
|
60
|
+
from agentinputlab.observability import ResourceLimits, ResourceReceipt, ResourceUsage
|
|
61
|
+
|
|
62
|
+
CONFIG_PATH_ENV = "AILAB_PARITY_PROCESS_CONFIG"
|
|
63
|
+
CONFIG_SHA256_ENV = "AILAB_PARITY_PROCESS_CONFIG_SHA256"
|
|
64
|
+
UNIT_INVENTORY_ENV = "AILAB_PARITY_UNIT_INVENTORY"
|
|
65
|
+
UNIT_INVENTORY_SHA256_ENV = "AILAB_PARITY_UNIT_INVENTORY_SHA256"
|
|
66
|
+
REQUEST_PROTOCOL = "agentinputlab.parity-process-request.v1"
|
|
67
|
+
RESPONSE_PROTOCOL = "agentinputlab.parity-process-response.v1"
|
|
68
|
+
DRIVER_PROTOCOL = "agentinputlab.parity-process-driver.v1"
|
|
69
|
+
|
|
70
|
+
_UPSTREAM_ARM = "upstream_reference"
|
|
71
|
+
_CANDIDATE_ARM = "agentinputlab_candidate"
|
|
72
|
+
_ARM_IDS = (_UPSTREAM_ARM, _CANDIDATE_ARM)
|
|
73
|
+
_CHECKOUT_ENVIRONMENTS = {
|
|
74
|
+
_UPSTREAM_ARM: "AILAB_UPSTREAM_CHECKOUT",
|
|
75
|
+
_CANDIDATE_ARM: "AILAB_CANDIDATE_CHECKOUT",
|
|
76
|
+
}
|
|
77
|
+
_BASE_PASSTHROUGH_ENVIRONMENTS = (
|
|
78
|
+
"AILAB_PARITY_ASSET_ROOT",
|
|
79
|
+
"AILAB_MODEL_CONFIG",
|
|
80
|
+
"AILAB_RETRIEVER_CONFIG",
|
|
81
|
+
)
|
|
82
|
+
_RESERVED_WORKER_ENVIRONMENTS = frozenset(
|
|
83
|
+
{
|
|
84
|
+
"AILAB_PARITY_ARM_ID",
|
|
85
|
+
"AILAB_PARITY_PROTOCOL",
|
|
86
|
+
"AILAB_PROCESS_ARM_ID",
|
|
87
|
+
"AILAB_PROCESS_PROTOCOL",
|
|
88
|
+
"LANG",
|
|
89
|
+
"LC_ALL",
|
|
90
|
+
"PWD",
|
|
91
|
+
"PYTHONPATH",
|
|
92
|
+
"PYTHONDONTWRITEBYTECODE",
|
|
93
|
+
"PYTHONIOENCODING",
|
|
94
|
+
"PYTHONNOUSERSITE",
|
|
95
|
+
"TZ",
|
|
96
|
+
}
|
|
97
|
+
)
|
|
98
|
+
_ENVIRONMENT_NAME = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
|
99
|
+
_GIT_OBJECT_ID = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
|
|
100
|
+
_USAGE_FIELDS = frozenset(
|
|
101
|
+
{
|
|
102
|
+
"model_calls",
|
|
103
|
+
"retrieval_calls",
|
|
104
|
+
"tool_calls",
|
|
105
|
+
"embedding_calls",
|
|
106
|
+
"unpriced_model_calls",
|
|
107
|
+
"input_tokens",
|
|
108
|
+
"output_tokens",
|
|
109
|
+
"cpu_seconds",
|
|
110
|
+
"accelerator_seconds",
|
|
111
|
+
"cost_usd",
|
|
112
|
+
"compute_cost_usd",
|
|
113
|
+
}
|
|
114
|
+
)
|
|
115
|
+
_PIPE_FIELDS = frozenset(
|
|
116
|
+
{
|
|
117
|
+
"max_input_bytes",
|
|
118
|
+
"max_stdout_bytes",
|
|
119
|
+
"max_stderr_bytes",
|
|
120
|
+
}
|
|
121
|
+
)
|
|
122
|
+
_WORKER_FIELDS = frozenset(
|
|
123
|
+
{
|
|
124
|
+
"checkout_environment",
|
|
125
|
+
"expected_checkout_revision",
|
|
126
|
+
"expected_checkout_tree",
|
|
127
|
+
"adapter_checkout_environment",
|
|
128
|
+
"expected_adapter_checkout_revision",
|
|
129
|
+
"expected_adapter_checkout_tree",
|
|
130
|
+
"executable_path",
|
|
131
|
+
"executable_sha256",
|
|
132
|
+
"entrypoint",
|
|
133
|
+
"entrypoint_sha256",
|
|
134
|
+
"arguments",
|
|
135
|
+
"pass_environment",
|
|
136
|
+
"request_parameters",
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
_CONFIG_FIELDS = frozenset(
|
|
140
|
+
{
|
|
141
|
+
"schema_version",
|
|
142
|
+
"protocol",
|
|
143
|
+
"method_id",
|
|
144
|
+
"candidate_release_id",
|
|
145
|
+
"workers",
|
|
146
|
+
"resource_limits",
|
|
147
|
+
"pipe_limits",
|
|
148
|
+
"shared_request_parameters",
|
|
149
|
+
}
|
|
150
|
+
)
|
|
151
|
+
_RESPONSE_FIELDS = frozenset(
|
|
152
|
+
{
|
|
153
|
+
"schema_version",
|
|
154
|
+
"protocol",
|
|
155
|
+
"request_hash",
|
|
156
|
+
"status",
|
|
157
|
+
"metrics",
|
|
158
|
+
"usage",
|
|
159
|
+
"output",
|
|
160
|
+
"failure",
|
|
161
|
+
}
|
|
162
|
+
)
|
|
163
|
+
_RESPONSE_FORBIDDEN_KEYS = frozenset(
|
|
164
|
+
{
|
|
165
|
+
"implementation_identity",
|
|
166
|
+
"host_execution_receipt",
|
|
167
|
+
"resource_receipt",
|
|
168
|
+
}
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@dataclass(frozen=True, slots=True)
|
|
173
|
+
class _CheckoutIdentity:
|
|
174
|
+
resolved_path_sha256: str
|
|
175
|
+
git_head: str
|
|
176
|
+
git_tree: str
|
|
177
|
+
|
|
178
|
+
def to_dict(self) -> dict[str, Any]:
|
|
179
|
+
return to_jsonable(self)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass(frozen=True, slots=True)
|
|
183
|
+
class _CommandIdentity:
|
|
184
|
+
executable_resolved_path_sha256: str
|
|
185
|
+
executable_content_sha256: str
|
|
186
|
+
entrypoint_relative_path: str
|
|
187
|
+
entrypoint_resolved_path_sha256: str
|
|
188
|
+
entrypoint_content_sha256: str
|
|
189
|
+
argv_sha256: str
|
|
190
|
+
cwd_resolved_path_sha256: str
|
|
191
|
+
pythonpath_resolved_path_sha256: tuple[str, ...]
|
|
192
|
+
|
|
193
|
+
def to_dict(self) -> dict[str, Any]:
|
|
194
|
+
return to_jsonable(self)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass(frozen=True, slots=True)
|
|
198
|
+
class _WorkerSpec:
|
|
199
|
+
arm_id: str
|
|
200
|
+
checkout_environment: str
|
|
201
|
+
checkout: Path
|
|
202
|
+
expected_checkout_revision: str
|
|
203
|
+
expected_checkout_tree: str
|
|
204
|
+
adapter_checkout_environment: str
|
|
205
|
+
adapter_checkout: Path
|
|
206
|
+
expected_adapter_checkout_revision: str
|
|
207
|
+
expected_adapter_checkout_tree: str
|
|
208
|
+
executable: Path
|
|
209
|
+
executable_sha256: str
|
|
210
|
+
entrypoint_relative_path: str
|
|
211
|
+
entrypoint: Path
|
|
212
|
+
entrypoint_sha256: str
|
|
213
|
+
arguments: tuple[str, ...]
|
|
214
|
+
pass_environment: tuple[str, ...]
|
|
215
|
+
request_parameters: Mapping[str, Any]
|
|
216
|
+
checkout_identity: _CheckoutIdentity
|
|
217
|
+
adapter_checkout_identity: _CheckoutIdentity
|
|
218
|
+
command_identity: _CommandIdentity
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def argv(self) -> tuple[str, ...]:
|
|
222
|
+
return (str(self.executable), str(self.entrypoint), *self.arguments)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@dataclass(frozen=True, slots=True)
|
|
226
|
+
class _DriverConfiguration:
|
|
227
|
+
method_id: str
|
|
228
|
+
candidate_release_id: str
|
|
229
|
+
workers: Mapping[str, _WorkerSpec]
|
|
230
|
+
resource_limits: ResourceLimits
|
|
231
|
+
pipe_limits: Mapping[str, int]
|
|
232
|
+
shared_request_parameters: Mapping[str, Any]
|
|
233
|
+
config_sha256: str
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class ParityProcessProtocolError(AlgorithmOutputError):
|
|
237
|
+
"""A worker violated the parity-process protocol after it was started."""
|
|
238
|
+
|
|
239
|
+
def __init__(self, message: str, *, resource_receipt: ResourceReceipt) -> None:
|
|
240
|
+
super().__init__(message)
|
|
241
|
+
self.resource_receipt = resource_receipt
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class ParityProcessExperimentDriver:
|
|
245
|
+
"""Run exact upstream and candidate adapters as independent JSON workers."""
|
|
246
|
+
|
|
247
|
+
def __init__(
|
|
248
|
+
self,
|
|
249
|
+
configuration: _DriverConfiguration,
|
|
250
|
+
*,
|
|
251
|
+
environment: Mapping[str, str],
|
|
252
|
+
supervisor: WorkerSupervisor | None = None,
|
|
253
|
+
) -> None:
|
|
254
|
+
self._configuration = configuration
|
|
255
|
+
self._environment = dict(environment)
|
|
256
|
+
self._supervisor = supervisor or LocalSubprocessSupervisor()
|
|
257
|
+
if not isinstance(self._supervisor, WorkerSupervisor):
|
|
258
|
+
raise TypeError("parity-process supervisor must implement WorkerSupervisor")
|
|
259
|
+
self._manifest_hash: str | None = None
|
|
260
|
+
self._binding_receipt_hash: str | None = None
|
|
261
|
+
self._contract_hash: str | None = None
|
|
262
|
+
self._preflight_worker_state: Mapping[str, Mapping[str, Any]] = freeze_mapping({})
|
|
263
|
+
self._fixture_unit_ids: frozenset[str] = frozenset()
|
|
264
|
+
self._checkout_binding_hashes: Mapping[str, str] = freeze_mapping({})
|
|
265
|
+
self._adapter_checkout_binding_hashes: Mapping[str, str] = freeze_mapping({})
|
|
266
|
+
self._fingerprint = freeze_mapping(self._build_fingerprint())
|
|
267
|
+
|
|
268
|
+
@classmethod
|
|
269
|
+
def from_config(
|
|
270
|
+
cls,
|
|
271
|
+
path: str | Path,
|
|
272
|
+
*,
|
|
273
|
+
expected_sha256: str,
|
|
274
|
+
environment: Mapping[str, str] | None = None,
|
|
275
|
+
supervisor: WorkerSupervisor | None = None,
|
|
276
|
+
) -> ParityProcessExperimentDriver:
|
|
277
|
+
env = dict(os.environ if environment is None else environment)
|
|
278
|
+
configuration = _load_configuration(
|
|
279
|
+
Path(path),
|
|
280
|
+
expected_sha256=expected_sha256,
|
|
281
|
+
environment=env,
|
|
282
|
+
)
|
|
283
|
+
return cls(
|
|
284
|
+
configuration,
|
|
285
|
+
environment=env,
|
|
286
|
+
supervisor=supervisor,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
def fingerprint(self) -> Mapping[str, Any]:
|
|
290
|
+
return self._fingerprint
|
|
291
|
+
|
|
292
|
+
def preflight(self, manifest: ExperimentManifest) -> Sequence[str]:
|
|
293
|
+
blockers: list[str] = []
|
|
294
|
+
if manifest.stage is not ExperimentStage.PARITY:
|
|
295
|
+
blockers.append("parity-process driver requires stage=parity")
|
|
296
|
+
if manifest.metadata.get("method_id") != self._configuration.method_id:
|
|
297
|
+
blockers.append("parity-process config method_id differs from manifest")
|
|
298
|
+
if (
|
|
299
|
+
manifest.metadata.get("candidate_release_id")
|
|
300
|
+
!= self._configuration.candidate_release_id
|
|
301
|
+
):
|
|
302
|
+
blockers.append("parity-process candidate release differs from manifest")
|
|
303
|
+
if set(arm.arm_id for arm in manifest.arms) != set(_ARM_IDS):
|
|
304
|
+
blockers.append("parity-process driver requires exact upstream and candidate arms")
|
|
305
|
+
upstream_commit = manifest.metadata.get("upstream_commit")
|
|
306
|
+
if upstream_commit != self._configuration.workers[_UPSTREAM_ARM].expected_checkout_revision:
|
|
307
|
+
blockers.append("parity-process upstream checkout revision differs from manifest")
|
|
308
|
+
if isinstance(self._supervisor, LocalSubprocessSupervisor) and any(
|
|
309
|
+
value is not None
|
|
310
|
+
for value in (
|
|
311
|
+
self._configuration.resource_limits.memory_bytes,
|
|
312
|
+
self._configuration.resource_limits.cpus,
|
|
313
|
+
self._configuration.resource_limits.pids_limit,
|
|
314
|
+
)
|
|
315
|
+
):
|
|
316
|
+
blockers.append(
|
|
317
|
+
"local parity-process supervision supports hard wall/pipe limits only; "
|
|
318
|
+
"use a container supervisor for memory/CPU/PID limits"
|
|
319
|
+
)
|
|
320
|
+
for worker in self._configuration.workers.values():
|
|
321
|
+
missing = sorted(
|
|
322
|
+
name
|
|
323
|
+
for name in (
|
|
324
|
+
*_BASE_PASSTHROUGH_ENVIRONMENTS,
|
|
325
|
+
*worker.pass_environment,
|
|
326
|
+
)
|
|
327
|
+
if not self._environment.get(name)
|
|
328
|
+
)
|
|
329
|
+
if missing:
|
|
330
|
+
blockers.append(
|
|
331
|
+
f"parity-process {worker.arm_id} missing worker environments: "
|
|
332
|
+
+ ", ".join(missing)
|
|
333
|
+
)
|
|
334
|
+
try:
|
|
335
|
+
fixture_units = _load_fixture_inventory(self._environment)
|
|
336
|
+
except (ArtifactError, ValidationError) as exc:
|
|
337
|
+
blockers.append(redact_secret_text(str(exc)))
|
|
338
|
+
fixture_units = ()
|
|
339
|
+
if blockers:
|
|
340
|
+
return tuple(blockers)
|
|
341
|
+
try:
|
|
342
|
+
state = {
|
|
343
|
+
arm_id: _attest_worker(worker).to_dict()
|
|
344
|
+
for arm_id, worker in self._configuration.workers.items()
|
|
345
|
+
}
|
|
346
|
+
receipt = resolve_frozen_binding_receipt(
|
|
347
|
+
manifest,
|
|
348
|
+
environment=self._environment,
|
|
349
|
+
)
|
|
350
|
+
contract = build_parity_implementation_contract(
|
|
351
|
+
manifest,
|
|
352
|
+
receipt,
|
|
353
|
+
self._fingerprint,
|
|
354
|
+
)
|
|
355
|
+
except (ArtifactError, ExternalDependencyError, ValidationError) as exc:
|
|
356
|
+
return (redact_secret_text(str(exc)),)
|
|
357
|
+
self._manifest_hash = manifest.manifest_hash
|
|
358
|
+
self._binding_receipt_hash = receipt.receipt_hash
|
|
359
|
+
self._contract_hash = contract.contract_hash
|
|
360
|
+
self._preflight_worker_state = freeze_mapping(state)
|
|
361
|
+
self._fixture_unit_ids = frozenset(fixture_units)
|
|
362
|
+
self._checkout_binding_hashes = freeze_mapping(
|
|
363
|
+
{
|
|
364
|
+
arm_id: receipt.binding(
|
|
365
|
+
worker.checkout_environment,
|
|
366
|
+
kind="directory_env",
|
|
367
|
+
).binding_hash
|
|
368
|
+
for arm_id, worker in self._configuration.workers.items()
|
|
369
|
+
}
|
|
370
|
+
)
|
|
371
|
+
self._adapter_checkout_binding_hashes = freeze_mapping(
|
|
372
|
+
{
|
|
373
|
+
arm_id: receipt.binding(
|
|
374
|
+
worker.adapter_checkout_environment,
|
|
375
|
+
kind="directory_env",
|
|
376
|
+
).binding_hash
|
|
377
|
+
for arm_id, worker in self._configuration.workers.items()
|
|
378
|
+
}
|
|
379
|
+
)
|
|
380
|
+
return ()
|
|
381
|
+
|
|
382
|
+
def preflight_units(
|
|
383
|
+
self,
|
|
384
|
+
manifest: ExperimentManifest,
|
|
385
|
+
resolved_units: Mapping[str, Sequence[str]],
|
|
386
|
+
) -> Sequence[str]:
|
|
387
|
+
"""Prove the plan uses exactly the checksum-locked fixture inventory."""
|
|
388
|
+
|
|
389
|
+
if self._manifest_hash != manifest.manifest_hash or not self._fixture_unit_ids:
|
|
390
|
+
return ("unit inventory validation requires a successful driver preflight",)
|
|
391
|
+
expected_keys = {f"{target.cell_id}:{target.split}" for target in manifest.benchmarks}
|
|
392
|
+
if set(resolved_units) != expected_keys:
|
|
393
|
+
return ("resolved unit inventories differ from parity manifest targets",)
|
|
394
|
+
blockers: list[str] = []
|
|
395
|
+
observed: set[str] = set()
|
|
396
|
+
for key in sorted(expected_keys):
|
|
397
|
+
values = tuple(resolved_units[key])
|
|
398
|
+
if len(values) != len(set(values)):
|
|
399
|
+
blockers.append(f"{key}: resolved parity units contain duplicates")
|
|
400
|
+
continue
|
|
401
|
+
observed.update(values)
|
|
402
|
+
missing = sorted(self._fixture_unit_ids - observed)
|
|
403
|
+
extra = sorted(observed - self._fixture_unit_ids)
|
|
404
|
+
if missing or extra:
|
|
405
|
+
blockers.append(
|
|
406
|
+
"resolved parity units differ from the checksum-locked fixture inventory; "
|
|
407
|
+
f"missing={missing[:5]}, extra={extra[:5]}"
|
|
408
|
+
)
|
|
409
|
+
return tuple(blockers)
|
|
410
|
+
|
|
411
|
+
def execute_cell(
|
|
412
|
+
self,
|
|
413
|
+
manifest: ExperimentManifest,
|
|
414
|
+
cell: ExperimentCell,
|
|
415
|
+
) -> ExperimentCellResult:
|
|
416
|
+
self._require_preflight(manifest, cell)
|
|
417
|
+
arm_outcomes: list[dict[str, Any]] = []
|
|
418
|
+
resource_receipts: list[ResourceReceipt] = []
|
|
419
|
+
for arm_id in cell.arm_ids:
|
|
420
|
+
worker = self._configuration.workers[arm_id]
|
|
421
|
+
response, host_receipt, resource_receipt = self._execute_arm(
|
|
422
|
+
manifest,
|
|
423
|
+
cell,
|
|
424
|
+
worker,
|
|
425
|
+
)
|
|
426
|
+
resource_receipts.append(resource_receipt)
|
|
427
|
+
outcome: dict[str, Any] = {
|
|
428
|
+
"arm_id": arm_id,
|
|
429
|
+
"status": response["status"],
|
|
430
|
+
"request_hash": response["request_hash"],
|
|
431
|
+
"usage": response["usage"],
|
|
432
|
+
"output": response["output"],
|
|
433
|
+
"host_execution_receipt": host_receipt,
|
|
434
|
+
}
|
|
435
|
+
if response["status"] == "succeeded":
|
|
436
|
+
outcome["metrics"] = response["metrics"]
|
|
437
|
+
else:
|
|
438
|
+
outcome["failure"] = response["failure"]
|
|
439
|
+
arm_outcomes.append(outcome)
|
|
440
|
+
return ExperimentCellResult(
|
|
441
|
+
cell_id=cell.cell_id,
|
|
442
|
+
status="succeeded",
|
|
443
|
+
payload={
|
|
444
|
+
"frozen_binding_receipt_hash": self._binding_receipt_hash,
|
|
445
|
+
"parity_contract_hash": self._contract_hash,
|
|
446
|
+
"host_execution_receipt_protocol": HOST_EXECUTION_RECEIPT_PROTOCOL,
|
|
447
|
+
"arms": arm_outcomes,
|
|
448
|
+
},
|
|
449
|
+
resource_receipts=tuple(resource_receipts),
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
def _build_fingerprint(self) -> dict[str, Any]:
|
|
453
|
+
declarations: dict[str, Any] = {}
|
|
454
|
+
bindings: dict[str, Any] = {}
|
|
455
|
+
for arm_id, worker in self._configuration.workers.items():
|
|
456
|
+
release_id = (
|
|
457
|
+
f"upstream:{self._configuration.method_id}"
|
|
458
|
+
if arm_id == _UPSTREAM_ARM
|
|
459
|
+
else self._configuration.candidate_release_id
|
|
460
|
+
)
|
|
461
|
+
runtime_entrypoint = (
|
|
462
|
+
f"{worker.adapter_checkout_environment}:{worker.entrypoint_relative_path}"
|
|
463
|
+
)
|
|
464
|
+
declarations[arm_id] = {
|
|
465
|
+
"implementation_kind": (
|
|
466
|
+
"upstream_reference" if arm_id == _UPSTREAM_ARM else "agentinputlab_candidate"
|
|
467
|
+
),
|
|
468
|
+
"implementation_id": (
|
|
469
|
+
f"host-process:{content_hash({'arm_id': arm_id, 'command': worker.command_identity})}"
|
|
470
|
+
),
|
|
471
|
+
"runtime_entrypoint": runtime_entrypoint,
|
|
472
|
+
"source_revision": worker.expected_checkout_revision,
|
|
473
|
+
"source_tree": worker.expected_checkout_tree,
|
|
474
|
+
"release_id": release_id,
|
|
475
|
+
}
|
|
476
|
+
bindings[arm_id] = {
|
|
477
|
+
"checkout_environment": worker.checkout_environment,
|
|
478
|
+
"checkout": worker.checkout_identity.to_dict(),
|
|
479
|
+
"adapter_checkout_environment": worker.adapter_checkout_environment,
|
|
480
|
+
"adapter_checkout": worker.adapter_checkout_identity.to_dict(),
|
|
481
|
+
"command": worker.command_identity.to_dict(),
|
|
482
|
+
}
|
|
483
|
+
return {
|
|
484
|
+
"schema_version": 1,
|
|
485
|
+
"driver": DRIVER_PROTOCOL,
|
|
486
|
+
"config_sha256": self._configuration.config_sha256,
|
|
487
|
+
"method_id": self._configuration.method_id,
|
|
488
|
+
"candidate_release_id": self._configuration.candidate_release_id,
|
|
489
|
+
"execution_receipt_protocol": HOST_EXECUTION_RECEIPT_PROTOCOL,
|
|
490
|
+
"parity_implementations": declarations,
|
|
491
|
+
"host_process_bindings": bindings,
|
|
492
|
+
"resource_limits": self._configuration.resource_limits.as_dict(),
|
|
493
|
+
"pipe_limits": dict(self._configuration.pipe_limits),
|
|
494
|
+
"supervisor": dict(self._supervisor.fingerprint()),
|
|
495
|
+
"fixture_inventory_sha256": self._environment.get(UNIT_INVENTORY_SHA256_ENV),
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
def _require_preflight(
|
|
499
|
+
self,
|
|
500
|
+
manifest: ExperimentManifest,
|
|
501
|
+
cell: ExperimentCell,
|
|
502
|
+
) -> None:
|
|
503
|
+
if (
|
|
504
|
+
self._manifest_hash is None
|
|
505
|
+
or self._binding_receipt_hash is None
|
|
506
|
+
or self._contract_hash is None
|
|
507
|
+
):
|
|
508
|
+
raise ArtifactError("parity-process execute_cell requires successful preflight")
|
|
509
|
+
if manifest.manifest_hash != self._manifest_hash:
|
|
510
|
+
raise ArtifactError("parity-process driver received another manifest")
|
|
511
|
+
if cell.manifest_hash != manifest.manifest_hash:
|
|
512
|
+
raise ValidationError("parity-process cell belongs to another manifest")
|
|
513
|
+
if set(cell.arm_ids) != set(_ARM_IDS):
|
|
514
|
+
raise ValidationError("parity-process cell requires exact implementation arms")
|
|
515
|
+
if cell.unit_id not in self._fixture_unit_ids:
|
|
516
|
+
raise ArtifactError("parity-process cell is outside the preflighted fixture inventory")
|
|
517
|
+
|
|
518
|
+
def _execute_arm(
|
|
519
|
+
self,
|
|
520
|
+
manifest: ExperimentManifest,
|
|
521
|
+
cell: ExperimentCell,
|
|
522
|
+
worker: _WorkerSpec,
|
|
523
|
+
) -> tuple[Mapping[str, Any], Mapping[str, Any], ResourceReceipt]:
|
|
524
|
+
expected_state = self._preflight_worker_state.get(worker.arm_id)
|
|
525
|
+
request = _build_request(
|
|
526
|
+
configuration=self._configuration,
|
|
527
|
+
manifest=manifest,
|
|
528
|
+
cell=cell,
|
|
529
|
+
worker=worker,
|
|
530
|
+
)
|
|
531
|
+
input_text = canonical_json(request) + "\n"
|
|
532
|
+
environment = self._worker_environment(worker)
|
|
533
|
+
passed_values = environment_secret_values(
|
|
534
|
+
self._environment,
|
|
535
|
+
(
|
|
536
|
+
*_BASE_PASSTHROUGH_ENVIRONMENTS,
|
|
537
|
+
*worker.pass_environment,
|
|
538
|
+
),
|
|
539
|
+
)
|
|
540
|
+
process = execute_locked_process_worker(
|
|
541
|
+
worker,
|
|
542
|
+
supervisor=self._supervisor,
|
|
543
|
+
input_text=input_text,
|
|
544
|
+
limits=self._configuration.resource_limits,
|
|
545
|
+
pipe_limits=self._configuration.pipe_limits,
|
|
546
|
+
environment=environment,
|
|
547
|
+
process_name=f"parity-process-{worker.arm_id}",
|
|
548
|
+
config_sha256=self._configuration.config_sha256,
|
|
549
|
+
expected_worker_state=expected_state,
|
|
550
|
+
)
|
|
551
|
+
if process.returncode != 0:
|
|
552
|
+
message = redact_secret_text(
|
|
553
|
+
process.stderr.strip() or "worker exited non-zero",
|
|
554
|
+
secret_values=passed_values,
|
|
555
|
+
)
|
|
556
|
+
raise SupervisedWorkerError(
|
|
557
|
+
f"parity-process {worker.arm_id} failed: {message}",
|
|
558
|
+
resource_receipt=process.resource_receipt,
|
|
559
|
+
)
|
|
560
|
+
response = _parse_response(
|
|
561
|
+
process,
|
|
562
|
+
expected_request_hash=request["request_hash"],
|
|
563
|
+
secret_values=passed_values,
|
|
564
|
+
)
|
|
565
|
+
host_receipt = _host_execution_receipt(
|
|
566
|
+
config_sha256=self._configuration.config_sha256,
|
|
567
|
+
manifest=manifest,
|
|
568
|
+
cell=cell,
|
|
569
|
+
worker=worker,
|
|
570
|
+
request=request,
|
|
571
|
+
process=process,
|
|
572
|
+
checkout_binding_hash=self._checkout_binding_hashes[worker.arm_id],
|
|
573
|
+
adapter_checkout_binding_hash=self._adapter_checkout_binding_hashes[worker.arm_id],
|
|
574
|
+
)
|
|
575
|
+
return response, host_receipt, process.resource_receipt
|
|
576
|
+
|
|
577
|
+
def _worker_environment(self, worker: _WorkerSpec) -> Mapping[str, str]:
|
|
578
|
+
names = (
|
|
579
|
+
*_BASE_PASSTHROUGH_ENVIRONMENTS,
|
|
580
|
+
*worker.pass_environment,
|
|
581
|
+
)
|
|
582
|
+
environment = dict(
|
|
583
|
+
locked_process_worker_environment(
|
|
584
|
+
worker,
|
|
585
|
+
environment=self._environment,
|
|
586
|
+
pass_environment=names,
|
|
587
|
+
request_protocol=REQUEST_PROTOCOL,
|
|
588
|
+
)
|
|
589
|
+
)
|
|
590
|
+
environment.update(
|
|
591
|
+
{
|
|
592
|
+
"AILAB_PARITY_ARM_ID": worker.arm_id,
|
|
593
|
+
"AILAB_PARITY_PROTOCOL": REQUEST_PROTOCOL,
|
|
594
|
+
}
|
|
595
|
+
)
|
|
596
|
+
return freeze_mapping(environment)
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def factory() -> ParityProcessExperimentDriver:
|
|
600
|
+
"""Load the checksum-locked parity-process configuration from the environment."""
|
|
601
|
+
|
|
602
|
+
path = os.environ.get(CONFIG_PATH_ENV)
|
|
603
|
+
expected = os.environ.get(CONFIG_SHA256_ENV)
|
|
604
|
+
if not path:
|
|
605
|
+
raise ArtifactError(f"{CONFIG_PATH_ENV} is required")
|
|
606
|
+
if not expected:
|
|
607
|
+
raise ArtifactError(f"{CONFIG_SHA256_ENV} is required")
|
|
608
|
+
return ParityProcessExperimentDriver.from_config(
|
|
609
|
+
path,
|
|
610
|
+
expected_sha256=expected,
|
|
611
|
+
supervisor=worker_supervisor_from_environment(),
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
LockedProcessWorker = _WorkerSpec
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def load_locked_process_worker(
|
|
619
|
+
arm_id: str,
|
|
620
|
+
value: Any,
|
|
621
|
+
*,
|
|
622
|
+
environment: Mapping[str, str],
|
|
623
|
+
checkout_environment: str,
|
|
624
|
+
adapter_checkout_environment: str | None = None,
|
|
625
|
+
) -> LockedProcessWorker:
|
|
626
|
+
"""Resolve one exact-SHA worker for another host-process experiment."""
|
|
627
|
+
|
|
628
|
+
return _load_worker(
|
|
629
|
+
arm_id,
|
|
630
|
+
value,
|
|
631
|
+
environment=environment,
|
|
632
|
+
expected_checkout_environment=checkout_environment,
|
|
633
|
+
expected_adapter_checkout_environment=(
|
|
634
|
+
checkout_environment
|
|
635
|
+
if adapter_checkout_environment is None
|
|
636
|
+
else adapter_checkout_environment
|
|
637
|
+
),
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def attest_locked_process_worker(
|
|
642
|
+
worker: LockedProcessWorker,
|
|
643
|
+
) -> Mapping[str, Any]:
|
|
644
|
+
"""Independently re-attest a worker checkout and command."""
|
|
645
|
+
|
|
646
|
+
return freeze_mapping(_attest_worker(worker).to_dict())
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def locked_process_worker_environment(
|
|
650
|
+
worker: LockedProcessWorker,
|
|
651
|
+
*,
|
|
652
|
+
environment: Mapping[str, str],
|
|
653
|
+
pass_environment: Sequence[str],
|
|
654
|
+
request_protocol: str,
|
|
655
|
+
) -> Mapping[str, str]:
|
|
656
|
+
"""Build a minimal environment with checkout-confined Python imports."""
|
|
657
|
+
|
|
658
|
+
missing = sorted(name for name in pass_environment if not environment.get(name))
|
|
659
|
+
if missing:
|
|
660
|
+
raise ArtifactError(
|
|
661
|
+
f"locked process {worker.arm_id} missing environments: " + ", ".join(missing)
|
|
662
|
+
)
|
|
663
|
+
result = {name: environment[name] for name in pass_environment}
|
|
664
|
+
python_paths = [str(worker.checkout)]
|
|
665
|
+
source_root = worker.checkout / "src"
|
|
666
|
+
if source_root.is_dir():
|
|
667
|
+
python_paths.append(str(source_root))
|
|
668
|
+
result.update(
|
|
669
|
+
{
|
|
670
|
+
"AILAB_PROCESS_ARM_ID": worker.arm_id,
|
|
671
|
+
"AILAB_PROCESS_PROTOCOL": request_protocol,
|
|
672
|
+
"LANG": "C",
|
|
673
|
+
"LC_ALL": "C",
|
|
674
|
+
"PWD": str(worker.checkout),
|
|
675
|
+
"PYTHONPATH": os.pathsep.join(python_paths),
|
|
676
|
+
"PYTHONDONTWRITEBYTECODE": "1",
|
|
677
|
+
"PYTHONIOENCODING": "utf-8",
|
|
678
|
+
"PYTHONNOUSERSITE": "1",
|
|
679
|
+
"TZ": "UTC",
|
|
680
|
+
}
|
|
681
|
+
)
|
|
682
|
+
return freeze_mapping(result)
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def execute_locked_process_worker(
|
|
686
|
+
worker: LockedProcessWorker,
|
|
687
|
+
*,
|
|
688
|
+
supervisor: WorkerSupervisor,
|
|
689
|
+
input_text: str,
|
|
690
|
+
limits: ResourceLimits,
|
|
691
|
+
pipe_limits: Mapping[str, int],
|
|
692
|
+
environment: Mapping[str, str],
|
|
693
|
+
process_name: str,
|
|
694
|
+
config_sha256: str,
|
|
695
|
+
expected_worker_state: Mapping[str, Any],
|
|
696
|
+
) -> SupervisedProcessResult:
|
|
697
|
+
"""Run one fresh worker and fail if its identity changes around the call."""
|
|
698
|
+
|
|
699
|
+
before = attest_locked_process_worker(worker)
|
|
700
|
+
if before != freeze_mapping(expected_worker_state):
|
|
701
|
+
raise ArtifactError(
|
|
702
|
+
f"locked process {worker.arm_id} command or checkout drifted after preflight"
|
|
703
|
+
)
|
|
704
|
+
try:
|
|
705
|
+
identity = {
|
|
706
|
+
"kind": "content_attested_checkout_process",
|
|
707
|
+
"arm_id": worker.arm_id,
|
|
708
|
+
"checkout": worker.checkout_identity.to_dict(),
|
|
709
|
+
"adapter_checkout": worker.adapter_checkout_identity.to_dict(),
|
|
710
|
+
"command": worker.command_identity.to_dict(),
|
|
711
|
+
"config_sha256": config_sha256,
|
|
712
|
+
}
|
|
713
|
+
if isinstance(
|
|
714
|
+
supervisor,
|
|
715
|
+
(DockerWorkerSupervisor, LocalSubprocessSupervisor),
|
|
716
|
+
):
|
|
717
|
+
process = supervisor.execute_locked_worker(
|
|
718
|
+
worker.argv,
|
|
719
|
+
checkout=worker.checkout,
|
|
720
|
+
adapter_checkout=worker.adapter_checkout,
|
|
721
|
+
entrypoint_relative_path=str(worker.entrypoint_relative_path),
|
|
722
|
+
input_text=input_text,
|
|
723
|
+
limits=limits,
|
|
724
|
+
max_input_bytes=pipe_limits["max_input_bytes"],
|
|
725
|
+
max_stdout_bytes=pipe_limits["max_stdout_bytes"],
|
|
726
|
+
max_stderr_bytes=pipe_limits["max_stderr_bytes"],
|
|
727
|
+
environment=environment,
|
|
728
|
+
process_name=process_name,
|
|
729
|
+
image_identity=identity,
|
|
730
|
+
)
|
|
731
|
+
else:
|
|
732
|
+
process = supervisor.execute(
|
|
733
|
+
worker.argv,
|
|
734
|
+
input_text=input_text,
|
|
735
|
+
limits=limits,
|
|
736
|
+
max_input_bytes=pipe_limits["max_input_bytes"],
|
|
737
|
+
max_stdout_bytes=pipe_limits["max_stdout_bytes"],
|
|
738
|
+
max_stderr_bytes=pipe_limits["max_stderr_bytes"],
|
|
739
|
+
cwd=str(worker.checkout),
|
|
740
|
+
environment=environment,
|
|
741
|
+
process_name=process_name,
|
|
742
|
+
image_identity=identity,
|
|
743
|
+
)
|
|
744
|
+
except SupervisedWorkerError as exc:
|
|
745
|
+
try:
|
|
746
|
+
after = attest_locked_process_worker(worker)
|
|
747
|
+
except (ArtifactError, ExternalDependencyError, ValidationError) as drift:
|
|
748
|
+
error = ArtifactError(
|
|
749
|
+
f"locked process {worker.arm_id} identity changed during failed execution"
|
|
750
|
+
)
|
|
751
|
+
error.resource_receipt = exc.resource_receipt # type: ignore[attr-defined]
|
|
752
|
+
raise error from drift
|
|
753
|
+
if after != freeze_mapping(expected_worker_state):
|
|
754
|
+
error = ArtifactError(
|
|
755
|
+
f"locked process {worker.arm_id} identity drifted during failed execution"
|
|
756
|
+
)
|
|
757
|
+
error.resource_receipt = exc.resource_receipt # type: ignore[attr-defined]
|
|
758
|
+
raise error from exc
|
|
759
|
+
raise
|
|
760
|
+
try:
|
|
761
|
+
after = attest_locked_process_worker(worker)
|
|
762
|
+
except (ArtifactError, ExternalDependencyError, ValidationError) as exc:
|
|
763
|
+
error = ArtifactError(f"locked process {worker.arm_id} identity changed during execution")
|
|
764
|
+
error.resource_receipt = process.resource_receipt # type: ignore[attr-defined]
|
|
765
|
+
raise error from exc
|
|
766
|
+
if after != freeze_mapping(expected_worker_state):
|
|
767
|
+
error = ArtifactError(f"locked process {worker.arm_id} identity drifted during execution")
|
|
768
|
+
error.resource_receipt = process.resource_receipt # type: ignore[attr-defined]
|
|
769
|
+
raise error
|
|
770
|
+
return process
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def parse_locked_process_response(
|
|
774
|
+
process: SupervisedProcessResult,
|
|
775
|
+
*,
|
|
776
|
+
expected_request_hash: str,
|
|
777
|
+
secret_values: Sequence[str] = (),
|
|
778
|
+
) -> Mapping[str, Any]:
|
|
779
|
+
"""Parse the shared strict response schema and reject identity echoes."""
|
|
780
|
+
|
|
781
|
+
return _parse_response(
|
|
782
|
+
process,
|
|
783
|
+
expected_request_hash=expected_request_hash,
|
|
784
|
+
secret_values=secret_values,
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def build_host_execution_receipt(
|
|
789
|
+
*,
|
|
790
|
+
config_sha256: str,
|
|
791
|
+
manifest: ExperimentManifest,
|
|
792
|
+
cell: ExperimentCell,
|
|
793
|
+
worker: LockedProcessWorker,
|
|
794
|
+
request: Mapping[str, Any],
|
|
795
|
+
process: SupervisedProcessResult,
|
|
796
|
+
checkout_binding_hash: str,
|
|
797
|
+
adapter_checkout_binding_hash: str,
|
|
798
|
+
) -> Mapping[str, Any]:
|
|
799
|
+
"""Build the host-owned receipt consumed by the guarded validator."""
|
|
800
|
+
|
|
801
|
+
return _host_execution_receipt(
|
|
802
|
+
config_sha256=config_sha256,
|
|
803
|
+
manifest=manifest,
|
|
804
|
+
cell=cell,
|
|
805
|
+
worker=worker,
|
|
806
|
+
request=request,
|
|
807
|
+
process=process,
|
|
808
|
+
checkout_binding_hash=checkout_binding_hash,
|
|
809
|
+
adapter_checkout_binding_hash=adapter_checkout_binding_hash,
|
|
810
|
+
)
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def _load_configuration(
|
|
814
|
+
path: Path,
|
|
815
|
+
*,
|
|
816
|
+
expected_sha256: str,
|
|
817
|
+
environment: Mapping[str, str],
|
|
818
|
+
) -> _DriverConfiguration:
|
|
819
|
+
require_sha256(expected_sha256, label=CONFIG_SHA256_ENV)
|
|
820
|
+
unresolved_source = path.expanduser()
|
|
821
|
+
if unresolved_source.is_symlink():
|
|
822
|
+
raise ArtifactError("parity-process config must be a regular non-symlink file")
|
|
823
|
+
source = unresolved_source.resolve()
|
|
824
|
+
if not source.is_file():
|
|
825
|
+
raise ArtifactError("parity-process config must be a regular non-symlink file")
|
|
826
|
+
actual_sha256 = _sha256_file(source)
|
|
827
|
+
if actual_sha256 != expected_sha256:
|
|
828
|
+
raise ArtifactError("parity-process config checksum differs from its environment lock")
|
|
829
|
+
try:
|
|
830
|
+
raw = json.loads(source.read_text(encoding="utf-8"))
|
|
831
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
832
|
+
raise ArtifactError("parity-process config is not valid UTF-8 JSON") from exc
|
|
833
|
+
data = _strict_mapping(raw, _CONFIG_FIELDS, label="parity-process config")
|
|
834
|
+
require_secret_free(data, label="parity-process config")
|
|
835
|
+
if data["schema_version"] != 1 or data["protocol"] != DRIVER_PROTOCOL:
|
|
836
|
+
raise ValidationError("unsupported parity-process config schema or protocol")
|
|
837
|
+
method_id = _nonempty_text(data["method_id"], "method_id")
|
|
838
|
+
candidate_release_id = _nonempty_text(
|
|
839
|
+
data["candidate_release_id"],
|
|
840
|
+
"candidate_release_id",
|
|
841
|
+
)
|
|
842
|
+
raw_workers = _strict_mapping(
|
|
843
|
+
data["workers"],
|
|
844
|
+
frozenset(_ARM_IDS),
|
|
845
|
+
label="parity-process workers",
|
|
846
|
+
)
|
|
847
|
+
workers = {
|
|
848
|
+
arm_id: _load_worker(
|
|
849
|
+
arm_id,
|
|
850
|
+
raw_workers[arm_id],
|
|
851
|
+
environment=environment,
|
|
852
|
+
)
|
|
853
|
+
for arm_id in _ARM_IDS
|
|
854
|
+
}
|
|
855
|
+
if workers[_UPSTREAM_ARM].checkout == workers[_CANDIDATE_ARM].checkout:
|
|
856
|
+
raise ValidationError("parity-process arms cannot use the same checkout")
|
|
857
|
+
if workers[_UPSTREAM_ARM].entrypoint == workers[_CANDIDATE_ARM].entrypoint:
|
|
858
|
+
raise ValidationError("parity-process arms cannot use the same entrypoint")
|
|
859
|
+
if workers[_UPSTREAM_ARM].argv == workers[_CANDIDATE_ARM].argv:
|
|
860
|
+
raise ValidationError("parity-process arms cannot use the same command")
|
|
861
|
+
limits_data = _strict_mapping(
|
|
862
|
+
data["resource_limits"],
|
|
863
|
+
frozenset(
|
|
864
|
+
{
|
|
865
|
+
"wall_timeout_seconds",
|
|
866
|
+
"memory_bytes",
|
|
867
|
+
"cpus",
|
|
868
|
+
"pids_limit",
|
|
869
|
+
}
|
|
870
|
+
),
|
|
871
|
+
label="parity-process resource_limits",
|
|
872
|
+
)
|
|
873
|
+
resource_limits = ResourceLimits(**limits_data)
|
|
874
|
+
pipe_data = _strict_mapping(
|
|
875
|
+
data["pipe_limits"],
|
|
876
|
+
_PIPE_FIELDS,
|
|
877
|
+
label="parity-process pipe_limits",
|
|
878
|
+
)
|
|
879
|
+
pipe_limits = {
|
|
880
|
+
name: _positive_integer(pipe_data[name], f"pipe_limits.{name}")
|
|
881
|
+
for name in sorted(_PIPE_FIELDS)
|
|
882
|
+
}
|
|
883
|
+
shared_parameters = _canonical_mapping(
|
|
884
|
+
data["shared_request_parameters"],
|
|
885
|
+
label="shared_request_parameters",
|
|
886
|
+
)
|
|
887
|
+
require_secret_free(shared_parameters, label="parity-process shared parameters")
|
|
888
|
+
return _DriverConfiguration(
|
|
889
|
+
method_id=method_id,
|
|
890
|
+
candidate_release_id=candidate_release_id,
|
|
891
|
+
workers=workers,
|
|
892
|
+
resource_limits=resource_limits,
|
|
893
|
+
pipe_limits=freeze_mapping(pipe_limits),
|
|
894
|
+
shared_request_parameters=shared_parameters,
|
|
895
|
+
config_sha256=actual_sha256,
|
|
896
|
+
)
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def _load_worker(
|
|
900
|
+
arm_id: str,
|
|
901
|
+
raw: Any,
|
|
902
|
+
*,
|
|
903
|
+
environment: Mapping[str, str],
|
|
904
|
+
expected_checkout_environment: str | None = None,
|
|
905
|
+
expected_adapter_checkout_environment: str | None = None,
|
|
906
|
+
) -> _WorkerSpec:
|
|
907
|
+
data = _strict_mapping(raw, _WORKER_FIELDS, label=f"worker {arm_id}")
|
|
908
|
+
checkout_environment = _nonempty_text(
|
|
909
|
+
data["checkout_environment"],
|
|
910
|
+
f"{arm_id}.checkout_environment",
|
|
911
|
+
)
|
|
912
|
+
required_checkout_environment = (
|
|
913
|
+
_CHECKOUT_ENVIRONMENTS[arm_id]
|
|
914
|
+
if expected_checkout_environment is None
|
|
915
|
+
else expected_checkout_environment
|
|
916
|
+
)
|
|
917
|
+
if checkout_environment != required_checkout_environment:
|
|
918
|
+
raise ValidationError(f"{arm_id} must use {required_checkout_environment}")
|
|
919
|
+
checkout_value = environment.get(checkout_environment)
|
|
920
|
+
if not checkout_value:
|
|
921
|
+
raise ArtifactError(f"missing checkout environment {checkout_environment}")
|
|
922
|
+
checkout = Path(checkout_value).expanduser().resolve()
|
|
923
|
+
expected_revision = _git_object_id(
|
|
924
|
+
data["expected_checkout_revision"],
|
|
925
|
+
f"{arm_id}.expected_checkout_revision",
|
|
926
|
+
)
|
|
927
|
+
expected_tree = _git_object_id(
|
|
928
|
+
data["expected_checkout_tree"],
|
|
929
|
+
f"{arm_id}.expected_checkout_tree",
|
|
930
|
+
)
|
|
931
|
+
checkout_identity = _attest_checkout(
|
|
932
|
+
checkout,
|
|
933
|
+
expected_revision=expected_revision,
|
|
934
|
+
expected_tree=expected_tree,
|
|
935
|
+
)
|
|
936
|
+
adapter_checkout_environment = _nonempty_text(
|
|
937
|
+
data["adapter_checkout_environment"],
|
|
938
|
+
f"{arm_id}.adapter_checkout_environment",
|
|
939
|
+
)
|
|
940
|
+
required_adapter_environment = (
|
|
941
|
+
"AILAB_CANDIDATE_CHECKOUT"
|
|
942
|
+
if expected_adapter_checkout_environment is None
|
|
943
|
+
else expected_adapter_checkout_environment
|
|
944
|
+
)
|
|
945
|
+
if adapter_checkout_environment != required_adapter_environment:
|
|
946
|
+
raise ValidationError(f"{arm_id} adapter must use {required_adapter_environment}")
|
|
947
|
+
adapter_checkout_value = environment.get(adapter_checkout_environment)
|
|
948
|
+
if not adapter_checkout_value:
|
|
949
|
+
raise ArtifactError(f"missing adapter checkout environment {adapter_checkout_environment}")
|
|
950
|
+
adapter_checkout = Path(adapter_checkout_value).expanduser().resolve()
|
|
951
|
+
expected_adapter_revision = _git_object_id(
|
|
952
|
+
data["expected_adapter_checkout_revision"],
|
|
953
|
+
f"{arm_id}.expected_adapter_checkout_revision",
|
|
954
|
+
)
|
|
955
|
+
expected_adapter_tree = _git_object_id(
|
|
956
|
+
data["expected_adapter_checkout_tree"],
|
|
957
|
+
f"{arm_id}.expected_adapter_checkout_tree",
|
|
958
|
+
)
|
|
959
|
+
adapter_checkout_identity = _attest_checkout(
|
|
960
|
+
adapter_checkout,
|
|
961
|
+
expected_revision=expected_adapter_revision,
|
|
962
|
+
expected_tree=expected_adapter_tree,
|
|
963
|
+
)
|
|
964
|
+
executable = (
|
|
965
|
+
Path(_nonempty_text(data["executable_path"], f"{arm_id}.executable_path"))
|
|
966
|
+
.expanduser()
|
|
967
|
+
.resolve()
|
|
968
|
+
)
|
|
969
|
+
if not executable.is_file():
|
|
970
|
+
raise ArtifactError(f"{arm_id} executable does not exist")
|
|
971
|
+
executable_sha256 = _sha256_text_field(
|
|
972
|
+
data["executable_sha256"],
|
|
973
|
+
f"{arm_id}.executable_sha256",
|
|
974
|
+
)
|
|
975
|
+
if _sha256_file(executable) != executable_sha256:
|
|
976
|
+
raise ArtifactError(f"{arm_id} executable checksum differs from config")
|
|
977
|
+
entrypoint_relative_path = _relative_entrypoint(
|
|
978
|
+
data["entrypoint"],
|
|
979
|
+
label=f"{arm_id}.entrypoint",
|
|
980
|
+
)
|
|
981
|
+
unresolved_entrypoint = adapter_checkout / entrypoint_relative_path
|
|
982
|
+
if unresolved_entrypoint.is_symlink():
|
|
983
|
+
raise ArtifactError(f"{arm_id} entrypoint must not be a symlink")
|
|
984
|
+
entrypoint = unresolved_entrypoint.resolve()
|
|
985
|
+
if not entrypoint.is_relative_to(adapter_checkout):
|
|
986
|
+
raise ValidationError(f"{arm_id} entrypoint escapes its adapter checkout")
|
|
987
|
+
if not entrypoint.is_file() or not stat.S_ISREG(entrypoint.stat().st_mode):
|
|
988
|
+
raise ArtifactError(f"{arm_id} entrypoint must be a regular non-symlink file")
|
|
989
|
+
_require_tracked_file(adapter_checkout, entrypoint_relative_path)
|
|
990
|
+
entrypoint_sha256 = _sha256_text_field(
|
|
991
|
+
data["entrypoint_sha256"],
|
|
992
|
+
f"{arm_id}.entrypoint_sha256",
|
|
993
|
+
)
|
|
994
|
+
if _sha256_file(entrypoint) != entrypoint_sha256:
|
|
995
|
+
raise ArtifactError(f"{arm_id} entrypoint checksum differs from config")
|
|
996
|
+
arguments = _string_sequence(data["arguments"], label=f"{arm_id}.arguments")
|
|
997
|
+
pass_environment = _environment_names(
|
|
998
|
+
data["pass_environment"],
|
|
999
|
+
label=f"{arm_id}.pass_environment",
|
|
1000
|
+
)
|
|
1001
|
+
reserved = sorted(set(pass_environment) & _RESERVED_WORKER_ENVIRONMENTS)
|
|
1002
|
+
if reserved:
|
|
1003
|
+
raise ValidationError(
|
|
1004
|
+
f"{arm_id} pass_environment contains host-controlled names: " + ", ".join(reserved)
|
|
1005
|
+
)
|
|
1006
|
+
request_parameters = _canonical_mapping(
|
|
1007
|
+
data["request_parameters"],
|
|
1008
|
+
label=f"{arm_id}.request_parameters",
|
|
1009
|
+
)
|
|
1010
|
+
require_secret_free(
|
|
1011
|
+
request_parameters,
|
|
1012
|
+
label=f"parity-process {arm_id} request parameters",
|
|
1013
|
+
)
|
|
1014
|
+
python_paths = [checkout]
|
|
1015
|
+
if (checkout / "src").is_dir():
|
|
1016
|
+
python_paths.append(checkout / "src")
|
|
1017
|
+
command_identity = _CommandIdentity(
|
|
1018
|
+
executable_resolved_path_sha256=_sha256_text(str(executable)),
|
|
1019
|
+
executable_content_sha256=executable_sha256,
|
|
1020
|
+
entrypoint_relative_path=entrypoint_relative_path,
|
|
1021
|
+
entrypoint_resolved_path_sha256=_sha256_text(str(entrypoint)),
|
|
1022
|
+
entrypoint_content_sha256=entrypoint_sha256,
|
|
1023
|
+
argv_sha256=content_hash(
|
|
1024
|
+
{
|
|
1025
|
+
"argv": (
|
|
1026
|
+
str(executable),
|
|
1027
|
+
str(entrypoint),
|
|
1028
|
+
*arguments,
|
|
1029
|
+
)
|
|
1030
|
+
}
|
|
1031
|
+
),
|
|
1032
|
+
cwd_resolved_path_sha256=checkout_identity.resolved_path_sha256,
|
|
1033
|
+
pythonpath_resolved_path_sha256=tuple(
|
|
1034
|
+
_sha256_text(str(item.resolve())) for item in python_paths
|
|
1035
|
+
),
|
|
1036
|
+
)
|
|
1037
|
+
return _WorkerSpec(
|
|
1038
|
+
arm_id=arm_id,
|
|
1039
|
+
checkout_environment=checkout_environment,
|
|
1040
|
+
checkout=checkout,
|
|
1041
|
+
expected_checkout_revision=expected_revision,
|
|
1042
|
+
expected_checkout_tree=expected_tree,
|
|
1043
|
+
adapter_checkout_environment=adapter_checkout_environment,
|
|
1044
|
+
adapter_checkout=adapter_checkout,
|
|
1045
|
+
expected_adapter_checkout_revision=expected_adapter_revision,
|
|
1046
|
+
expected_adapter_checkout_tree=expected_adapter_tree,
|
|
1047
|
+
executable=executable,
|
|
1048
|
+
executable_sha256=executable_sha256,
|
|
1049
|
+
entrypoint_relative_path=entrypoint_relative_path,
|
|
1050
|
+
entrypoint=entrypoint,
|
|
1051
|
+
entrypoint_sha256=entrypoint_sha256,
|
|
1052
|
+
arguments=arguments,
|
|
1053
|
+
pass_environment=pass_environment,
|
|
1054
|
+
request_parameters=request_parameters,
|
|
1055
|
+
checkout_identity=checkout_identity,
|
|
1056
|
+
adapter_checkout_identity=adapter_checkout_identity,
|
|
1057
|
+
command_identity=command_identity,
|
|
1058
|
+
)
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _attest_worker(worker: _WorkerSpec) -> _CommandIdentity:
|
|
1062
|
+
checkout = _attest_checkout(
|
|
1063
|
+
worker.checkout,
|
|
1064
|
+
expected_revision=worker.expected_checkout_revision,
|
|
1065
|
+
expected_tree=worker.expected_checkout_tree,
|
|
1066
|
+
)
|
|
1067
|
+
adapter_checkout = _attest_checkout(
|
|
1068
|
+
worker.adapter_checkout,
|
|
1069
|
+
expected_revision=worker.expected_adapter_checkout_revision,
|
|
1070
|
+
expected_tree=worker.expected_adapter_checkout_tree,
|
|
1071
|
+
)
|
|
1072
|
+
if _sha256_file(worker.executable) != worker.executable_sha256:
|
|
1073
|
+
raise ArtifactError(f"{worker.arm_id} executable changed")
|
|
1074
|
+
unresolved_entrypoint = worker.adapter_checkout / worker.entrypoint_relative_path
|
|
1075
|
+
if unresolved_entrypoint.is_symlink():
|
|
1076
|
+
raise ArtifactError(f"{worker.arm_id} entrypoint path changed")
|
|
1077
|
+
entrypoint = unresolved_entrypoint.resolve()
|
|
1078
|
+
if (
|
|
1079
|
+
entrypoint != worker.entrypoint
|
|
1080
|
+
or not entrypoint.is_relative_to(worker.adapter_checkout)
|
|
1081
|
+
or not entrypoint.is_file()
|
|
1082
|
+
):
|
|
1083
|
+
raise ArtifactError(f"{worker.arm_id} entrypoint path changed")
|
|
1084
|
+
_require_tracked_file(worker.adapter_checkout, worker.entrypoint_relative_path)
|
|
1085
|
+
if _sha256_file(entrypoint) != worker.entrypoint_sha256:
|
|
1086
|
+
raise ArtifactError(f"{worker.arm_id} entrypoint changed")
|
|
1087
|
+
if checkout.to_dict() != worker.checkout_identity.to_dict():
|
|
1088
|
+
raise ArtifactError(f"{worker.arm_id} checkout identity changed")
|
|
1089
|
+
if adapter_checkout.to_dict() != worker.adapter_checkout_identity.to_dict():
|
|
1090
|
+
raise ArtifactError(f"{worker.arm_id} adapter checkout identity changed")
|
|
1091
|
+
return worker.command_identity
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _attest_checkout(
|
|
1095
|
+
checkout: Path,
|
|
1096
|
+
*,
|
|
1097
|
+
expected_revision: str,
|
|
1098
|
+
expected_tree: str,
|
|
1099
|
+
) -> _CheckoutIdentity:
|
|
1100
|
+
if not checkout.is_dir():
|
|
1101
|
+
raise ArtifactError("parity-process checkout is unavailable")
|
|
1102
|
+
top_level = Path(_git(checkout, "rev-parse", "--show-toplevel")).resolve()
|
|
1103
|
+
if top_level != checkout:
|
|
1104
|
+
raise ArtifactError("parity-process checkout must identify its Git root")
|
|
1105
|
+
head = _git(checkout, "rev-parse", "HEAD")
|
|
1106
|
+
tree = _git(checkout, "rev-parse", "HEAD^{tree}")
|
|
1107
|
+
if head != expected_revision:
|
|
1108
|
+
raise ArtifactError("parity-process checkout HEAD differs from config")
|
|
1109
|
+
if tree != expected_tree:
|
|
1110
|
+
raise ArtifactError("parity-process checkout tree differs from config")
|
|
1111
|
+
status = _git(checkout, "status", "--porcelain=v1", "--untracked-files=all")
|
|
1112
|
+
if status:
|
|
1113
|
+
raise ArtifactError("parity-process checkout must have a clean working tree")
|
|
1114
|
+
return _CheckoutIdentity(
|
|
1115
|
+
resolved_path_sha256=_sha256_text(str(checkout)),
|
|
1116
|
+
git_head=head,
|
|
1117
|
+
git_tree=tree,
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def _require_tracked_file(checkout: Path, relative_path: str) -> None:
|
|
1122
|
+
try:
|
|
1123
|
+
_git(checkout, "ls-files", "--error-unmatch", "--", relative_path)
|
|
1124
|
+
except ArtifactError as exc:
|
|
1125
|
+
raise ArtifactError("parity-process entrypoint must be tracked by Git") from exc
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def _build_request(
|
|
1129
|
+
*,
|
|
1130
|
+
configuration: _DriverConfiguration,
|
|
1131
|
+
manifest: ExperimentManifest,
|
|
1132
|
+
cell: ExperimentCell,
|
|
1133
|
+
worker: _WorkerSpec,
|
|
1134
|
+
) -> Mapping[str, Any]:
|
|
1135
|
+
arm = next(item for item in manifest.arms if item.arm_id == worker.arm_id)
|
|
1136
|
+
payload = {
|
|
1137
|
+
"schema_version": 1,
|
|
1138
|
+
"protocol": REQUEST_PROTOCOL,
|
|
1139
|
+
"manifest_hash": manifest.manifest_hash,
|
|
1140
|
+
"cell": {
|
|
1141
|
+
"cell_id": cell.cell_id,
|
|
1142
|
+
"study_id": cell.study_id,
|
|
1143
|
+
"benchmark_id": cell.benchmark_id,
|
|
1144
|
+
"split": cell.split,
|
|
1145
|
+
"unit_id": cell.unit_id,
|
|
1146
|
+
"model_slot": cell.model_slot,
|
|
1147
|
+
"seed": cell.seed,
|
|
1148
|
+
},
|
|
1149
|
+
"method": {
|
|
1150
|
+
"method_id": configuration.method_id,
|
|
1151
|
+
"candidate_release_id": configuration.candidate_release_id,
|
|
1152
|
+
"profile_kind": arm.profile_kind.value,
|
|
1153
|
+
"family": arm.family.value,
|
|
1154
|
+
},
|
|
1155
|
+
"arm": {
|
|
1156
|
+
"arm_id": worker.arm_id,
|
|
1157
|
+
"release_id": (
|
|
1158
|
+
f"upstream:{configuration.method_id}"
|
|
1159
|
+
if worker.arm_id == _UPSTREAM_ARM
|
|
1160
|
+
else configuration.candidate_release_id
|
|
1161
|
+
),
|
|
1162
|
+
},
|
|
1163
|
+
"parameters": {
|
|
1164
|
+
"shared": configuration.shared_request_parameters,
|
|
1165
|
+
"worker": worker.request_parameters,
|
|
1166
|
+
},
|
|
1167
|
+
"asset_environments": list(_BASE_PASSTHROUGH_ENVIRONMENTS),
|
|
1168
|
+
}
|
|
1169
|
+
require_secret_free(payload, label="parity-process worker request")
|
|
1170
|
+
request_hash = content_hash(payload)
|
|
1171
|
+
return freeze_mapping({**payload, "request_hash": request_hash})
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _parse_response(
|
|
1175
|
+
process: SupervisedProcessResult,
|
|
1176
|
+
*,
|
|
1177
|
+
expected_request_hash: str,
|
|
1178
|
+
secret_values: Sequence[str] = (),
|
|
1179
|
+
) -> Mapping[str, Any]:
|
|
1180
|
+
try:
|
|
1181
|
+
raw = json.loads(process.stdout)
|
|
1182
|
+
except json.JSONDecodeError as exc:
|
|
1183
|
+
raise ParityProcessProtocolError(
|
|
1184
|
+
"parity-process worker stdout is not one JSON value",
|
|
1185
|
+
resource_receipt=process.resource_receipt,
|
|
1186
|
+
) from exc
|
|
1187
|
+
try:
|
|
1188
|
+
require_secret_values_absent(
|
|
1189
|
+
raw,
|
|
1190
|
+
secret_values=secret_values,
|
|
1191
|
+
label="parity-process worker response",
|
|
1192
|
+
)
|
|
1193
|
+
_reject_forbidden_worker_keys(raw)
|
|
1194
|
+
data = _strict_mapping(
|
|
1195
|
+
raw,
|
|
1196
|
+
_RESPONSE_FIELDS,
|
|
1197
|
+
label="parity-process response",
|
|
1198
|
+
)
|
|
1199
|
+
if data["schema_version"] != 1 or data["protocol"] != RESPONSE_PROTOCOL:
|
|
1200
|
+
raise ValidationError("unsupported parity-process worker response")
|
|
1201
|
+
if data["request_hash"] != expected_request_hash:
|
|
1202
|
+
raise ValidationError("parity-process response belongs to another request")
|
|
1203
|
+
status = data["status"]
|
|
1204
|
+
if status not in {"succeeded", "failed"}:
|
|
1205
|
+
raise ValidationError("parity-process response status is invalid")
|
|
1206
|
+
metrics = _metrics(data["metrics"])
|
|
1207
|
+
usage_data = _strict_mapping(
|
|
1208
|
+
data["usage"],
|
|
1209
|
+
_USAGE_FIELDS,
|
|
1210
|
+
label="parity-process usage",
|
|
1211
|
+
)
|
|
1212
|
+
usage = ResourceUsage(**usage_data).as_dict()
|
|
1213
|
+
usage.pop("total_cost_usd")
|
|
1214
|
+
output = _canonical_mapping(data["output"], label="parity-process output")
|
|
1215
|
+
failure = data["failure"]
|
|
1216
|
+
if status == "succeeded":
|
|
1217
|
+
if not metrics:
|
|
1218
|
+
raise ValidationError("successful parity-process response requires metrics")
|
|
1219
|
+
if failure is not None:
|
|
1220
|
+
raise ValidationError("successful parity-process response cannot have failure")
|
|
1221
|
+
else:
|
|
1222
|
+
if metrics or output:
|
|
1223
|
+
raise ValidationError("failed parity-process response cannot have metrics/output")
|
|
1224
|
+
failure = _strict_mapping(
|
|
1225
|
+
failure,
|
|
1226
|
+
frozenset({"scope", "category", "message"}),
|
|
1227
|
+
label="parity-process failure",
|
|
1228
|
+
)
|
|
1229
|
+
if failure["scope"] != "algorithm":
|
|
1230
|
+
raise ValidationError("parity-process arm failure must be algorithm-scoped")
|
|
1231
|
+
_nonempty_text(failure["category"], "failure.category")
|
|
1232
|
+
_nonempty_text(failure["message"], "failure.message")
|
|
1233
|
+
normalized = {
|
|
1234
|
+
"schema_version": 1,
|
|
1235
|
+
"protocol": RESPONSE_PROTOCOL,
|
|
1236
|
+
"request_hash": expected_request_hash,
|
|
1237
|
+
"status": status,
|
|
1238
|
+
"metrics": metrics,
|
|
1239
|
+
"usage": usage,
|
|
1240
|
+
"output": output,
|
|
1241
|
+
"failure": failure,
|
|
1242
|
+
}
|
|
1243
|
+
require_secret_free(normalized, label="parity-process worker response")
|
|
1244
|
+
return freeze_mapping(normalized)
|
|
1245
|
+
except (ArtifactError, TypeError, ValueError, ValidationError) as exc:
|
|
1246
|
+
raise ParityProcessProtocolError(
|
|
1247
|
+
redact_secret_text(str(exc), secret_values=secret_values),
|
|
1248
|
+
resource_receipt=process.resource_receipt,
|
|
1249
|
+
) from exc
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
def _host_execution_receipt(
|
|
1253
|
+
*,
|
|
1254
|
+
config_sha256: str,
|
|
1255
|
+
manifest: ExperimentManifest,
|
|
1256
|
+
cell: ExperimentCell,
|
|
1257
|
+
worker: _WorkerSpec,
|
|
1258
|
+
request: Mapping[str, Any],
|
|
1259
|
+
process: SupervisedProcessResult,
|
|
1260
|
+
checkout_binding_hash: str,
|
|
1261
|
+
adapter_checkout_binding_hash: str,
|
|
1262
|
+
) -> Mapping[str, Any]:
|
|
1263
|
+
receipt_without_hash = {
|
|
1264
|
+
"schema_version": 1,
|
|
1265
|
+
"protocol": HOST_EXECUTION_RECEIPT_PROTOCOL,
|
|
1266
|
+
"arm_id": worker.arm_id,
|
|
1267
|
+
"manifest_hash": manifest.manifest_hash,
|
|
1268
|
+
"cell_id": cell.cell_id,
|
|
1269
|
+
"request_hash": request["request_hash"],
|
|
1270
|
+
"config_sha256": config_sha256,
|
|
1271
|
+
"checkout_environment": worker.checkout_environment,
|
|
1272
|
+
"checkout_binding_hash": checkout_binding_hash,
|
|
1273
|
+
"adapter_checkout_environment": worker.adapter_checkout_environment,
|
|
1274
|
+
"adapter_checkout_binding_hash": adapter_checkout_binding_hash,
|
|
1275
|
+
"checkout": {
|
|
1276
|
+
**worker.checkout_identity.to_dict(),
|
|
1277
|
+
"git_clean_before": True,
|
|
1278
|
+
"git_clean_after": True,
|
|
1279
|
+
},
|
|
1280
|
+
"adapter_checkout": {
|
|
1281
|
+
**worker.adapter_checkout_identity.to_dict(),
|
|
1282
|
+
"git_clean_before": True,
|
|
1283
|
+
"git_clean_after": True,
|
|
1284
|
+
},
|
|
1285
|
+
"command": worker.command_identity.to_dict(),
|
|
1286
|
+
"process": {
|
|
1287
|
+
"resource_receipt_sha256": content_hash(process.resource_receipt.as_dict()),
|
|
1288
|
+
"stdout_sha256": _sha256_text(process.stdout),
|
|
1289
|
+
"stderr_sha256": _sha256_text(process.stderr),
|
|
1290
|
+
},
|
|
1291
|
+
}
|
|
1292
|
+
return freeze_mapping(
|
|
1293
|
+
{
|
|
1294
|
+
**receipt_without_hash,
|
|
1295
|
+
"receipt_hash": content_hash(receipt_without_hash),
|
|
1296
|
+
}
|
|
1297
|
+
)
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
def _strict_mapping(
|
|
1301
|
+
value: Any,
|
|
1302
|
+
fields: frozenset[str],
|
|
1303
|
+
*,
|
|
1304
|
+
label: str,
|
|
1305
|
+
) -> dict[str, Any]:
|
|
1306
|
+
if not isinstance(value, Mapping) or set(value) != set(fields):
|
|
1307
|
+
raise ValidationError(f"{label} fields are invalid")
|
|
1308
|
+
return dict(value)
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def _load_fixture_inventory(
|
|
1312
|
+
environment: Mapping[str, str],
|
|
1313
|
+
) -> tuple[str, ...]:
|
|
1314
|
+
raw_path = environment.get(UNIT_INVENTORY_ENV)
|
|
1315
|
+
expected_sha256 = environment.get(UNIT_INVENTORY_SHA256_ENV)
|
|
1316
|
+
if not raw_path:
|
|
1317
|
+
raise ArtifactError(f"{UNIT_INVENTORY_ENV} is required")
|
|
1318
|
+
if not expected_sha256:
|
|
1319
|
+
raise ArtifactError(f"{UNIT_INVENTORY_SHA256_ENV} is required")
|
|
1320
|
+
require_sha256(expected_sha256, label=UNIT_INVENTORY_SHA256_ENV)
|
|
1321
|
+
path = Path(raw_path).expanduser().resolve()
|
|
1322
|
+
if not path.is_file():
|
|
1323
|
+
raise ArtifactError("parity-process fixture inventory is unavailable")
|
|
1324
|
+
if _sha256_file(path) != expected_sha256:
|
|
1325
|
+
raise ArtifactError("parity-process fixture inventory checksum differs")
|
|
1326
|
+
try:
|
|
1327
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
1328
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
1329
|
+
raise ArtifactError("parity-process fixture inventory is not valid UTF-8 JSON") from exc
|
|
1330
|
+
if (
|
|
1331
|
+
isinstance(raw, (str, bytes))
|
|
1332
|
+
or not isinstance(raw, Sequence)
|
|
1333
|
+
or not raw
|
|
1334
|
+
or any(not isinstance(item, str) or not item.strip() for item in raw)
|
|
1335
|
+
or len(raw) != len(set(raw))
|
|
1336
|
+
):
|
|
1337
|
+
raise ValidationError(
|
|
1338
|
+
"parity-process fixture inventory must be a non-empty unique string array"
|
|
1339
|
+
)
|
|
1340
|
+
return tuple(raw)
|
|
1341
|
+
|
|
1342
|
+
|
|
1343
|
+
def _canonical_mapping(value: Any, *, label: str) -> Mapping[str, Any]:
|
|
1344
|
+
if not isinstance(value, Mapping):
|
|
1345
|
+
raise ValidationError(f"{label} must be an object")
|
|
1346
|
+
try:
|
|
1347
|
+
return freeze_mapping(value)
|
|
1348
|
+
except (TypeError, ValueError) as exc:
|
|
1349
|
+
raise ValidationError(f"{label} must be canonical JSON") from exc
|
|
1350
|
+
|
|
1351
|
+
|
|
1352
|
+
def _metrics(value: Any) -> Mapping[str, float]:
|
|
1353
|
+
if not isinstance(value, Mapping):
|
|
1354
|
+
raise ValidationError("parity-process metrics must be an object")
|
|
1355
|
+
result: dict[str, float] = {}
|
|
1356
|
+
for name, raw in value.items():
|
|
1357
|
+
if (
|
|
1358
|
+
not isinstance(name, str)
|
|
1359
|
+
or not name
|
|
1360
|
+
or isinstance(raw, bool)
|
|
1361
|
+
or not isinstance(raw, (int, float))
|
|
1362
|
+
):
|
|
1363
|
+
raise ValidationError("parity-process metrics must be named numerics")
|
|
1364
|
+
numeric = float(raw)
|
|
1365
|
+
if not -float("inf") < numeric < float("inf"):
|
|
1366
|
+
raise ValidationError("parity-process metrics must be finite")
|
|
1367
|
+
result[name] = numeric
|
|
1368
|
+
return freeze_mapping(result)
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
def _reject_forbidden_worker_keys(value: Any) -> None:
|
|
1372
|
+
if isinstance(value, Mapping):
|
|
1373
|
+
forbidden = sorted(set(value) & _RESPONSE_FORBIDDEN_KEYS)
|
|
1374
|
+
if forbidden:
|
|
1375
|
+
raise ValidationError(
|
|
1376
|
+
"worker may not emit host-owned identity fields: " + ", ".join(forbidden)
|
|
1377
|
+
)
|
|
1378
|
+
for item in value.values():
|
|
1379
|
+
_reject_forbidden_worker_keys(item)
|
|
1380
|
+
elif isinstance(value, Sequence) and not isinstance(
|
|
1381
|
+
value,
|
|
1382
|
+
(str, bytes, bytearray),
|
|
1383
|
+
):
|
|
1384
|
+
for item in value:
|
|
1385
|
+
_reject_forbidden_worker_keys(item)
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
def _relative_entrypoint(value: Any, *, label: str) -> str:
|
|
1389
|
+
text = _nonempty_text(value, label)
|
|
1390
|
+
path = PurePosixPath(text)
|
|
1391
|
+
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
|
|
1392
|
+
raise ValidationError(f"{label} must be a normalized relative POSIX path")
|
|
1393
|
+
return path.as_posix()
|
|
1394
|
+
|
|
1395
|
+
|
|
1396
|
+
def _environment_names(value: Any, *, label: str) -> tuple[str, ...]:
|
|
1397
|
+
names = _string_sequence(value, label=label)
|
|
1398
|
+
if len(names) != len(set(names)):
|
|
1399
|
+
raise ValidationError(f"{label} cannot contain duplicates")
|
|
1400
|
+
if any(_ENVIRONMENT_NAME.fullmatch(name) is None for name in names):
|
|
1401
|
+
raise ValidationError(f"{label} contains an invalid environment name")
|
|
1402
|
+
return names
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
def _string_sequence(value: Any, *, label: str) -> tuple[str, ...]:
|
|
1406
|
+
if (
|
|
1407
|
+
isinstance(value, (str, bytes))
|
|
1408
|
+
or not isinstance(value, Sequence)
|
|
1409
|
+
or any(not isinstance(item, str) or "\x00" in item for item in value)
|
|
1410
|
+
):
|
|
1411
|
+
raise ValidationError(f"{label} must be a string array")
|
|
1412
|
+
return tuple(value)
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
def _nonempty_text(value: Any, label: str) -> str:
|
|
1416
|
+
if not isinstance(value, str) or not value.strip() or "\x00" in value:
|
|
1417
|
+
raise ValidationError(f"{label} must be a non-empty string")
|
|
1418
|
+
return value
|
|
1419
|
+
|
|
1420
|
+
|
|
1421
|
+
def _positive_integer(value: Any, label: str) -> int:
|
|
1422
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|
1423
|
+
raise ValidationError(f"{label} must be a positive integer")
|
|
1424
|
+
return value
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def _sha256_text_field(value: Any, label: str) -> str:
|
|
1428
|
+
if not isinstance(value, str):
|
|
1429
|
+
raise ValidationError(f"{label} must be a SHA-256 digest")
|
|
1430
|
+
require_sha256(value, label=label)
|
|
1431
|
+
return value
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
def _git_object_id(value: Any, label: str) -> str:
|
|
1435
|
+
if not isinstance(value, str) or _GIT_OBJECT_ID.fullmatch(value) is None:
|
|
1436
|
+
raise ValidationError(f"{label} must be a full lowercase Git object ID")
|
|
1437
|
+
return value
|
|
1438
|
+
|
|
1439
|
+
|
|
1440
|
+
def _git(path: Path, *arguments: str) -> str:
|
|
1441
|
+
try:
|
|
1442
|
+
completed = subprocess.run(
|
|
1443
|
+
("git", "-C", str(path), *arguments),
|
|
1444
|
+
check=True,
|
|
1445
|
+
capture_output=True,
|
|
1446
|
+
text=True,
|
|
1447
|
+
timeout=30,
|
|
1448
|
+
)
|
|
1449
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
1450
|
+
raise ArtifactError("cannot independently attest parity-process Git checkout") from exc
|
|
1451
|
+
return completed.stdout.strip()
|
|
1452
|
+
|
|
1453
|
+
|
|
1454
|
+
def _sha256_file(path: Path) -> str:
|
|
1455
|
+
digest = hashlib.sha256()
|
|
1456
|
+
with path.open("rb") as handle:
|
|
1457
|
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
1458
|
+
digest.update(block)
|
|
1459
|
+
return digest.hexdigest()
|
|
1460
|
+
|
|
1461
|
+
|
|
1462
|
+
def _sha256_text(value: str) -> str:
|
|
1463
|
+
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
__all__ = [
|
|
1467
|
+
"CONFIG_PATH_ENV",
|
|
1468
|
+
"CONFIG_SHA256_ENV",
|
|
1469
|
+
"DRIVER_PROTOCOL",
|
|
1470
|
+
"REQUEST_PROTOCOL",
|
|
1471
|
+
"RESPONSE_PROTOCOL",
|
|
1472
|
+
"UNIT_INVENTORY_ENV",
|
|
1473
|
+
"UNIT_INVENTORY_SHA256_ENV",
|
|
1474
|
+
"LockedProcessWorker",
|
|
1475
|
+
"ParityProcessExperimentDriver",
|
|
1476
|
+
"ParityProcessProtocolError",
|
|
1477
|
+
"attest_locked_process_worker",
|
|
1478
|
+
"build_host_execution_receipt",
|
|
1479
|
+
"execute_locked_process_worker",
|
|
1480
|
+
"factory",
|
|
1481
|
+
"load_locked_process_worker",
|
|
1482
|
+
"locked_process_worker_environment",
|
|
1483
|
+
"parse_locked_process_response",
|
|
1484
|
+
]
|