agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,2908 @@
|
|
|
1
|
+
"""Checksum-locked multi-arm driver for stateful external arenas.
|
|
2
|
+
|
|
3
|
+
The public atomic replay contract is intentionally pairwise: one external
|
|
4
|
+
runtime operation restores the same protected origin for a baseline and one
|
|
5
|
+
treatment. This driver lifts that contract to a frozen multi-arm experiment
|
|
6
|
+
cell by constructing every payload before execution and then running one
|
|
7
|
+
atomic pair per treatment. Every pair is bound to the same checkpoint,
|
|
8
|
+
origin, replay identity, seed, and dependency set. The repeated NO_OP result
|
|
9
|
+
must also have identical official metrics, trace, usage, and logical receipt
|
|
10
|
+
fields or the whole cell aborts without a terminal result.
|
|
11
|
+
|
|
12
|
+
No upstream runtime, model, selector, or intervention is selected here.
|
|
13
|
+
Operators provide checksum-locked zero-argument builders in one strict JSON
|
|
14
|
+
configuration. Loading and preflight never download assets or execute a
|
|
15
|
+
benchmark cell.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
import json
|
|
22
|
+
import math
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
from collections.abc import Mapping, Sequence
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from types import MappingProxyType
|
|
29
|
+
from typing import Any, Protocol, runtime_checkable
|
|
30
|
+
|
|
31
|
+
from agentinputlab.benchmarks import (
|
|
32
|
+
ATOMIC_EXTERNAL_PAIR_CONTRACT,
|
|
33
|
+
AtomicExternalReplayBridge,
|
|
34
|
+
AtomicExternalReplayRunner,
|
|
35
|
+
AtomicPairedReplayRecord,
|
|
36
|
+
AtomicReplayArmRequest,
|
|
37
|
+
AtomicReplayRequest,
|
|
38
|
+
BenchmarkRegistry,
|
|
39
|
+
interactive_benchmark_specs,
|
|
40
|
+
)
|
|
41
|
+
from agentinputlab.contracts.enums import ProfileKind, SemanticFamily
|
|
42
|
+
from agentinputlab.contracts.errors import ArtifactError, ValidationError
|
|
43
|
+
from agentinputlab.contracts.serialization import (
|
|
44
|
+
content_hash,
|
|
45
|
+
deep_freeze_jsonable,
|
|
46
|
+
freeze_mapping,
|
|
47
|
+
to_jsonable,
|
|
48
|
+
)
|
|
49
|
+
from agentinputlab.contracts.state import FrozenAgentState
|
|
50
|
+
from agentinputlab.experiments.executor import ExperimentCellResult
|
|
51
|
+
from agentinputlab.experiments.loader import resolve_driver_entrypoint
|
|
52
|
+
from agentinputlab.experiments.plan import ExperimentCell
|
|
53
|
+
from agentinputlab.experiments.schema import (
|
|
54
|
+
ExperimentArmKind,
|
|
55
|
+
ExperimentArmSpec,
|
|
56
|
+
ExperimentBenchmarkTarget,
|
|
57
|
+
ExperimentManifest,
|
|
58
|
+
PrerequisiteKind,
|
|
59
|
+
)
|
|
60
|
+
from agentinputlab.selectors import (
|
|
61
|
+
SelectorArtifactBundle,
|
|
62
|
+
SelectorArtifactStage,
|
|
63
|
+
VisibleStateFeatureExtractor,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
CONFIG_PATH_ENV = "AILAB_EXTERNAL_ARENA_DRIVER_CONFIG"
|
|
67
|
+
CONFIG_SHA256_ENV = "AILAB_EXTERNAL_ARENA_DRIVER_CONFIG_SHA256"
|
|
68
|
+
|
|
69
|
+
_CONFIG_SCHEMA_VERSION = 3
|
|
70
|
+
_DRIVER_IMPLEMENTATION_ID = "agentinputlab.external_arena.atomic_multi_arm"
|
|
71
|
+
_VALID_SUCCESS_FORMULA_ID = "agentinputlab.valid_success.v1"
|
|
72
|
+
_CONTRACT_PRESERVED_METRIC = "contract_preserved"
|
|
73
|
+
_PRIVILEGED_LEAKAGE_METRIC = "privileged_leakage"
|
|
74
|
+
_VALID_SUCCESS_METRIC = "valid_success"
|
|
75
|
+
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
76
|
+
_ANONYMIZED_GROUP = re.compile(r"^group_[0-9a-f]{64}$")
|
|
77
|
+
_ENTRYPOINT = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*:[A-Za-z_][A-Za-z0-9_.]*$")
|
|
78
|
+
_ENVIRONMENT_NAME = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
|
79
|
+
_MAX_CONFIG_BYTES = 4 * 1024 * 1024
|
|
80
|
+
_MAX_POLICY_ARTIFACT_BYTES = 32 * 1024 * 1024
|
|
81
|
+
_BUILDER_KINDS = frozenset({"arena_checkpoint", "atomic_bridge", "arm_payload"})
|
|
82
|
+
_CORE_FAMILIES = frozenset(
|
|
83
|
+
{
|
|
84
|
+
SemanticFamily.REFORMULATE,
|
|
85
|
+
SemanticFamily.DECOMPOSE,
|
|
86
|
+
SemanticFamily.ACQUIRE,
|
|
87
|
+
SemanticFamily.CLARIFY,
|
|
88
|
+
SemanticFamily.CURATE,
|
|
89
|
+
}
|
|
90
|
+
)
|
|
91
|
+
_EXECUTABLE_RELEASE_FAMILIES = frozenset(
|
|
92
|
+
{
|
|
93
|
+
*_CORE_FAMILIES,
|
|
94
|
+
SemanticFamily.COMPOSITE,
|
|
95
|
+
}
|
|
96
|
+
)
|
|
97
|
+
_SUPPORTED_ARENAS = frozenset(
|
|
98
|
+
{
|
|
99
|
+
"tau3.banking_knowledge",
|
|
100
|
+
"tau3.retail",
|
|
101
|
+
"state_bench.common_support",
|
|
102
|
+
"toolsandbox.common_support",
|
|
103
|
+
"appworld_ul.common_support",
|
|
104
|
+
"webarena.external",
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
_REGISTERED_OFFICIAL_SUCCESS_METRICS = {
|
|
108
|
+
"tau3.banking_knowledge": "official_reward",
|
|
109
|
+
# Native RewardInfo is retained separately. A completed native evaluator
|
|
110
|
+
# must not turn an episode timeout or shared-budget stop into paper success.
|
|
111
|
+
"tau3.retail": "terminal_success",
|
|
112
|
+
"state_bench.common_support": "task_completion_pass",
|
|
113
|
+
"toolsandbox.common_support": "official_similarity",
|
|
114
|
+
"appworld_ul.common_support": "task_success",
|
|
115
|
+
}
|
|
116
|
+
_POLICY_KIND_BY_RELEASE = {
|
|
117
|
+
"policy.locked_best_eligible_global": "locked_best_eligible_global",
|
|
118
|
+
"policy.eligibility_aware_selector": "frozen_selector",
|
|
119
|
+
}
|
|
120
|
+
_REQUIRED_SHARED_DEPENDENCY_SLOTS = frozenset(
|
|
121
|
+
{
|
|
122
|
+
"agent_model",
|
|
123
|
+
"official_runtime",
|
|
124
|
+
"corpus",
|
|
125
|
+
"preprocessing",
|
|
126
|
+
"index",
|
|
127
|
+
"embedder",
|
|
128
|
+
"retriever",
|
|
129
|
+
"reranker",
|
|
130
|
+
"reader",
|
|
131
|
+
"scorer",
|
|
132
|
+
}
|
|
133
|
+
)
|
|
134
|
+
_RETAIL_ENVIRONMENT_DEPENDENCY_SLOTS = frozenset({"user_simulator", "nl_assertion_judge"})
|
|
135
|
+
_NOT_APPLICABLE_DEPENDENCY_HASHES = MappingProxyType(
|
|
136
|
+
{
|
|
137
|
+
slot: content_hash(
|
|
138
|
+
{
|
|
139
|
+
"agentinputlab_dependency": "not_applicable",
|
|
140
|
+
"schema_version": 1,
|
|
141
|
+
"slot": slot,
|
|
142
|
+
}
|
|
143
|
+
)
|
|
144
|
+
for slot in _REQUIRED_SHARED_DEPENDENCY_SLOTS
|
|
145
|
+
}
|
|
146
|
+
)
|
|
147
|
+
_GATE_BINDINGS = {
|
|
148
|
+
"toolsandbox.common_support": (
|
|
149
|
+
"AILAB_TOOLSANDBOX_GATE_RECEIPT",
|
|
150
|
+
"AILAB_TOOLSANDBOX_GATE_RECEIPT_SHA256",
|
|
151
|
+
"atomic_restore",
|
|
152
|
+
),
|
|
153
|
+
"appworld_ul.common_support": (
|
|
154
|
+
"AILAB_APPWORLD_UL_GATE_RECEIPT",
|
|
155
|
+
"AILAB_APPWORLD_UL_GATE_RECEIPT_SHA256",
|
|
156
|
+
"atomic_restore",
|
|
157
|
+
),
|
|
158
|
+
"webarena.external": (
|
|
159
|
+
"AILAB_WEBARENA_STACK_RECEIPT",
|
|
160
|
+
"AILAB_WEBARENA_STACK_RECEIPT_SHA256",
|
|
161
|
+
"independent_stack_or_verified_reset",
|
|
162
|
+
),
|
|
163
|
+
}
|
|
164
|
+
_SECRET_TEXT_PATTERNS = (
|
|
165
|
+
re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{8,}"),
|
|
166
|
+
re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"),
|
|
167
|
+
re.compile(r"\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b"),
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@dataclass(frozen=True, slots=True)
|
|
172
|
+
class ExternalArenaCheckpoint:
|
|
173
|
+
"""Operator-captured visible state plus opaque protected-runtime identities."""
|
|
174
|
+
|
|
175
|
+
benchmark_id: str
|
|
176
|
+
checkpoint_id: str
|
|
177
|
+
group_id: str
|
|
178
|
+
source_revision: str
|
|
179
|
+
origin_state_hash: str
|
|
180
|
+
replay_identity: str
|
|
181
|
+
locked_dependency_hashes: Mapping[str, str]
|
|
182
|
+
visible_state: FrozenAgentState
|
|
183
|
+
eligible_families: tuple[SemanticFamily, ...]
|
|
184
|
+
|
|
185
|
+
def __post_init__(self) -> None:
|
|
186
|
+
for label in ("benchmark_id", "checkpoint_id", "group_id", "source_revision"):
|
|
187
|
+
_require_text(getattr(self, label), label=f"external checkpoint {label}")
|
|
188
|
+
_require_sha256(
|
|
189
|
+
self.origin_state_hash,
|
|
190
|
+
label="external checkpoint origin_state_hash",
|
|
191
|
+
)
|
|
192
|
+
_require_sha256(
|
|
193
|
+
self.replay_identity,
|
|
194
|
+
label="external checkpoint replay_identity",
|
|
195
|
+
)
|
|
196
|
+
if not isinstance(self.visible_state, FrozenAgentState):
|
|
197
|
+
raise TypeError("external checkpoint visible_state must be FrozenAgentState")
|
|
198
|
+
families = tuple(self.eligible_families)
|
|
199
|
+
if len(families) != len(set(families)) or any(
|
|
200
|
+
family not in _EXECUTABLE_RELEASE_FAMILIES for family in families
|
|
201
|
+
):
|
|
202
|
+
raise ValidationError(
|
|
203
|
+
"external checkpoint eligible_families must be unique executable families"
|
|
204
|
+
)
|
|
205
|
+
object.__setattr__(
|
|
206
|
+
self,
|
|
207
|
+
"locked_dependency_hashes",
|
|
208
|
+
_validated_dependency_hashes(self.locked_dependency_hashes, benchmark_id=self.benchmark_id),
|
|
209
|
+
)
|
|
210
|
+
object.__setattr__(self, "eligible_families", families)
|
|
211
|
+
|
|
212
|
+
@property
|
|
213
|
+
def identity_hash(self) -> str:
|
|
214
|
+
return content_hash(
|
|
215
|
+
{
|
|
216
|
+
"benchmark_id": self.benchmark_id,
|
|
217
|
+
"checkpoint_id": self.checkpoint_id,
|
|
218
|
+
"group_id": self.group_id,
|
|
219
|
+
"source_revision": self.source_revision,
|
|
220
|
+
"origin_state_hash": self.origin_state_hash,
|
|
221
|
+
"replay_identity": self.replay_identity,
|
|
222
|
+
"locked_dependency_hashes": self.locked_dependency_hashes,
|
|
223
|
+
"visible_state_hash": self.visible_state.state_hash,
|
|
224
|
+
"eligible_families": self.eligible_families,
|
|
225
|
+
}
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@dataclass(frozen=True, slots=True)
|
|
230
|
+
class ExternalArmPayload:
|
|
231
|
+
"""One typed, canonically serializable artifact sent to the external runtime."""
|
|
232
|
+
|
|
233
|
+
arm_id: str
|
|
234
|
+
release_id: str
|
|
235
|
+
family: SemanticFamily
|
|
236
|
+
profile_kind: ProfileKind
|
|
237
|
+
payload: Any = field(repr=False)
|
|
238
|
+
selector_decision: Mapping[str, Any] | None = None
|
|
239
|
+
selector_input_state_hash: str | None = None
|
|
240
|
+
selector_feature_hash: str | None = None
|
|
241
|
+
|
|
242
|
+
def __post_init__(self) -> None:
|
|
243
|
+
_require_text(self.arm_id, label="external arm payload arm_id")
|
|
244
|
+
_require_text(self.release_id, label="external arm payload release_id")
|
|
245
|
+
if not isinstance(self.family, SemanticFamily):
|
|
246
|
+
raise ValidationError("external arm payload family must be typed")
|
|
247
|
+
if not isinstance(self.profile_kind, ProfileKind):
|
|
248
|
+
raise ValidationError("external arm payload profile_kind must be typed")
|
|
249
|
+
try:
|
|
250
|
+
payload = deep_freeze_jsonable(self.payload)
|
|
251
|
+
content_hash(payload)
|
|
252
|
+
except (TypeError, ValueError) as exc:
|
|
253
|
+
raise ValidationError(f"external arm payload must be canonical JSON: {exc}") from exc
|
|
254
|
+
if payload is None:
|
|
255
|
+
raise ValidationError("external arm payload cannot be null")
|
|
256
|
+
decision = self.selector_decision
|
|
257
|
+
if decision is not None:
|
|
258
|
+
decision = _validated_policy_decision(decision)
|
|
259
|
+
selector_hashes = (
|
|
260
|
+
self.selector_input_state_hash,
|
|
261
|
+
self.selector_feature_hash,
|
|
262
|
+
)
|
|
263
|
+
if all(value is None for value in selector_hashes):
|
|
264
|
+
pass
|
|
265
|
+
elif any(value is None for value in selector_hashes):
|
|
266
|
+
raise ValidationError(
|
|
267
|
+
"selector input state_hash and feature_hash must be supplied together"
|
|
268
|
+
)
|
|
269
|
+
else:
|
|
270
|
+
_require_sha256(
|
|
271
|
+
self.selector_input_state_hash,
|
|
272
|
+
label="selector input state_hash",
|
|
273
|
+
)
|
|
274
|
+
_require_sha256(
|
|
275
|
+
self.selector_feature_hash,
|
|
276
|
+
label="selector input feature_hash",
|
|
277
|
+
)
|
|
278
|
+
object.__setattr__(self, "payload", payload)
|
|
279
|
+
object.__setattr__(self, "selector_decision", decision)
|
|
280
|
+
|
|
281
|
+
@property
|
|
282
|
+
def payload_sha256(self) -> str:
|
|
283
|
+
return content_hash(
|
|
284
|
+
{
|
|
285
|
+
"arm_id": self.arm_id,
|
|
286
|
+
"release_id": self.release_id,
|
|
287
|
+
"family": self.family,
|
|
288
|
+
"profile_kind": self.profile_kind,
|
|
289
|
+
"payload": self.payload,
|
|
290
|
+
"selector_decision": self.selector_decision,
|
|
291
|
+
"selector_input_state_hash": self.selector_input_state_hash,
|
|
292
|
+
"selector_feature_hash": self.selector_feature_hash,
|
|
293
|
+
}
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
@dataclass(frozen=True, slots=True)
|
|
298
|
+
class ExternalCheckpointBuildRequest:
|
|
299
|
+
benchmark_id: str
|
|
300
|
+
split: str
|
|
301
|
+
checkpoint_id: str
|
|
302
|
+
source_revision: str
|
|
303
|
+
model_slot: str
|
|
304
|
+
seed: int
|
|
305
|
+
builder_parameters: Mapping[str, Any]
|
|
306
|
+
|
|
307
|
+
def __post_init__(self) -> None:
|
|
308
|
+
for label in (
|
|
309
|
+
"benchmark_id",
|
|
310
|
+
"split",
|
|
311
|
+
"checkpoint_id",
|
|
312
|
+
"source_revision",
|
|
313
|
+
"model_slot",
|
|
314
|
+
):
|
|
315
|
+
_require_text(getattr(self, label), label=f"checkpoint build {label}")
|
|
316
|
+
_require_seed(self.seed)
|
|
317
|
+
object.__setattr__(
|
|
318
|
+
self,
|
|
319
|
+
"builder_parameters",
|
|
320
|
+
freeze_mapping(self.builder_parameters),
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
@dataclass(frozen=True, slots=True)
|
|
325
|
+
class ExternalUnitPreflightRequest:
|
|
326
|
+
"""Exact resolved unit inventory presented to a read-only checkpoint builder."""
|
|
327
|
+
|
|
328
|
+
benchmark_id: str
|
|
329
|
+
split: str
|
|
330
|
+
source_revision: str
|
|
331
|
+
unit_ids: tuple[str, ...]
|
|
332
|
+
builder_parameters: Mapping[str, Any]
|
|
333
|
+
|
|
334
|
+
def __post_init__(self) -> None:
|
|
335
|
+
for label in ("benchmark_id", "split", "source_revision"):
|
|
336
|
+
_require_text(
|
|
337
|
+
getattr(self, label),
|
|
338
|
+
label=f"unit preflight {label}",
|
|
339
|
+
)
|
|
340
|
+
unit_ids = tuple(self.unit_ids)
|
|
341
|
+
if (
|
|
342
|
+
not unit_ids
|
|
343
|
+
or len(unit_ids) != len(set(unit_ids))
|
|
344
|
+
or any(
|
|
345
|
+
not isinstance(item, str) or not item.strip() or item != item.strip()
|
|
346
|
+
for item in unit_ids
|
|
347
|
+
)
|
|
348
|
+
):
|
|
349
|
+
raise ValidationError("unit preflight requires unique trimmed non-empty unit IDs")
|
|
350
|
+
object.__setattr__(self, "unit_ids", unit_ids)
|
|
351
|
+
object.__setattr__(
|
|
352
|
+
self,
|
|
353
|
+
"builder_parameters",
|
|
354
|
+
freeze_mapping(self.builder_parameters),
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
@dataclass(frozen=True, slots=True)
|
|
359
|
+
class ExternalBridgeBuildRequest:
|
|
360
|
+
checkpoint: ExternalArenaCheckpoint
|
|
361
|
+
model_slot: str
|
|
362
|
+
seed: int
|
|
363
|
+
builder_parameters: Mapping[str, Any]
|
|
364
|
+
|
|
365
|
+
def __post_init__(self) -> None:
|
|
366
|
+
if not isinstance(self.checkpoint, ExternalArenaCheckpoint):
|
|
367
|
+
raise TypeError("bridge build request requires ExternalArenaCheckpoint")
|
|
368
|
+
_require_text(self.model_slot, label="bridge build model_slot")
|
|
369
|
+
_require_seed(self.seed)
|
|
370
|
+
object.__setattr__(
|
|
371
|
+
self,
|
|
372
|
+
"builder_parameters",
|
|
373
|
+
freeze_mapping(self.builder_parameters),
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
@dataclass(frozen=True, slots=True)
|
|
378
|
+
class ExternalArmPayloadBuildRequest:
|
|
379
|
+
checkpoint: ExternalArenaCheckpoint
|
|
380
|
+
arm_id: str
|
|
381
|
+
release_id: str
|
|
382
|
+
family: SemanticFamily
|
|
383
|
+
profile_kind: ProfileKind
|
|
384
|
+
arm_kind: ExperimentArmKind
|
|
385
|
+
model_slot: str
|
|
386
|
+
seed: int
|
|
387
|
+
builder_parameters: Mapping[str, Any]
|
|
388
|
+
arm_parameters: Mapping[str, Any]
|
|
389
|
+
pre_action: Mapping[str, Any]
|
|
390
|
+
|
|
391
|
+
def __post_init__(self) -> None:
|
|
392
|
+
if not isinstance(self.checkpoint, ExternalArenaCheckpoint):
|
|
393
|
+
raise TypeError("payload build request requires ExternalArenaCheckpoint")
|
|
394
|
+
for label in ("arm_id", "release_id", "model_slot"):
|
|
395
|
+
_require_text(getattr(self, label), label=f"payload build {label}")
|
|
396
|
+
if not isinstance(self.family, SemanticFamily):
|
|
397
|
+
raise ValidationError("payload build family must be typed")
|
|
398
|
+
if not isinstance(self.profile_kind, ProfileKind):
|
|
399
|
+
raise ValidationError("payload build profile_kind must be typed")
|
|
400
|
+
if not isinstance(self.arm_kind, ExperimentArmKind):
|
|
401
|
+
raise ValidationError("payload build arm_kind must be typed")
|
|
402
|
+
_require_seed(self.seed)
|
|
403
|
+
object.__setattr__(
|
|
404
|
+
self,
|
|
405
|
+
"builder_parameters",
|
|
406
|
+
freeze_mapping(self.builder_parameters),
|
|
407
|
+
)
|
|
408
|
+
object.__setattr__(self, "arm_parameters", freeze_mapping(self.arm_parameters))
|
|
409
|
+
frozen_pre_action = freeze_mapping(self.pre_action)
|
|
410
|
+
for key in ("state_hash", "feature_hash"):
|
|
411
|
+
_require_sha256(
|
|
412
|
+
frozen_pre_action.get(key),
|
|
413
|
+
label=f"payload build pre_action {key}",
|
|
414
|
+
)
|
|
415
|
+
object.__setattr__(self, "pre_action", frozen_pre_action)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
@runtime_checkable
|
|
419
|
+
class LockedExternalArenaBuilder(Protocol):
|
|
420
|
+
"""Shared contract for operator-owned, checksum-locked component builders."""
|
|
421
|
+
|
|
422
|
+
builder_kind: str
|
|
423
|
+
|
|
424
|
+
def fingerprint(self) -> Mapping[str, Any]: ...
|
|
425
|
+
|
|
426
|
+
def preflight(self, parameters: Mapping[str, Any]) -> Sequence[str]: ...
|
|
427
|
+
|
|
428
|
+
def build(self, request: Any) -> Any: ...
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
@dataclass(frozen=True, slots=True)
|
|
432
|
+
class _LockedBuilderSpec:
|
|
433
|
+
entrypoint: str
|
|
434
|
+
fingerprint_sha256: str
|
|
435
|
+
parameters: Mapping[str, Any] = field(default_factory=dict)
|
|
436
|
+
|
|
437
|
+
def __post_init__(self) -> None:
|
|
438
|
+
if not isinstance(self.entrypoint, str) or _ENTRYPOINT.fullmatch(self.entrypoint) is None:
|
|
439
|
+
raise ValidationError("external builder entrypoint must use module:factory syntax")
|
|
440
|
+
_require_sha256(
|
|
441
|
+
self.fingerprint_sha256,
|
|
442
|
+
label="external builder fingerprint_sha256",
|
|
443
|
+
)
|
|
444
|
+
object.__setattr__(self, "parameters", freeze_mapping(self.parameters))
|
|
445
|
+
|
|
446
|
+
@classmethod
|
|
447
|
+
def from_mapping(cls, value: Any, *, path: str) -> _LockedBuilderSpec:
|
|
448
|
+
data = _strict_mapping(
|
|
449
|
+
value,
|
|
450
|
+
path=path,
|
|
451
|
+
keys={"entrypoint", "fingerprint_sha256", "parameters"},
|
|
452
|
+
required={"entrypoint", "fingerprint_sha256"},
|
|
453
|
+
)
|
|
454
|
+
parameters = data.get("parameters", {})
|
|
455
|
+
if not isinstance(parameters, Mapping):
|
|
456
|
+
raise ValidationError(f"{path}.parameters must be an object")
|
|
457
|
+
return cls(
|
|
458
|
+
entrypoint=_text(data["entrypoint"], f"{path}.entrypoint"),
|
|
459
|
+
fingerprint_sha256=_text(
|
|
460
|
+
data["fingerprint_sha256"],
|
|
461
|
+
f"{path}.fingerprint_sha256",
|
|
462
|
+
),
|
|
463
|
+
parameters=parameters,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
@property
|
|
467
|
+
def binding_identity(self) -> Mapping[str, Any]:
|
|
468
|
+
return freeze_mapping(
|
|
469
|
+
{
|
|
470
|
+
"entrypoint": self.entrypoint,
|
|
471
|
+
"fingerprint_sha256": self.fingerprint_sha256,
|
|
472
|
+
"parameters_sha256": content_hash(self.parameters),
|
|
473
|
+
}
|
|
474
|
+
)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
@dataclass(frozen=True, slots=True)
|
|
478
|
+
class _BridgeIdentityLock:
|
|
479
|
+
bridge_id: str
|
|
480
|
+
bridge_revision: str
|
|
481
|
+
scorer_id: str
|
|
482
|
+
scorer_revision: str
|
|
483
|
+
dependency_hashes_by_model_slot: Mapping[str, Mapping[str, str]]
|
|
484
|
+
benchmark_id: str = ""
|
|
485
|
+
|
|
486
|
+
def __post_init__(self) -> None:
|
|
487
|
+
for label in ("bridge_id", "scorer_id"):
|
|
488
|
+
_require_text(getattr(self, label), label=f"bridge lock {label}")
|
|
489
|
+
for label in ("bridge_revision", "scorer_revision"):
|
|
490
|
+
_require_sha256(getattr(self, label), label=f"bridge lock {label}")
|
|
491
|
+
raw = self.dependency_hashes_by_model_slot
|
|
492
|
+
if not isinstance(raw, Mapping) or not raw:
|
|
493
|
+
raise ValidationError("bridge lock dependency_hashes_by_model_slot must be non-empty")
|
|
494
|
+
normalized: dict[str, Mapping[str, str]] = {}
|
|
495
|
+
for model_slot, dependencies in raw.items():
|
|
496
|
+
_require_text(model_slot, label="bridge lock model_slot")
|
|
497
|
+
normalized[model_slot] = _validated_dependency_hashes(dependencies, benchmark_id=self.benchmark_id)
|
|
498
|
+
object.__setattr__(
|
|
499
|
+
self,
|
|
500
|
+
"dependency_hashes_by_model_slot",
|
|
501
|
+
MappingProxyType(normalized),
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
@classmethod
|
|
505
|
+
def from_mapping(cls, value: Any, *, path: str, benchmark_id: str = "") -> _BridgeIdentityLock:
|
|
506
|
+
data = _strict_mapping(
|
|
507
|
+
value,
|
|
508
|
+
path=path,
|
|
509
|
+
keys={
|
|
510
|
+
"bridge_id",
|
|
511
|
+
"bridge_revision",
|
|
512
|
+
"scorer_id",
|
|
513
|
+
"scorer_revision",
|
|
514
|
+
"dependency_hashes_by_model_slot",
|
|
515
|
+
},
|
|
516
|
+
required={
|
|
517
|
+
"bridge_id",
|
|
518
|
+
"bridge_revision",
|
|
519
|
+
"scorer_id",
|
|
520
|
+
"scorer_revision",
|
|
521
|
+
"dependency_hashes_by_model_slot",
|
|
522
|
+
},
|
|
523
|
+
)
|
|
524
|
+
dependencies = data["dependency_hashes_by_model_slot"]
|
|
525
|
+
if not isinstance(dependencies, Mapping):
|
|
526
|
+
raise ValidationError(f"{path}.dependency_hashes_by_model_slot must be an object")
|
|
527
|
+
return cls(
|
|
528
|
+
bridge_id=_text(data["bridge_id"], f"{path}.bridge_id"),
|
|
529
|
+
bridge_revision=_text(
|
|
530
|
+
data["bridge_revision"],
|
|
531
|
+
f"{path}.bridge_revision",
|
|
532
|
+
),
|
|
533
|
+
scorer_id=_text(data["scorer_id"], f"{path}.scorer_id"),
|
|
534
|
+
scorer_revision=_text(
|
|
535
|
+
data["scorer_revision"],
|
|
536
|
+
f"{path}.scorer_revision",
|
|
537
|
+
),
|
|
538
|
+
dependency_hashes_by_model_slot=dependencies,
|
|
539
|
+
benchmark_id=benchmark_id,
|
|
540
|
+
)
|
|
541
|
+
|
|
542
|
+
def dependencies(self, model_slot: str) -> Mapping[str, str]:
|
|
543
|
+
try:
|
|
544
|
+
return self.dependency_hashes_by_model_slot[model_slot]
|
|
545
|
+
except KeyError as exc:
|
|
546
|
+
raise ValidationError(
|
|
547
|
+
f"bridge lock has no dependency identity for model slot {model_slot!r}"
|
|
548
|
+
) from exc
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
@dataclass(frozen=True, slots=True)
|
|
552
|
+
class _OutcomeContract:
|
|
553
|
+
formula_id: str
|
|
554
|
+
official_success_metric: str
|
|
555
|
+
|
|
556
|
+
def __post_init__(self) -> None:
|
|
557
|
+
if self.formula_id != _VALID_SUCCESS_FORMULA_ID:
|
|
558
|
+
raise ValidationError(
|
|
559
|
+
f"external outcome formula_id must be {_VALID_SUCCESS_FORMULA_ID!r}"
|
|
560
|
+
)
|
|
561
|
+
_require_text(
|
|
562
|
+
self.official_success_metric,
|
|
563
|
+
label="external outcome official_success_metric",
|
|
564
|
+
)
|
|
565
|
+
if self.official_success_metric in {
|
|
566
|
+
_CONTRACT_PRESERVED_METRIC,
|
|
567
|
+
_PRIVILEGED_LEAKAGE_METRIC,
|
|
568
|
+
_VALID_SUCCESS_METRIC,
|
|
569
|
+
}:
|
|
570
|
+
raise ValidationError(
|
|
571
|
+
"external official_success_metric conflicts with a host outcome field"
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
@classmethod
|
|
575
|
+
def from_mapping(cls, value: Any, *, path: str) -> _OutcomeContract:
|
|
576
|
+
data = _strict_mapping(
|
|
577
|
+
value,
|
|
578
|
+
path=path,
|
|
579
|
+
keys={"formula_id", "official_success_metric"},
|
|
580
|
+
required={"formula_id", "official_success_metric"},
|
|
581
|
+
)
|
|
582
|
+
return cls(
|
|
583
|
+
formula_id=_text(data["formula_id"], f"{path}.formula_id"),
|
|
584
|
+
official_success_metric=_text(
|
|
585
|
+
data["official_success_metric"],
|
|
586
|
+
f"{path}.official_success_metric",
|
|
587
|
+
),
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
@dataclass(frozen=True, slots=True)
|
|
592
|
+
class _TargetBinding:
|
|
593
|
+
benchmark_id: str
|
|
594
|
+
split: str
|
|
595
|
+
source_revision: str
|
|
596
|
+
checkpoint_builder: _LockedBuilderSpec
|
|
597
|
+
bridge_builder: _LockedBuilderSpec
|
|
598
|
+
bridge_lock: _BridgeIdentityLock
|
|
599
|
+
outcome_contract: _OutcomeContract
|
|
600
|
+
|
|
601
|
+
@property
|
|
602
|
+
def target_key(self) -> str:
|
|
603
|
+
return f"{self.benchmark_id}:{self.split}"
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
@dataclass(frozen=True, slots=True)
|
|
607
|
+
class _ArmBinding:
|
|
608
|
+
release_id: str
|
|
609
|
+
family: SemanticFamily
|
|
610
|
+
profile_kind: ProfileKind
|
|
611
|
+
arm_kind: ExperimentArmKind
|
|
612
|
+
payload_builder: _LockedBuilderSpec
|
|
613
|
+
policy_kind: str | None = None
|
|
614
|
+
|
|
615
|
+
def __post_init__(self) -> None:
|
|
616
|
+
if self.arm_kind is ExperimentArmKind.POLICY:
|
|
617
|
+
expected = _POLICY_KIND_BY_RELEASE.get(self.release_id)
|
|
618
|
+
if expected is None or self.policy_kind != expected:
|
|
619
|
+
raise ValidationError(
|
|
620
|
+
"policy arm binding kind differs from its supported policy release"
|
|
621
|
+
)
|
|
622
|
+
elif self.policy_kind is not None:
|
|
623
|
+
raise ValidationError("policy_kind is valid only for policy arm bindings")
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
@dataclass(frozen=True, slots=True)
|
|
627
|
+
class _GroupPartitionCommitment:
|
|
628
|
+
members_by_split: Mapping[str, tuple[str, ...]]
|
|
629
|
+
commitments_by_split: Mapping[str, str]
|
|
630
|
+
|
|
631
|
+
def __post_init__(self) -> None:
|
|
632
|
+
members = dict(self.members_by_split)
|
|
633
|
+
commitments = dict(self.commitments_by_split)
|
|
634
|
+
if set(members) != {"train", "calibration"} or set(commitments) != set(members):
|
|
635
|
+
raise ValidationError(
|
|
636
|
+
"policy group partition must contain exact train and calibration splits"
|
|
637
|
+
)
|
|
638
|
+
normalized_members: dict[str, tuple[str, ...]] = {}
|
|
639
|
+
observed: set[str] = set()
|
|
640
|
+
for split in ("train", "calibration"):
|
|
641
|
+
values = tuple(members[split])
|
|
642
|
+
if (
|
|
643
|
+
not values
|
|
644
|
+
or values != tuple(sorted(values))
|
|
645
|
+
or len(values) != len(set(values))
|
|
646
|
+
or any(
|
|
647
|
+
not isinstance(item, str) or _ANONYMIZED_GROUP.fullmatch(item) is None
|
|
648
|
+
for item in values
|
|
649
|
+
)
|
|
650
|
+
):
|
|
651
|
+
raise ValidationError(
|
|
652
|
+
f"policy group partition {split} members must be sorted unique "
|
|
653
|
+
"privacy-safe group hashes"
|
|
654
|
+
)
|
|
655
|
+
if observed.intersection(values):
|
|
656
|
+
raise ValidationError("policy group partition members cross splits")
|
|
657
|
+
observed.update(values)
|
|
658
|
+
expected = content_hash({"split": split, "group_ids": list(values)})
|
|
659
|
+
if commitments[split] != expected:
|
|
660
|
+
raise ValidationError(
|
|
661
|
+
f"policy group partition {split} commitment differs from its members"
|
|
662
|
+
)
|
|
663
|
+
normalized_members[split] = values
|
|
664
|
+
object.__setattr__(
|
|
665
|
+
self,
|
|
666
|
+
"members_by_split",
|
|
667
|
+
MappingProxyType(normalized_members),
|
|
668
|
+
)
|
|
669
|
+
object.__setattr__(
|
|
670
|
+
self,
|
|
671
|
+
"commitments_by_split",
|
|
672
|
+
MappingProxyType(commitments),
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
@classmethod
|
|
676
|
+
def from_mapping(
|
|
677
|
+
cls,
|
|
678
|
+
value: Any,
|
|
679
|
+
*,
|
|
680
|
+
path: str,
|
|
681
|
+
) -> _GroupPartitionCommitment:
|
|
682
|
+
data = _strict_mapping(
|
|
683
|
+
value,
|
|
684
|
+
path=path,
|
|
685
|
+
keys={"schema_version", "hash_scheme", "splits"},
|
|
686
|
+
required={"schema_version", "hash_scheme", "splits"},
|
|
687
|
+
)
|
|
688
|
+
if data["schema_version"] != 1 or data["hash_scheme"] != "benchmark_group.content_hash.v1":
|
|
689
|
+
raise ValidationError(f"{path} has an unsupported group commitment schema")
|
|
690
|
+
splits = _strict_mapping(
|
|
691
|
+
data["splits"],
|
|
692
|
+
path=f"{path}.splits",
|
|
693
|
+
keys={"train", "calibration"},
|
|
694
|
+
required={"train", "calibration"},
|
|
695
|
+
)
|
|
696
|
+
members: dict[str, tuple[str, ...]] = {}
|
|
697
|
+
commitments: dict[str, str] = {}
|
|
698
|
+
for split in ("train", "calibration"):
|
|
699
|
+
row = _strict_mapping(
|
|
700
|
+
splits[split],
|
|
701
|
+
path=f"{path}.splits.{split}",
|
|
702
|
+
keys={"member_hashes", "commitment_sha256"},
|
|
703
|
+
required={"member_hashes", "commitment_sha256"},
|
|
704
|
+
)
|
|
705
|
+
raw_members = row["member_hashes"]
|
|
706
|
+
if isinstance(raw_members, (str, bytes)) or not isinstance(
|
|
707
|
+
raw_members,
|
|
708
|
+
Sequence,
|
|
709
|
+
):
|
|
710
|
+
raise ValidationError(f"{path}.splits.{split}.member_hashes must be an array")
|
|
711
|
+
members[split] = tuple(raw_members)
|
|
712
|
+
commitments[split] = _text(
|
|
713
|
+
row["commitment_sha256"],
|
|
714
|
+
f"{path}.splits.{split}.commitment_sha256",
|
|
715
|
+
)
|
|
716
|
+
return cls(members, commitments)
|
|
717
|
+
|
|
718
|
+
@property
|
|
719
|
+
def excluded_members(self) -> frozenset[str]:
|
|
720
|
+
return frozenset(member for values in self.members_by_split.values() for member in values)
|
|
721
|
+
|
|
722
|
+
@property
|
|
723
|
+
def split_provenance_hash(self) -> str:
|
|
724
|
+
return content_hash({"group_split_digests": dict(self.commitments_by_split)})
|
|
725
|
+
|
|
726
|
+
def to_receipt(self) -> Mapping[str, Any]:
|
|
727
|
+
return freeze_mapping(
|
|
728
|
+
{
|
|
729
|
+
"hash_scheme": "benchmark_group.content_hash.v1",
|
|
730
|
+
"split_commitments": dict(self.commitments_by_split),
|
|
731
|
+
"excluded_member_count": len(self.excluded_members),
|
|
732
|
+
"excluded_members_sha256": content_hash(
|
|
733
|
+
{"member_hashes": sorted(self.excluded_members)}
|
|
734
|
+
),
|
|
735
|
+
}
|
|
736
|
+
)
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
@dataclass(frozen=True, slots=True)
|
|
740
|
+
class _SelectorPolicyManifest:
|
|
741
|
+
discovery_revision: str
|
|
742
|
+
discovery_sha256: str
|
|
743
|
+
training_table_sha256: str
|
|
744
|
+
split_provenance_hash: str
|
|
745
|
+
comparison_profile_hash: str
|
|
746
|
+
representative_by_family: Mapping[SemanticFamily, str]
|
|
747
|
+
selected_family: SemanticFamily
|
|
748
|
+
selected_release_id: str
|
|
749
|
+
calibration_split_digest: str
|
|
750
|
+
group_partition: _GroupPartitionCommitment
|
|
751
|
+
|
|
752
|
+
def __post_init__(self) -> None:
|
|
753
|
+
_require_text(self.discovery_revision, label="policy manifest discovery revision")
|
|
754
|
+
for label in (
|
|
755
|
+
"discovery_sha256",
|
|
756
|
+
"training_table_sha256",
|
|
757
|
+
"split_provenance_hash",
|
|
758
|
+
"comparison_profile_hash",
|
|
759
|
+
"calibration_split_digest",
|
|
760
|
+
):
|
|
761
|
+
_require_sha256(getattr(self, label), label=f"policy manifest {label}")
|
|
762
|
+
representatives = _validated_policy_representatives(self.representative_by_family)
|
|
763
|
+
if self.selected_family not in _CORE_FAMILIES:
|
|
764
|
+
raise ValidationError("policy manifest selected_family must be a core family")
|
|
765
|
+
_require_text(
|
|
766
|
+
self.selected_release_id,
|
|
767
|
+
label="policy manifest selected_release_id",
|
|
768
|
+
)
|
|
769
|
+
if representatives.get(self.selected_family) != self.selected_release_id:
|
|
770
|
+
raise ValidationError(
|
|
771
|
+
"policy manifest selected release differs from its frozen representative"
|
|
772
|
+
)
|
|
773
|
+
if not isinstance(self.group_partition, _GroupPartitionCommitment):
|
|
774
|
+
raise ValidationError("policy manifest requires a typed group partition")
|
|
775
|
+
if self.group_partition.split_provenance_hash != self.split_provenance_hash:
|
|
776
|
+
raise ValidationError("policy group partition differs from split_provenance_hash")
|
|
777
|
+
if (
|
|
778
|
+
self.group_partition.commitments_by_split["calibration"]
|
|
779
|
+
!= self.calibration_split_digest
|
|
780
|
+
):
|
|
781
|
+
raise ValidationError(
|
|
782
|
+
"policy group partition calibration commitment differs from selection receipt"
|
|
783
|
+
)
|
|
784
|
+
object.__setattr__(
|
|
785
|
+
self,
|
|
786
|
+
"representative_by_family",
|
|
787
|
+
MappingProxyType(representatives),
|
|
788
|
+
)
|
|
789
|
+
|
|
790
|
+
@classmethod
|
|
791
|
+
def from_mapping(cls, value: Any, *, path: str) -> _SelectorPolicyManifest:
|
|
792
|
+
data = _strict_mapping(
|
|
793
|
+
value,
|
|
794
|
+
path=path,
|
|
795
|
+
keys={
|
|
796
|
+
"schema_version",
|
|
797
|
+
"artifact",
|
|
798
|
+
"stage",
|
|
799
|
+
"discovery",
|
|
800
|
+
"training_table_sha256",
|
|
801
|
+
"split_provenance_hash",
|
|
802
|
+
"comparison_profile_hash",
|
|
803
|
+
"representative_by_family",
|
|
804
|
+
"group_partition",
|
|
805
|
+
"locked_best_eligible_global",
|
|
806
|
+
},
|
|
807
|
+
required={
|
|
808
|
+
"schema_version",
|
|
809
|
+
"artifact",
|
|
810
|
+
"stage",
|
|
811
|
+
"discovery",
|
|
812
|
+
"training_table_sha256",
|
|
813
|
+
"split_provenance_hash",
|
|
814
|
+
"comparison_profile_hash",
|
|
815
|
+
"representative_by_family",
|
|
816
|
+
"group_partition",
|
|
817
|
+
"locked_best_eligible_global",
|
|
818
|
+
},
|
|
819
|
+
)
|
|
820
|
+
if (
|
|
821
|
+
data["schema_version"] != 1
|
|
822
|
+
or data["artifact"] != "agentinputlab.selector_policy_manifest"
|
|
823
|
+
or data["stage"] != "frozen"
|
|
824
|
+
):
|
|
825
|
+
raise ValidationError(f"{path} must be a frozen selector policy manifest schema v1")
|
|
826
|
+
discovery = _strict_mapping(
|
|
827
|
+
data["discovery"],
|
|
828
|
+
path=f"{path}.discovery",
|
|
829
|
+
keys={"revision", "sha256"},
|
|
830
|
+
required={"revision", "sha256"},
|
|
831
|
+
)
|
|
832
|
+
locked = _strict_mapping(
|
|
833
|
+
data["locked_best_eligible_global"],
|
|
834
|
+
path=f"{path}.locked_best_eligible_global",
|
|
835
|
+
keys={"selected_family", "selected_release_id", "selection_receipt"},
|
|
836
|
+
required={"selected_family", "selected_release_id", "selection_receipt"},
|
|
837
|
+
)
|
|
838
|
+
receipt = _strict_mapping(
|
|
839
|
+
locked["selection_receipt"],
|
|
840
|
+
path=f"{path}.locked_best_eligible_global.selection_receipt",
|
|
841
|
+
keys={
|
|
842
|
+
"schema_version",
|
|
843
|
+
"rule_id",
|
|
844
|
+
"selection_split",
|
|
845
|
+
"training_table_sha256",
|
|
846
|
+
"calibration_split_digest",
|
|
847
|
+
},
|
|
848
|
+
required={
|
|
849
|
+
"schema_version",
|
|
850
|
+
"rule_id",
|
|
851
|
+
"selection_split",
|
|
852
|
+
"training_table_sha256",
|
|
853
|
+
"calibration_split_digest",
|
|
854
|
+
},
|
|
855
|
+
)
|
|
856
|
+
if (
|
|
857
|
+
receipt["schema_version"] != 1
|
|
858
|
+
or receipt["rule_id"] != "best_eligible_global.v1"
|
|
859
|
+
or receipt["selection_split"] != "calibration"
|
|
860
|
+
or receipt["training_table_sha256"] != data["training_table_sha256"]
|
|
861
|
+
):
|
|
862
|
+
raise ValidationError(f"{path} locked-best selection receipt is inconsistent")
|
|
863
|
+
representatives = _policy_representatives_from_mapping(
|
|
864
|
+
data["representative_by_family"],
|
|
865
|
+
path=f"{path}.representative_by_family",
|
|
866
|
+
)
|
|
867
|
+
try:
|
|
868
|
+
selected_family = SemanticFamily(
|
|
869
|
+
_text(
|
|
870
|
+
locked["selected_family"],
|
|
871
|
+
f"{path}.locked_best_eligible_global.selected_family",
|
|
872
|
+
)
|
|
873
|
+
)
|
|
874
|
+
except ValueError as exc:
|
|
875
|
+
raise ValidationError(f"{path} locked-best selected_family is unknown") from exc
|
|
876
|
+
return cls(
|
|
877
|
+
discovery_revision=_text(
|
|
878
|
+
discovery["revision"],
|
|
879
|
+
f"{path}.discovery.revision",
|
|
880
|
+
),
|
|
881
|
+
discovery_sha256=_text(
|
|
882
|
+
discovery["sha256"],
|
|
883
|
+
f"{path}.discovery.sha256",
|
|
884
|
+
),
|
|
885
|
+
training_table_sha256=_text(
|
|
886
|
+
data["training_table_sha256"],
|
|
887
|
+
f"{path}.training_table_sha256",
|
|
888
|
+
),
|
|
889
|
+
split_provenance_hash=_text(
|
|
890
|
+
data["split_provenance_hash"],
|
|
891
|
+
f"{path}.split_provenance_hash",
|
|
892
|
+
),
|
|
893
|
+
comparison_profile_hash=_text(
|
|
894
|
+
data["comparison_profile_hash"],
|
|
895
|
+
f"{path}.comparison_profile_hash",
|
|
896
|
+
),
|
|
897
|
+
representative_by_family=representatives,
|
|
898
|
+
selected_family=selected_family,
|
|
899
|
+
selected_release_id=_text(
|
|
900
|
+
locked["selected_release_id"],
|
|
901
|
+
f"{path}.locked_best_eligible_global.selected_release_id",
|
|
902
|
+
),
|
|
903
|
+
calibration_split_digest=_text(
|
|
904
|
+
receipt["calibration_split_digest"],
|
|
905
|
+
(f"{path}.locked_best_eligible_global.selection_receipt.calibration_split_digest"),
|
|
906
|
+
),
|
|
907
|
+
group_partition=_GroupPartitionCommitment.from_mapping(
|
|
908
|
+
data["group_partition"],
|
|
909
|
+
path=f"{path}.group_partition",
|
|
910
|
+
),
|
|
911
|
+
)
|
|
912
|
+
|
|
913
|
+
@property
|
|
914
|
+
def manifest_hash(self) -> str:
|
|
915
|
+
return content_hash(self)
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
@dataclass(frozen=True, slots=True)
|
|
919
|
+
class _DriverConfig:
|
|
920
|
+
driver_id: str
|
|
921
|
+
driver_revision_sha256: str
|
|
922
|
+
baseline_arm_id: str
|
|
923
|
+
benchmark_targets: Mapping[str, _TargetBinding]
|
|
924
|
+
arms: Mapping[str, _ArmBinding]
|
|
925
|
+
|
|
926
|
+
def __post_init__(self) -> None:
|
|
927
|
+
_require_text(self.driver_id, label="external driver_id")
|
|
928
|
+
_require_sha256(
|
|
929
|
+
self.driver_revision_sha256,
|
|
930
|
+
label="external driver_revision_sha256",
|
|
931
|
+
)
|
|
932
|
+
_require_text(self.baseline_arm_id, label="external baseline_arm_id")
|
|
933
|
+
targets = dict(self.benchmark_targets)
|
|
934
|
+
arms = dict(self.arms)
|
|
935
|
+
if not targets or any(key != binding.target_key for key, binding in targets.items()):
|
|
936
|
+
raise ValidationError(
|
|
937
|
+
"external benchmark target keys must be exact cell_id:split identities"
|
|
938
|
+
)
|
|
939
|
+
if not arms or self.baseline_arm_id not in arms:
|
|
940
|
+
raise ValidationError("external driver requires its configured baseline arm")
|
|
941
|
+
baseline = arms[self.baseline_arm_id]
|
|
942
|
+
if (
|
|
943
|
+
baseline.release_id != "baseline.no_op"
|
|
944
|
+
or baseline.family is not SemanticFamily.NO_OP
|
|
945
|
+
or baseline.arm_kind is not ExperimentArmKind.RELEASE
|
|
946
|
+
):
|
|
947
|
+
raise ValidationError(
|
|
948
|
+
"external driver baseline must bind baseline.no_op in the no_op family"
|
|
949
|
+
)
|
|
950
|
+
object.__setattr__(self, "benchmark_targets", MappingProxyType(targets))
|
|
951
|
+
object.__setattr__(self, "arms", MappingProxyType(arms))
|
|
952
|
+
|
|
953
|
+
@classmethod
|
|
954
|
+
def from_mapping(cls, value: Any) -> _DriverConfig:
|
|
955
|
+
data = _strict_mapping(
|
|
956
|
+
value,
|
|
957
|
+
path="external_driver_config",
|
|
958
|
+
keys={
|
|
959
|
+
"schema_version",
|
|
960
|
+
"driver_id",
|
|
961
|
+
"driver_revision_sha256",
|
|
962
|
+
"baseline_arm_id",
|
|
963
|
+
"benchmark_targets",
|
|
964
|
+
"arms",
|
|
965
|
+
},
|
|
966
|
+
required={
|
|
967
|
+
"schema_version",
|
|
968
|
+
"driver_id",
|
|
969
|
+
"driver_revision_sha256",
|
|
970
|
+
"baseline_arm_id",
|
|
971
|
+
"benchmark_targets",
|
|
972
|
+
"arms",
|
|
973
|
+
},
|
|
974
|
+
)
|
|
975
|
+
if data["schema_version"] != _CONFIG_SCHEMA_VERSION:
|
|
976
|
+
raise ValidationError("unsupported external arena config schema_version")
|
|
977
|
+
raw_targets = _nonempty_object(
|
|
978
|
+
data["benchmark_targets"],
|
|
979
|
+
path="external_driver_config.benchmark_targets",
|
|
980
|
+
)
|
|
981
|
+
raw_arms = _nonempty_object(
|
|
982
|
+
data["arms"],
|
|
983
|
+
path="external_driver_config.arms",
|
|
984
|
+
)
|
|
985
|
+
targets = {
|
|
986
|
+
key: _target_binding(
|
|
987
|
+
item,
|
|
988
|
+
path=f"external_driver_config.benchmark_targets[{key!r}]",
|
|
989
|
+
)
|
|
990
|
+
for key, item in raw_targets.items()
|
|
991
|
+
}
|
|
992
|
+
arms = {
|
|
993
|
+
key: _arm_binding(
|
|
994
|
+
item,
|
|
995
|
+
path=f"external_driver_config.arms[{key!r}]",
|
|
996
|
+
)
|
|
997
|
+
for key, item in raw_arms.items()
|
|
998
|
+
}
|
|
999
|
+
return cls(
|
|
1000
|
+
driver_id=_text(data["driver_id"], "external_driver_config.driver_id"),
|
|
1001
|
+
driver_revision_sha256=_text(
|
|
1002
|
+
data["driver_revision_sha256"],
|
|
1003
|
+
"external_driver_config.driver_revision_sha256",
|
|
1004
|
+
),
|
|
1005
|
+
baseline_arm_id=_text(
|
|
1006
|
+
data["baseline_arm_id"],
|
|
1007
|
+
"external_driver_config.baseline_arm_id",
|
|
1008
|
+
),
|
|
1009
|
+
benchmark_targets=targets,
|
|
1010
|
+
arms=arms,
|
|
1011
|
+
)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
class ExternalArenaExperimentDriver:
|
|
1015
|
+
"""Execute exact multi-arm cells through locked atomic external bridges."""
|
|
1016
|
+
|
|
1017
|
+
def __init__(self, config: _DriverConfig, *, config_sha256: str) -> None:
|
|
1018
|
+
if not isinstance(config, _DriverConfig):
|
|
1019
|
+
raise TypeError("external arena driver requires parsed configuration")
|
|
1020
|
+
_require_sha256(config_sha256, label="external driver config_sha256")
|
|
1021
|
+
self._config = config
|
|
1022
|
+
self._config_sha256 = config_sha256
|
|
1023
|
+
self._feature_extractor = VisibleStateFeatureExtractor()
|
|
1024
|
+
self._preflight_manifest_hash: str | None = None
|
|
1025
|
+
self._policy_binding_receipts: Mapping[str, Mapping[str, Any]] = MappingProxyType({})
|
|
1026
|
+
self._gate_binding_receipts: Mapping[str, Mapping[str, Any]] = MappingProxyType({})
|
|
1027
|
+
|
|
1028
|
+
@classmethod
|
|
1029
|
+
def from_file(
|
|
1030
|
+
cls,
|
|
1031
|
+
path: str | Path,
|
|
1032
|
+
*,
|
|
1033
|
+
expected_sha256: str,
|
|
1034
|
+
) -> ExternalArenaExperimentDriver:
|
|
1035
|
+
"""Load exact UTF-8 JSON bytes after checking their caller-supplied hash."""
|
|
1036
|
+
|
|
1037
|
+
_require_sha256(expected_sha256, label="expected external config SHA-256")
|
|
1038
|
+
source = Path(path).expanduser()
|
|
1039
|
+
if not source.is_absolute():
|
|
1040
|
+
raise ArtifactError("external arena config path must be absolute")
|
|
1041
|
+
try:
|
|
1042
|
+
if source.stat().st_size > _MAX_CONFIG_BYTES:
|
|
1043
|
+
raise ArtifactError("external arena config exceeds the size limit")
|
|
1044
|
+
raw = source.read_bytes()
|
|
1045
|
+
except OSError as exc:
|
|
1046
|
+
raise ArtifactError("cannot read external arena driver config") from exc
|
|
1047
|
+
observed = hashlib.sha256(raw).hexdigest()
|
|
1048
|
+
if observed != expected_sha256:
|
|
1049
|
+
raise ArtifactError("external arena driver config checksum mismatch")
|
|
1050
|
+
return cls(_DriverConfig.from_mapping(_strict_json(raw)), config_sha256=observed)
|
|
1051
|
+
|
|
1052
|
+
def fingerprint(self) -> Mapping[str, Any]:
|
|
1053
|
+
return freeze_mapping(
|
|
1054
|
+
{
|
|
1055
|
+
"implementation_id": _DRIVER_IMPLEMENTATION_ID,
|
|
1056
|
+
"schema_version": _CONFIG_SCHEMA_VERSION,
|
|
1057
|
+
"driver_id": self._config.driver_id,
|
|
1058
|
+
"driver_revision_sha256": self._config.driver_revision_sha256,
|
|
1059
|
+
"config_sha256": self._config_sha256,
|
|
1060
|
+
"baseline_arm_id": self._config.baseline_arm_id,
|
|
1061
|
+
"benchmark_targets": {
|
|
1062
|
+
key: {
|
|
1063
|
+
"benchmark_id": binding.benchmark_id,
|
|
1064
|
+
"split": binding.split,
|
|
1065
|
+
"source_revision": binding.source_revision,
|
|
1066
|
+
"checkpoint_builder": binding.checkpoint_builder.binding_identity,
|
|
1067
|
+
"bridge_builder": binding.bridge_builder.binding_identity,
|
|
1068
|
+
"bridge_lock": to_jsonable(binding.bridge_lock),
|
|
1069
|
+
"outcome_contract": to_jsonable(binding.outcome_contract),
|
|
1070
|
+
}
|
|
1071
|
+
for key, binding in self._config.benchmark_targets.items()
|
|
1072
|
+
},
|
|
1073
|
+
"arms": {
|
|
1074
|
+
arm_id: {
|
|
1075
|
+
"release_id": binding.release_id,
|
|
1076
|
+
"family": binding.family.value,
|
|
1077
|
+
"profile_kind": binding.profile_kind.value,
|
|
1078
|
+
"arm_kind": binding.arm_kind.value,
|
|
1079
|
+
"payload_builder": binding.payload_builder.binding_identity,
|
|
1080
|
+
"policy_kind": binding.policy_kind,
|
|
1081
|
+
}
|
|
1082
|
+
for arm_id, binding in self._config.arms.items()
|
|
1083
|
+
},
|
|
1084
|
+
"resolved_policy_bindings": self._policy_binding_receipts,
|
|
1085
|
+
"resolved_gate_bindings": self._gate_binding_receipts,
|
|
1086
|
+
"feature_schema_id": self._feature_extractor.feature_schema_id,
|
|
1087
|
+
"feature_schema_hash": self._feature_extractor.feature_schema_hash,
|
|
1088
|
+
# Static capability attestation for Studio's second,
|
|
1089
|
+
# execution-confirmation gate. Successful preflight still has
|
|
1090
|
+
# to produce the concrete policy/gate receipts below; these
|
|
1091
|
+
# booleans only state which fail-closed checks this driver
|
|
1092
|
+
# implementation enforces.
|
|
1093
|
+
"studio_campaign_attestation": {
|
|
1094
|
+
"schema_version": 1,
|
|
1095
|
+
"shared_dependency_slots": True,
|
|
1096
|
+
"selector_group_exclusion": True,
|
|
1097
|
+
"typed_gate_receipts": True,
|
|
1098
|
+
},
|
|
1099
|
+
}
|
|
1100
|
+
)
|
|
1101
|
+
|
|
1102
|
+
def preflight(self, manifest: ExperimentManifest) -> Sequence[str]:
|
|
1103
|
+
"""Validate locks and builders without constructing checkpoints or bridges."""
|
|
1104
|
+
|
|
1105
|
+
self._preflight_manifest_hash = None
|
|
1106
|
+
self._policy_binding_receipts = MappingProxyType({})
|
|
1107
|
+
self._gate_binding_receipts = MappingProxyType({})
|
|
1108
|
+
blockers = self._manifest_blockers(manifest)
|
|
1109
|
+
if blockers:
|
|
1110
|
+
return tuple(blockers)
|
|
1111
|
+
used_specs: dict[tuple[str, str, str, str], _LockedBuilderSpec] = {}
|
|
1112
|
+
for target in manifest.benchmarks:
|
|
1113
|
+
binding = self._config.benchmark_targets[_target_key(target)]
|
|
1114
|
+
for kind, spec in (
|
|
1115
|
+
("arena_checkpoint", binding.checkpoint_builder),
|
|
1116
|
+
("atomic_bridge", binding.bridge_builder),
|
|
1117
|
+
):
|
|
1118
|
+
used_specs[
|
|
1119
|
+
(
|
|
1120
|
+
kind,
|
|
1121
|
+
spec.entrypoint,
|
|
1122
|
+
spec.fingerprint_sha256,
|
|
1123
|
+
content_hash(spec.parameters),
|
|
1124
|
+
)
|
|
1125
|
+
] = spec
|
|
1126
|
+
selected_arm_ids = {
|
|
1127
|
+
arm_id
|
|
1128
|
+
for target in manifest.benchmarks
|
|
1129
|
+
for arm_id in (target.arm_ids or tuple(arm.arm_id for arm in manifest.arms))
|
|
1130
|
+
}
|
|
1131
|
+
for arm_id in selected_arm_ids:
|
|
1132
|
+
spec = self._config.arms[arm_id].payload_builder
|
|
1133
|
+
used_specs[
|
|
1134
|
+
(
|
|
1135
|
+
"arm_payload",
|
|
1136
|
+
spec.entrypoint,
|
|
1137
|
+
spec.fingerprint_sha256,
|
|
1138
|
+
content_hash(spec.parameters),
|
|
1139
|
+
)
|
|
1140
|
+
] = spec
|
|
1141
|
+
for (kind, _entrypoint, _fingerprint, _parameters), spec in used_specs.items():
|
|
1142
|
+
try:
|
|
1143
|
+
builder = _resolve_locked_builder(spec, expected_kind=kind)
|
|
1144
|
+
messages = _builder_preflight(builder, spec.parameters)
|
|
1145
|
+
except Exception as exc:
|
|
1146
|
+
blockers.append(f"{kind} builder: {_safe_message(exc)}")
|
|
1147
|
+
continue
|
|
1148
|
+
blockers.extend(f"{kind} builder: {_safe_text(item)}" for item in messages)
|
|
1149
|
+
try:
|
|
1150
|
+
policy_receipts = self._resolve_policy_bindings(manifest)
|
|
1151
|
+
except Exception as exc:
|
|
1152
|
+
blockers.append("policy binding: " + _safe_message(exc))
|
|
1153
|
+
policy_receipts = MappingProxyType({})
|
|
1154
|
+
try:
|
|
1155
|
+
gate_receipts = self._resolve_gate_bindings(manifest)
|
|
1156
|
+
except Exception as exc:
|
|
1157
|
+
blockers.append("external gate binding: " + _safe_message(exc))
|
|
1158
|
+
gate_receipts = MappingProxyType({})
|
|
1159
|
+
if blockers:
|
|
1160
|
+
return tuple(blockers)
|
|
1161
|
+
self._policy_binding_receipts = policy_receipts
|
|
1162
|
+
self._gate_binding_receipts = gate_receipts
|
|
1163
|
+
self._preflight_manifest_hash = manifest.manifest_hash
|
|
1164
|
+
return ()
|
|
1165
|
+
|
|
1166
|
+
def preflight_units(
|
|
1167
|
+
self,
|
|
1168
|
+
manifest: ExperimentManifest,
|
|
1169
|
+
resolved_units: Mapping[str, Sequence[str]],
|
|
1170
|
+
) -> Sequence[str]:
|
|
1171
|
+
"""Ask each locked checkpoint builder to recognize the exact planned IDs."""
|
|
1172
|
+
|
|
1173
|
+
if not isinstance(manifest, ExperimentManifest):
|
|
1174
|
+
return ("unit preflight requires an ExperimentManifest",)
|
|
1175
|
+
requirement = manifest.metadata.get("require_driver_unit_preflight")
|
|
1176
|
+
if requirement is not None and not isinstance(requirement, bool):
|
|
1177
|
+
return ("require_driver_unit_preflight metadata must be boolean when supplied",)
|
|
1178
|
+
required = requirement is True
|
|
1179
|
+
if not isinstance(resolved_units, Mapping):
|
|
1180
|
+
return ("resolved unit inventory must be a mapping",)
|
|
1181
|
+
expected_keys = {_target_key(target) for target in manifest.benchmarks}
|
|
1182
|
+
if set(resolved_units) != expected_keys:
|
|
1183
|
+
return ("resolved unit inventories differ from the manifest targets",)
|
|
1184
|
+
blockers: list[str] = []
|
|
1185
|
+
forbidden_group_hashes = self._policy_excluded_group_hashes()
|
|
1186
|
+
for target in manifest.benchmarks:
|
|
1187
|
+
key = _target_key(target)
|
|
1188
|
+
binding = self._config.benchmark_targets.get(key)
|
|
1189
|
+
if binding is None:
|
|
1190
|
+
blockers.append(f"benchmark {key}: no locked external target binding")
|
|
1191
|
+
continue
|
|
1192
|
+
try:
|
|
1193
|
+
unit_ids = tuple(sorted(resolved_units[key]))
|
|
1194
|
+
request = ExternalUnitPreflightRequest(
|
|
1195
|
+
benchmark_id=target.cell_id,
|
|
1196
|
+
split=target.split,
|
|
1197
|
+
source_revision=binding.source_revision,
|
|
1198
|
+
unit_ids=unit_ids,
|
|
1199
|
+
builder_parameters=binding.checkpoint_builder.parameters,
|
|
1200
|
+
)
|
|
1201
|
+
builder = _resolve_locked_builder(
|
|
1202
|
+
binding.checkpoint_builder,
|
|
1203
|
+
expected_kind="arena_checkpoint",
|
|
1204
|
+
)
|
|
1205
|
+
callback = getattr(builder, "preflight_units", None)
|
|
1206
|
+
if callback is None:
|
|
1207
|
+
if required:
|
|
1208
|
+
blockers.append(
|
|
1209
|
+
f"benchmark {key}: checkpoint builder lacks required "
|
|
1210
|
+
"read-only preflight_units"
|
|
1211
|
+
)
|
|
1212
|
+
continue
|
|
1213
|
+
if not callable(callback):
|
|
1214
|
+
raise ValidationError("checkpoint builder preflight_units must be callable")
|
|
1215
|
+
messages = callback(request)
|
|
1216
|
+
if (
|
|
1217
|
+
isinstance(messages, (str, bytes))
|
|
1218
|
+
or not isinstance(messages, Sequence)
|
|
1219
|
+
or any(not isinstance(item, str) or not item.strip() for item in messages)
|
|
1220
|
+
):
|
|
1221
|
+
raise ValidationError(
|
|
1222
|
+
"checkpoint builder preflight_units must return a "
|
|
1223
|
+
"sequence of non-empty blocker strings"
|
|
1224
|
+
)
|
|
1225
|
+
blockers.extend(f"benchmark {key}: {_safe_text(item)}" for item in messages)
|
|
1226
|
+
if forbidden_group_hashes:
|
|
1227
|
+
group_callback = getattr(builder, "preflight_group_ids", None)
|
|
1228
|
+
if not callable(group_callback):
|
|
1229
|
+
blockers.append(
|
|
1230
|
+
f"benchmark {key}: checkpoint builder lacks required "
|
|
1231
|
+
"read-only preflight_group_ids for selector leakage exclusion"
|
|
1232
|
+
)
|
|
1233
|
+
continue
|
|
1234
|
+
raw_groups = group_callback(request)
|
|
1235
|
+
if not isinstance(raw_groups, Mapping) or set(raw_groups) != set(unit_ids):
|
|
1236
|
+
raise ValidationError(
|
|
1237
|
+
"checkpoint builder preflight_group_ids must return an "
|
|
1238
|
+
"exact unit_id-to-group_id mapping"
|
|
1239
|
+
)
|
|
1240
|
+
for unit_id, group_id in raw_groups.items():
|
|
1241
|
+
_require_text(
|
|
1242
|
+
group_id,
|
|
1243
|
+
label=f"checkpoint group for unit {unit_id!r}",
|
|
1244
|
+
)
|
|
1245
|
+
group_hash = _anonymized_group_hash(
|
|
1246
|
+
target.cell_id,
|
|
1247
|
+
group_id,
|
|
1248
|
+
)
|
|
1249
|
+
if group_hash in forbidden_group_hashes:
|
|
1250
|
+
blockers.append(
|
|
1251
|
+
f"benchmark {key}: unit {unit_id!r} belongs to a "
|
|
1252
|
+
"selector discovery train/calibration group"
|
|
1253
|
+
)
|
|
1254
|
+
except Exception as exc:
|
|
1255
|
+
blockers.append(f"benchmark {key}: {_safe_message(exc)}")
|
|
1256
|
+
return tuple(blockers)
|
|
1257
|
+
|
|
1258
|
+
def execute_cell(
|
|
1259
|
+
self,
|
|
1260
|
+
manifest: ExperimentManifest,
|
|
1261
|
+
cell: ExperimentCell,
|
|
1262
|
+
) -> ExperimentCellResult:
|
|
1263
|
+
"""Execute a complete cell or raise before any terminal result is written."""
|
|
1264
|
+
|
|
1265
|
+
if self._preflight_manifest_hash != manifest.manifest_hash:
|
|
1266
|
+
raise ArtifactError(
|
|
1267
|
+
"external arena driver requires successful preflight for this manifest"
|
|
1268
|
+
)
|
|
1269
|
+
if self._resolve_policy_bindings(manifest) != self._policy_binding_receipts:
|
|
1270
|
+
raise ArtifactError("external policy artifact identity changed after preflight")
|
|
1271
|
+
if self._resolve_gate_bindings(manifest) != self._gate_binding_receipts:
|
|
1272
|
+
raise ArtifactError("external gate artifact identity changed after preflight")
|
|
1273
|
+
target, target_binding = self._validate_cell(manifest, cell)
|
|
1274
|
+
expected_dependencies = target_binding.bridge_lock.dependencies(cell.model_slot)
|
|
1275
|
+
checkpoint = self._build_checkpoint(cell, target_binding)
|
|
1276
|
+
self._validate_checkpoint(
|
|
1277
|
+
checkpoint,
|
|
1278
|
+
cell=cell,
|
|
1279
|
+
binding=target_binding,
|
|
1280
|
+
expected_dependencies=expected_dependencies,
|
|
1281
|
+
)
|
|
1282
|
+
checkpoint_group_hash = _anonymized_group_hash(
|
|
1283
|
+
checkpoint.benchmark_id,
|
|
1284
|
+
checkpoint.group_id,
|
|
1285
|
+
)
|
|
1286
|
+
if checkpoint_group_hash in self._policy_excluded_group_hashes():
|
|
1287
|
+
raise ValidationError(
|
|
1288
|
+
"external checkpoint belongs to a selector discovery train/calibration group"
|
|
1289
|
+
)
|
|
1290
|
+
pre_action = self._pre_action_receipt(checkpoint.visible_state)
|
|
1291
|
+
bridge = self._build_bridge(
|
|
1292
|
+
checkpoint,
|
|
1293
|
+
cell=cell,
|
|
1294
|
+
binding=target_binding,
|
|
1295
|
+
)
|
|
1296
|
+
self._validate_bridge(
|
|
1297
|
+
bridge,
|
|
1298
|
+
binding=target_binding,
|
|
1299
|
+
expected_dependencies=expected_dependencies,
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
arm_specs = {arm.arm_id: arm for arm in manifest.arms}
|
|
1303
|
+
artifacts = {
|
|
1304
|
+
arm_id: self._build_arm_payload(
|
|
1305
|
+
checkpoint,
|
|
1306
|
+
cell=cell,
|
|
1307
|
+
arm_spec=arm_specs[arm_id],
|
|
1308
|
+
pre_action=pre_action,
|
|
1309
|
+
)
|
|
1310
|
+
for arm_id in cell.arm_ids
|
|
1311
|
+
}
|
|
1312
|
+
self._validate_artifact_set(
|
|
1313
|
+
cell,
|
|
1314
|
+
checkpoint=checkpoint,
|
|
1315
|
+
arm_specs=arm_specs,
|
|
1316
|
+
artifacts=artifacts,
|
|
1317
|
+
)
|
|
1318
|
+
baseline_id = self._config.baseline_arm_id
|
|
1319
|
+
baseline_artifact = artifacts[baseline_id]
|
|
1320
|
+
pair_records: list[tuple[str, AtomicReplayRequest, AtomicPairedReplayRecord]] = []
|
|
1321
|
+
baseline_consistency: str | None = None
|
|
1322
|
+
for treatment_id in cell.arm_ids:
|
|
1323
|
+
if treatment_id == baseline_id:
|
|
1324
|
+
continue
|
|
1325
|
+
request = AtomicReplayRequest(
|
|
1326
|
+
benchmark_id=cell.benchmark_id,
|
|
1327
|
+
checkpoint_id=cell.unit_id,
|
|
1328
|
+
source_revision=target_binding.source_revision,
|
|
1329
|
+
origin_state_hash=checkpoint.origin_state_hash,
|
|
1330
|
+
replay_identity=checkpoint.replay_identity,
|
|
1331
|
+
seed=cell.seed,
|
|
1332
|
+
locked_dependency_hashes=expected_dependencies,
|
|
1333
|
+
arms=(
|
|
1334
|
+
AtomicReplayArmRequest(
|
|
1335
|
+
role="baseline",
|
|
1336
|
+
arm_id=baseline_id,
|
|
1337
|
+
payload=baseline_artifact.payload,
|
|
1338
|
+
),
|
|
1339
|
+
AtomicReplayArmRequest(
|
|
1340
|
+
role="treatment",
|
|
1341
|
+
arm_id=treatment_id,
|
|
1342
|
+
payload=artifacts[treatment_id].payload,
|
|
1343
|
+
),
|
|
1344
|
+
),
|
|
1345
|
+
)
|
|
1346
|
+
record = AtomicExternalReplayRunner().run(bridge, request)
|
|
1347
|
+
observed_consistency = _baseline_consistency_hash(record)
|
|
1348
|
+
if baseline_consistency is None:
|
|
1349
|
+
baseline_consistency = observed_consistency
|
|
1350
|
+
elif observed_consistency != baseline_consistency:
|
|
1351
|
+
raise ValidationError(
|
|
1352
|
+
"atomic multi-arm NO_OP metrics, trace, usage, or logical "
|
|
1353
|
+
"receipt changed across treatment pairs"
|
|
1354
|
+
)
|
|
1355
|
+
pair_records.append((treatment_id, request, record))
|
|
1356
|
+
if len(pair_records) != len(cell.arm_ids) - 1 or baseline_consistency is None:
|
|
1357
|
+
raise ValidationError("external arena driver produced a partial pair inventory")
|
|
1358
|
+
|
|
1359
|
+
arms = self._arm_outcomes(
|
|
1360
|
+
cell,
|
|
1361
|
+
arm_specs=arm_specs,
|
|
1362
|
+
artifacts=artifacts,
|
|
1363
|
+
pair_records=pair_records,
|
|
1364
|
+
target_binding=target_binding,
|
|
1365
|
+
)
|
|
1366
|
+
if tuple(item["arm_id"] for item in arms) != cell.arm_ids:
|
|
1367
|
+
raise ValidationError("external arena driver produced a partial arm inventory")
|
|
1368
|
+
atomic_pairs = tuple(
|
|
1369
|
+
freeze_mapping(
|
|
1370
|
+
{
|
|
1371
|
+
"treatment_arm_id": treatment_id,
|
|
1372
|
+
"request_sha256": content_hash(request),
|
|
1373
|
+
"receipt": to_jsonable(record.receipt),
|
|
1374
|
+
"receipt_sha256": content_hash(record.receipt),
|
|
1375
|
+
"baseline_consistency_sha256": baseline_consistency,
|
|
1376
|
+
"metric_deltas": record.metric_deltas,
|
|
1377
|
+
"resource_receipt_sha256s": tuple(
|
|
1378
|
+
content_hash(receipt) for receipt in record.resource_receipts
|
|
1379
|
+
),
|
|
1380
|
+
}
|
|
1381
|
+
)
|
|
1382
|
+
for treatment_id, request, record in pair_records
|
|
1383
|
+
)
|
|
1384
|
+
resource_receipts = tuple(
|
|
1385
|
+
receipt
|
|
1386
|
+
for _treatment_id, _request, record in pair_records
|
|
1387
|
+
for receipt in record.resource_receipts
|
|
1388
|
+
)
|
|
1389
|
+
payload: dict[str, Any] = {
|
|
1390
|
+
"benchmark_id": cell.benchmark_id,
|
|
1391
|
+
"split": cell.split,
|
|
1392
|
+
"unit_id": cell.unit_id,
|
|
1393
|
+
"model_slot": cell.model_slot,
|
|
1394
|
+
"seed": cell.seed,
|
|
1395
|
+
"baseline_arm_id": baseline_id,
|
|
1396
|
+
"planned_arm_ids": cell.arm_ids,
|
|
1397
|
+
"checkpoint_identity": {
|
|
1398
|
+
"identity_sha256": checkpoint.identity_hash,
|
|
1399
|
+
"group_sha256": checkpoint_group_hash,
|
|
1400
|
+
"visible_state_sha256": checkpoint.visible_state.state_hash,
|
|
1401
|
+
"origin_state_sha256": checkpoint.origin_state_hash,
|
|
1402
|
+
"replay_identity_sha256": checkpoint.replay_identity,
|
|
1403
|
+
"source_revision": checkpoint.source_revision,
|
|
1404
|
+
},
|
|
1405
|
+
"pre_action": pre_action,
|
|
1406
|
+
"arms": arms,
|
|
1407
|
+
"atomic_pairs": atomic_pairs,
|
|
1408
|
+
"baseline_consistency_sha256": baseline_consistency,
|
|
1409
|
+
"observed_dependency_hashes": expected_dependencies,
|
|
1410
|
+
"not_applicable_dependency_slots": tuple(
|
|
1411
|
+
sorted(
|
|
1412
|
+
slot
|
|
1413
|
+
for slot, digest in expected_dependencies.items()
|
|
1414
|
+
if digest == _NOT_APPLICABLE_DEPENDENCY_HASHES[slot]
|
|
1415
|
+
)
|
|
1416
|
+
),
|
|
1417
|
+
"target_source_revision": target.source_revision,
|
|
1418
|
+
"atomic_replay_contract": ATOMIC_EXTERNAL_PAIR_CONTRACT,
|
|
1419
|
+
}
|
|
1420
|
+
gate_receipt = self._gate_binding_receipts.get(_target_key(target))
|
|
1421
|
+
if gate_receipt is not None:
|
|
1422
|
+
payload["external_gate_receipt"] = gate_receipt
|
|
1423
|
+
if manifest.metadata.get("selector_training_allowed") is True:
|
|
1424
|
+
payload["selector_record"] = {
|
|
1425
|
+
"schema_version": 1,
|
|
1426
|
+
"state": checkpoint.visible_state.canonical_dict(),
|
|
1427
|
+
"group_id": checkpoint.group_id,
|
|
1428
|
+
"eligible_families": tuple(
|
|
1429
|
+
sorted(
|
|
1430
|
+
{
|
|
1431
|
+
arm_specs[arm_id].family.value
|
|
1432
|
+
for arm_id in cell.arm_ids
|
|
1433
|
+
if arm_specs[arm_id].family in _CORE_FAMILIES
|
|
1434
|
+
}
|
|
1435
|
+
)
|
|
1436
|
+
),
|
|
1437
|
+
}
|
|
1438
|
+
return ExperimentCellResult(
|
|
1439
|
+
cell_id=cell.cell_id,
|
|
1440
|
+
status="succeeded",
|
|
1441
|
+
payload=payload,
|
|
1442
|
+
resource_receipts=resource_receipts,
|
|
1443
|
+
)
|
|
1444
|
+
|
|
1445
|
+
def _policy_excluded_group_hashes(self) -> frozenset[str]:
|
|
1446
|
+
members: set[str] = set()
|
|
1447
|
+
for receipt in self._policy_binding_receipts.values():
|
|
1448
|
+
partition = receipt.get("group_partition")
|
|
1449
|
+
if not isinstance(partition, Mapping):
|
|
1450
|
+
continue
|
|
1451
|
+
raw = partition.get("excluded_member_hashes")
|
|
1452
|
+
if isinstance(raw, (str, bytes)) or not isinstance(raw, Sequence):
|
|
1453
|
+
raise ArtifactError("resolved policy group partition is invalid")
|
|
1454
|
+
for item in raw:
|
|
1455
|
+
if not isinstance(item, str) or _ANONYMIZED_GROUP.fullmatch(item) is None:
|
|
1456
|
+
raise ArtifactError("resolved policy group member hash is invalid")
|
|
1457
|
+
members.add(item)
|
|
1458
|
+
return frozenset(members)
|
|
1459
|
+
|
|
1460
|
+
def _resolve_policy_bindings(
|
|
1461
|
+
self,
|
|
1462
|
+
manifest: ExperimentManifest,
|
|
1463
|
+
) -> Mapping[str, Mapping[str, Any]]:
|
|
1464
|
+
receipts: dict[str, Mapping[str, Any]] = {}
|
|
1465
|
+
configured_releases = {
|
|
1466
|
+
binding.release_id: binding.family
|
|
1467
|
+
for binding in self._config.arms.values()
|
|
1468
|
+
if binding.arm_kind is ExperimentArmKind.RELEASE
|
|
1469
|
+
}
|
|
1470
|
+
for arm in manifest.arms:
|
|
1471
|
+
if arm.arm_kind is not ExperimentArmKind.POLICY:
|
|
1472
|
+
continue
|
|
1473
|
+
binding = self._config.arms[arm.arm_id]
|
|
1474
|
+
parameters = dict(arm.parameters)
|
|
1475
|
+
common_fields = {
|
|
1476
|
+
"policy_kind",
|
|
1477
|
+
"policy_manifest_env",
|
|
1478
|
+
"policy_manifest_sha256_env",
|
|
1479
|
+
}
|
|
1480
|
+
selector_fields = {
|
|
1481
|
+
"selector_identity_env",
|
|
1482
|
+
"selector_bundle_env",
|
|
1483
|
+
"selector_bundle_sha256_env",
|
|
1484
|
+
}
|
|
1485
|
+
required = (
|
|
1486
|
+
common_fields | selector_fields
|
|
1487
|
+
if binding.policy_kind == "frozen_selector"
|
|
1488
|
+
else common_fields
|
|
1489
|
+
)
|
|
1490
|
+
if set(parameters) != required:
|
|
1491
|
+
raise ValidationError(f"policy arm {arm.arm_id!r} binding fields are invalid")
|
|
1492
|
+
if parameters["policy_kind"] != binding.policy_kind:
|
|
1493
|
+
raise ValidationError(
|
|
1494
|
+
f"policy arm {arm.arm_id!r} kind differs from the locked config"
|
|
1495
|
+
)
|
|
1496
|
+
policy_bytes, policy_file_sha256 = _read_manifest_file_binding(
|
|
1497
|
+
manifest,
|
|
1498
|
+
environment_name=parameters["policy_manifest_env"],
|
|
1499
|
+
checksum_environment_name=parameters["policy_manifest_sha256_env"],
|
|
1500
|
+
label=f"policy arm {arm.arm_id!r} manifest",
|
|
1501
|
+
)
|
|
1502
|
+
policy_manifest = _SelectorPolicyManifest.from_mapping(
|
|
1503
|
+
_strict_json(policy_bytes),
|
|
1504
|
+
path=f"policy arm {arm.arm_id!r} manifest",
|
|
1505
|
+
)
|
|
1506
|
+
for family, release_id in policy_manifest.representative_by_family.items():
|
|
1507
|
+
if configured_releases.get(release_id) is not family:
|
|
1508
|
+
raise ValidationError(
|
|
1509
|
+
f"policy arm {arm.arm_id!r} representative {release_id!r} "
|
|
1510
|
+
"is outside the locked release bindings"
|
|
1511
|
+
)
|
|
1512
|
+
receipt: dict[str, Any] = {
|
|
1513
|
+
"policy_kind": binding.policy_kind,
|
|
1514
|
+
"policy_manifest_file_sha256": policy_file_sha256,
|
|
1515
|
+
"policy_manifest_hash": policy_manifest.manifest_hash,
|
|
1516
|
+
"discovery": {
|
|
1517
|
+
"revision": policy_manifest.discovery_revision,
|
|
1518
|
+
"sha256": policy_manifest.discovery_sha256,
|
|
1519
|
+
},
|
|
1520
|
+
"training_table_sha256": policy_manifest.training_table_sha256,
|
|
1521
|
+
"split_provenance_hash": policy_manifest.split_provenance_hash,
|
|
1522
|
+
"comparison_profile_hash": (policy_manifest.comparison_profile_hash),
|
|
1523
|
+
"group_partition": {
|
|
1524
|
+
**dict(policy_manifest.group_partition.to_receipt()),
|
|
1525
|
+
"excluded_member_hashes": tuple(
|
|
1526
|
+
sorted(policy_manifest.group_partition.excluded_members)
|
|
1527
|
+
),
|
|
1528
|
+
},
|
|
1529
|
+
"representative_by_family": {
|
|
1530
|
+
family.value: release_id
|
|
1531
|
+
for family, release_id in sorted(
|
|
1532
|
+
policy_manifest.representative_by_family.items(),
|
|
1533
|
+
key=lambda item: item[0].value,
|
|
1534
|
+
)
|
|
1535
|
+
},
|
|
1536
|
+
}
|
|
1537
|
+
if binding.policy_kind == "locked_best_eligible_global":
|
|
1538
|
+
receipt["selected_representative"] = {
|
|
1539
|
+
"family": policy_manifest.selected_family.value,
|
|
1540
|
+
"release_id": policy_manifest.selected_release_id,
|
|
1541
|
+
"calibration_split_digest": (policy_manifest.calibration_split_digest),
|
|
1542
|
+
}
|
|
1543
|
+
else:
|
|
1544
|
+
bundle_bytes, bundle_file_sha256 = _read_manifest_file_binding(
|
|
1545
|
+
manifest,
|
|
1546
|
+
environment_name=parameters["selector_bundle_env"],
|
|
1547
|
+
checksum_environment_name=parameters["selector_bundle_sha256_env"],
|
|
1548
|
+
label=f"policy arm {arm.arm_id!r} selector bundle",
|
|
1549
|
+
)
|
|
1550
|
+
bundle = _load_frozen_selector_bundle(
|
|
1551
|
+
bundle_bytes,
|
|
1552
|
+
identity_environment_name=parameters["selector_identity_env"],
|
|
1553
|
+
manifest=manifest,
|
|
1554
|
+
label=f"policy arm {arm.arm_id!r} selector bundle",
|
|
1555
|
+
)
|
|
1556
|
+
_validate_selector_policy_provenance(
|
|
1557
|
+
bundle,
|
|
1558
|
+
policy_manifest=policy_manifest,
|
|
1559
|
+
label=f"policy arm {arm.arm_id!r}",
|
|
1560
|
+
)
|
|
1561
|
+
receipt["selector_bundle"] = {
|
|
1562
|
+
"file_sha256": bundle_file_sha256,
|
|
1563
|
+
"selector_id": bundle.selector_card_id,
|
|
1564
|
+
"revision": bundle.revision,
|
|
1565
|
+
"bundle_hash": bundle.bundle_hash,
|
|
1566
|
+
"model_hash": bundle.model_hash,
|
|
1567
|
+
"stage": bundle.stage.value,
|
|
1568
|
+
}
|
|
1569
|
+
receipts[arm.arm_id] = freeze_mapping(receipt)
|
|
1570
|
+
return MappingProxyType(receipts)
|
|
1571
|
+
|
|
1572
|
+
def _resolve_gate_bindings(
|
|
1573
|
+
self,
|
|
1574
|
+
manifest: ExperimentManifest,
|
|
1575
|
+
) -> Mapping[str, Mapping[str, Any]]:
|
|
1576
|
+
receipts: dict[str, Mapping[str, Any]] = {}
|
|
1577
|
+
for target in manifest.benchmarks:
|
|
1578
|
+
gate_binding = _GATE_BINDINGS.get(target.cell_id)
|
|
1579
|
+
if gate_binding is None:
|
|
1580
|
+
continue
|
|
1581
|
+
environment_name, checksum_environment_name, reset_mode = gate_binding
|
|
1582
|
+
raw, file_sha256 = _read_manifest_file_binding(
|
|
1583
|
+
manifest,
|
|
1584
|
+
environment_name=environment_name,
|
|
1585
|
+
checksum_environment_name=checksum_environment_name,
|
|
1586
|
+
label=f"benchmark {target.cell_id!r} external gate receipt",
|
|
1587
|
+
)
|
|
1588
|
+
binding = self._config.benchmark_targets[_target_key(target)]
|
|
1589
|
+
inventory_sha256 = os.environ.get(target.unit_inventory_sha256_env)
|
|
1590
|
+
if not inventory_sha256:
|
|
1591
|
+
raise ArtifactError(
|
|
1592
|
+
f"benchmark {target.cell_id!r} unit inventory SHA-256 is missing"
|
|
1593
|
+
)
|
|
1594
|
+
_require_sha256(
|
|
1595
|
+
inventory_sha256,
|
|
1596
|
+
label=f"benchmark {target.cell_id!r} unit inventory SHA-256",
|
|
1597
|
+
)
|
|
1598
|
+
receipt = _validate_external_gate_receipt(
|
|
1599
|
+
_strict_json(raw),
|
|
1600
|
+
benchmark_id=target.cell_id,
|
|
1601
|
+
split=target.split,
|
|
1602
|
+
source_revision=binding.source_revision,
|
|
1603
|
+
manifest_hash=manifest.manifest_hash,
|
|
1604
|
+
driver_config_sha256=self._config_sha256,
|
|
1605
|
+
unit_inventory_sha256=inventory_sha256,
|
|
1606
|
+
bridge_lock=binding.bridge_lock,
|
|
1607
|
+
reset_mode=reset_mode,
|
|
1608
|
+
)
|
|
1609
|
+
receipts[_target_key(target)] = freeze_mapping(
|
|
1610
|
+
{
|
|
1611
|
+
"file_sha256": file_sha256,
|
|
1612
|
+
"receipt_sha256": content_hash(receipt),
|
|
1613
|
+
"scope_sha256": receipt["approval"]["scope_sha256"],
|
|
1614
|
+
"benchmark_id": target.cell_id,
|
|
1615
|
+
"split": target.split,
|
|
1616
|
+
"source_revision": binding.source_revision,
|
|
1617
|
+
"manifest_hash": manifest.manifest_hash,
|
|
1618
|
+
"driver_config_sha256": self._config_sha256,
|
|
1619
|
+
"bridge": receipt["bridge"],
|
|
1620
|
+
"scorer": receipt["scorer"],
|
|
1621
|
+
"approval": receipt["approval"],
|
|
1622
|
+
"verification": receipt["verification"],
|
|
1623
|
+
"unit_inventory_sha256": inventory_sha256,
|
|
1624
|
+
}
|
|
1625
|
+
)
|
|
1626
|
+
return MappingProxyType(receipts)
|
|
1627
|
+
|
|
1628
|
+
def _manifest_blockers(self, manifest: ExperimentManifest) -> list[str]:
|
|
1629
|
+
if not isinstance(manifest, ExperimentManifest):
|
|
1630
|
+
return ["manifest must be an ExperimentManifest"]
|
|
1631
|
+
blockers: list[str] = []
|
|
1632
|
+
if manifest.estimand_profile_kind is ProfileKind.OFFICIAL_NATIVE:
|
|
1633
|
+
blockers.append(
|
|
1634
|
+
"external arena driver supports controlled_adapter or "
|
|
1635
|
+
"factorial_adapter estimands, not official_native"
|
|
1636
|
+
)
|
|
1637
|
+
manifest_arms = {arm.arm_id: arm for arm in manifest.arms}
|
|
1638
|
+
if self._config.baseline_arm_id not in manifest_arms:
|
|
1639
|
+
blockers.append("manifest omits the configured NO_OP baseline")
|
|
1640
|
+
elif manifest_arms[self._config.baseline_arm_id].family is not SemanticFamily.NO_OP:
|
|
1641
|
+
blockers.append("configured baseline arm is not no_op in the manifest")
|
|
1642
|
+
required_payload_builder = manifest.metadata.get("required_payload_builder")
|
|
1643
|
+
if required_payload_builder is not None and (
|
|
1644
|
+
not isinstance(required_payload_builder, str)
|
|
1645
|
+
or _ENTRYPOINT.fullmatch(required_payload_builder) is None
|
|
1646
|
+
):
|
|
1647
|
+
blockers.append("manifest required_payload_builder is not an entrypoint")
|
|
1648
|
+
for arm_id, arm in manifest_arms.items():
|
|
1649
|
+
binding = self._config.arms.get(arm_id)
|
|
1650
|
+
if binding is None:
|
|
1651
|
+
blockers.append(f"arm {arm_id}: no locked payload binding")
|
|
1652
|
+
continue
|
|
1653
|
+
if (
|
|
1654
|
+
binding.release_id != arm.release_id
|
|
1655
|
+
or binding.family is not arm.family
|
|
1656
|
+
or binding.profile_kind is not arm.profile_kind
|
|
1657
|
+
or binding.arm_kind is not arm.arm_kind
|
|
1658
|
+
):
|
|
1659
|
+
blockers.append(
|
|
1660
|
+
f"arm {arm_id}: manifest identity differs from locked payload binding"
|
|
1661
|
+
)
|
|
1662
|
+
elif (
|
|
1663
|
+
arm.arm_kind is ExperimentArmKind.POLICY
|
|
1664
|
+
and arm.parameters.get("policy_kind") != binding.policy_kind
|
|
1665
|
+
):
|
|
1666
|
+
blockers.append(
|
|
1667
|
+
f"arm {arm_id}: manifest policy kind differs from locked payload binding"
|
|
1668
|
+
)
|
|
1669
|
+
elif (
|
|
1670
|
+
arm.release_id == "baseline.no_op_cost_shadow"
|
|
1671
|
+
and required_payload_builder is not None
|
|
1672
|
+
and binding.payload_builder.entrypoint != required_payload_builder
|
|
1673
|
+
):
|
|
1674
|
+
blockers.append(
|
|
1675
|
+
f"arm {arm_id}: payload builder differs from the manifest-required "
|
|
1676
|
+
"cost-shadow builder"
|
|
1677
|
+
)
|
|
1678
|
+
for target in manifest.benchmarks:
|
|
1679
|
+
key = _target_key(target)
|
|
1680
|
+
binding = self._config.benchmark_targets.get(key)
|
|
1681
|
+
if binding is None:
|
|
1682
|
+
blockers.append(f"benchmark {key}: no locked external target binding")
|
|
1683
|
+
continue
|
|
1684
|
+
if target.cell_id not in _SUPPORTED_ARENAS:
|
|
1685
|
+
blockers.append(f"benchmark {target.cell_id}: unsupported external arena")
|
|
1686
|
+
if target.cell_id in _GATE_BINDINGS and (
|
|
1687
|
+
target.external is not True
|
|
1688
|
+
or target.unit_inventory_env is None
|
|
1689
|
+
or target.unit_inventory_sha256_env is None
|
|
1690
|
+
):
|
|
1691
|
+
blockers.append(
|
|
1692
|
+
f"benchmark {target.cell_id}: gated external arena requires "
|
|
1693
|
+
"external=true and a checksum-bound unit inventory"
|
|
1694
|
+
)
|
|
1695
|
+
if (
|
|
1696
|
+
target.source_revision is not None
|
|
1697
|
+
and target.source_revision != binding.source_revision
|
|
1698
|
+
):
|
|
1699
|
+
blockers.append(
|
|
1700
|
+
f"benchmark {target.cell_id}: manifest source revision differs "
|
|
1701
|
+
"from the external driver lock"
|
|
1702
|
+
)
|
|
1703
|
+
selected = target.arm_ids or tuple(manifest_arms)
|
|
1704
|
+
if self._config.baseline_arm_id not in selected:
|
|
1705
|
+
blockers.append(
|
|
1706
|
+
f"benchmark {target.cell_id}: selected arms omit the NO_OP baseline"
|
|
1707
|
+
)
|
|
1708
|
+
unknown = sorted(set(selected) - set(self._config.arms))
|
|
1709
|
+
if unknown:
|
|
1710
|
+
blockers.append(
|
|
1711
|
+
f"benchmark {target.cell_id}: selected arms lack payload bindings: "
|
|
1712
|
+
+ ", ".join(unknown)
|
|
1713
|
+
)
|
|
1714
|
+
for model_slot in manifest.model_slots:
|
|
1715
|
+
if model_slot not in binding.bridge_lock.dependency_hashes_by_model_slot:
|
|
1716
|
+
blockers.append(
|
|
1717
|
+
f"benchmark {target.cell_id}: bridge lock has no model slot {model_slot}"
|
|
1718
|
+
)
|
|
1719
|
+
return blockers
|
|
1720
|
+
|
|
1721
|
+
def _validate_cell(
|
|
1722
|
+
self,
|
|
1723
|
+
manifest: ExperimentManifest,
|
|
1724
|
+
cell: ExperimentCell,
|
|
1725
|
+
) -> tuple[ExperimentBenchmarkTarget, _TargetBinding]:
|
|
1726
|
+
if not isinstance(cell, ExperimentCell):
|
|
1727
|
+
raise TypeError("external arena driver requires ExperimentCell")
|
|
1728
|
+
if cell.manifest_hash != manifest.manifest_hash:
|
|
1729
|
+
raise ValidationError("external arena cell belongs to another manifest")
|
|
1730
|
+
matches = tuple(
|
|
1731
|
+
target
|
|
1732
|
+
for target in manifest.benchmarks
|
|
1733
|
+
if target.cell_id == cell.benchmark_id and target.split == cell.split
|
|
1734
|
+
)
|
|
1735
|
+
if len(matches) != 1:
|
|
1736
|
+
raise ValidationError("external arena cell has no unique manifest target")
|
|
1737
|
+
target = matches[0]
|
|
1738
|
+
expected_arms = target.arm_ids or tuple(arm.arm_id for arm in manifest.arms)
|
|
1739
|
+
if cell.arm_ids != expected_arms:
|
|
1740
|
+
raise ValidationError("external arena cell arms differ from the target")
|
|
1741
|
+
if cell.model_slot not in manifest.model_slots or cell.seed not in manifest.seeds:
|
|
1742
|
+
raise ValidationError("external arena cell model slot or seed is not frozen")
|
|
1743
|
+
return target, self._config.benchmark_targets[_target_key(target)]
|
|
1744
|
+
|
|
1745
|
+
def _build_checkpoint(
|
|
1746
|
+
self,
|
|
1747
|
+
cell: ExperimentCell,
|
|
1748
|
+
binding: _TargetBinding,
|
|
1749
|
+
) -> ExternalArenaCheckpoint:
|
|
1750
|
+
spec = binding.checkpoint_builder
|
|
1751
|
+
builder = _resolve_locked_builder(spec, expected_kind="arena_checkpoint")
|
|
1752
|
+
checkpoint = builder.build(
|
|
1753
|
+
ExternalCheckpointBuildRequest(
|
|
1754
|
+
benchmark_id=cell.benchmark_id,
|
|
1755
|
+
split=cell.split,
|
|
1756
|
+
checkpoint_id=cell.unit_id,
|
|
1757
|
+
source_revision=binding.source_revision,
|
|
1758
|
+
model_slot=cell.model_slot,
|
|
1759
|
+
seed=cell.seed,
|
|
1760
|
+
builder_parameters=spec.parameters,
|
|
1761
|
+
)
|
|
1762
|
+
)
|
|
1763
|
+
if not isinstance(checkpoint, ExternalArenaCheckpoint):
|
|
1764
|
+
raise ValidationError("arena checkpoint builder must return ExternalArenaCheckpoint")
|
|
1765
|
+
return checkpoint
|
|
1766
|
+
|
|
1767
|
+
def _validate_checkpoint(
|
|
1768
|
+
self,
|
|
1769
|
+
checkpoint: ExternalArenaCheckpoint,
|
|
1770
|
+
*,
|
|
1771
|
+
cell: ExperimentCell,
|
|
1772
|
+
binding: _TargetBinding,
|
|
1773
|
+
expected_dependencies: Mapping[str, str],
|
|
1774
|
+
) -> None:
|
|
1775
|
+
if (
|
|
1776
|
+
checkpoint.benchmark_id != cell.benchmark_id
|
|
1777
|
+
or checkpoint.checkpoint_id != cell.unit_id
|
|
1778
|
+
or checkpoint.source_revision != binding.source_revision
|
|
1779
|
+
or checkpoint.locked_dependency_hashes != expected_dependencies
|
|
1780
|
+
):
|
|
1781
|
+
raise ValidationError(
|
|
1782
|
+
"external checkpoint identity or dependencies differ from the frozen cell"
|
|
1783
|
+
)
|
|
1784
|
+
|
|
1785
|
+
def _build_bridge(
|
|
1786
|
+
self,
|
|
1787
|
+
checkpoint: ExternalArenaCheckpoint,
|
|
1788
|
+
*,
|
|
1789
|
+
cell: ExperimentCell,
|
|
1790
|
+
binding: _TargetBinding,
|
|
1791
|
+
) -> AtomicExternalReplayBridge:
|
|
1792
|
+
spec = binding.bridge_builder
|
|
1793
|
+
builder = _resolve_locked_builder(spec, expected_kind="atomic_bridge")
|
|
1794
|
+
bridge = builder.build(
|
|
1795
|
+
ExternalBridgeBuildRequest(
|
|
1796
|
+
checkpoint=checkpoint,
|
|
1797
|
+
model_slot=cell.model_slot,
|
|
1798
|
+
seed=cell.seed,
|
|
1799
|
+
builder_parameters=spec.parameters,
|
|
1800
|
+
)
|
|
1801
|
+
)
|
|
1802
|
+
if not isinstance(bridge, AtomicExternalReplayBridge):
|
|
1803
|
+
raise ValidationError("atomic bridge builder must return AtomicExternalReplayBridge")
|
|
1804
|
+
return bridge
|
|
1805
|
+
|
|
1806
|
+
def _validate_bridge(
|
|
1807
|
+
self,
|
|
1808
|
+
bridge: AtomicExternalReplayBridge,
|
|
1809
|
+
*,
|
|
1810
|
+
binding: _TargetBinding,
|
|
1811
|
+
expected_dependencies: Mapping[str, str],
|
|
1812
|
+
) -> None:
|
|
1813
|
+
lock = binding.bridge_lock
|
|
1814
|
+
if (
|
|
1815
|
+
bridge.atomic_replay_contract != ATOMIC_EXTERNAL_PAIR_CONTRACT
|
|
1816
|
+
or bridge.benchmark_id != binding.benchmark_id
|
|
1817
|
+
or bridge.source_revision != binding.source_revision
|
|
1818
|
+
or bridge.bridge_id != lock.bridge_id
|
|
1819
|
+
or bridge.bridge_revision != lock.bridge_revision
|
|
1820
|
+
or bridge.scorer_id != lock.scorer_id
|
|
1821
|
+
or bridge.scorer_revision != lock.scorer_revision
|
|
1822
|
+
or _validated_dependency_hashes(bridge.locked_dependency_hashes, benchmark_id=binding.benchmark_id)
|
|
1823
|
+
!= expected_dependencies
|
|
1824
|
+
):
|
|
1825
|
+
raise ValidationError(
|
|
1826
|
+
"constructed atomic bridge differs from its checksum-locked identity"
|
|
1827
|
+
)
|
|
1828
|
+
|
|
1829
|
+
def _build_arm_payload(
|
|
1830
|
+
self,
|
|
1831
|
+
checkpoint: ExternalArenaCheckpoint,
|
|
1832
|
+
*,
|
|
1833
|
+
cell: ExperimentCell,
|
|
1834
|
+
arm_spec: ExperimentArmSpec,
|
|
1835
|
+
pre_action: Mapping[str, Any],
|
|
1836
|
+
) -> ExternalArmPayload:
|
|
1837
|
+
binding = self._config.arms[arm_spec.arm_id]
|
|
1838
|
+
spec = binding.payload_builder
|
|
1839
|
+
builder = _resolve_locked_builder(spec, expected_kind="arm_payload")
|
|
1840
|
+
artifact = builder.build(
|
|
1841
|
+
ExternalArmPayloadBuildRequest(
|
|
1842
|
+
checkpoint=checkpoint,
|
|
1843
|
+
arm_id=arm_spec.arm_id,
|
|
1844
|
+
release_id=arm_spec.release_id,
|
|
1845
|
+
family=arm_spec.family,
|
|
1846
|
+
profile_kind=arm_spec.profile_kind,
|
|
1847
|
+
arm_kind=arm_spec.arm_kind,
|
|
1848
|
+
model_slot=cell.model_slot,
|
|
1849
|
+
seed=cell.seed,
|
|
1850
|
+
builder_parameters=spec.parameters,
|
|
1851
|
+
arm_parameters=arm_spec.parameters,
|
|
1852
|
+
pre_action=pre_action,
|
|
1853
|
+
)
|
|
1854
|
+
)
|
|
1855
|
+
if not isinstance(artifact, ExternalArmPayload):
|
|
1856
|
+
raise ValidationError("arm payload builder must return ExternalArmPayload")
|
|
1857
|
+
if (
|
|
1858
|
+
artifact.arm_id != arm_spec.arm_id
|
|
1859
|
+
or artifact.release_id != arm_spec.release_id
|
|
1860
|
+
or artifact.family is not arm_spec.family
|
|
1861
|
+
or artifact.profile_kind is not arm_spec.profile_kind
|
|
1862
|
+
):
|
|
1863
|
+
raise ValidationError(
|
|
1864
|
+
f"arm payload builder returned another identity for {arm_spec.arm_id!r}"
|
|
1865
|
+
)
|
|
1866
|
+
if arm_spec.arm_kind is ExperimentArmKind.POLICY:
|
|
1867
|
+
if artifact.selector_decision is None:
|
|
1868
|
+
raise ValidationError(
|
|
1869
|
+
f"policy arm {arm_spec.arm_id!r} requires selector decision receipt"
|
|
1870
|
+
)
|
|
1871
|
+
if (
|
|
1872
|
+
artifact.selector_input_state_hash != pre_action["state_hash"]
|
|
1873
|
+
or artifact.selector_feature_hash != pre_action["feature_hash"]
|
|
1874
|
+
):
|
|
1875
|
+
raise ValidationError(
|
|
1876
|
+
f"policy arm {arm_spec.arm_id!r} selector input identity differs "
|
|
1877
|
+
"from the visible pre_action receipt"
|
|
1878
|
+
)
|
|
1879
|
+
try:
|
|
1880
|
+
policy_receipt = self._policy_binding_receipts[arm_spec.arm_id]
|
|
1881
|
+
except KeyError as exc: # pragma: no cover - preflight invariant
|
|
1882
|
+
raise ArtifactError(
|
|
1883
|
+
f"policy arm {arm_spec.arm_id!r} lacks a resolved binding receipt"
|
|
1884
|
+
) from exc
|
|
1885
|
+
if binding.policy_kind == "locked_best_eligible_global":
|
|
1886
|
+
if (
|
|
1887
|
+
set(artifact.selector_decision)
|
|
1888
|
+
!= {
|
|
1889
|
+
"selected_family",
|
|
1890
|
+
"selected_release_id",
|
|
1891
|
+
"abstained",
|
|
1892
|
+
"predicted_gain",
|
|
1893
|
+
"confidence",
|
|
1894
|
+
"policy_manifest_sha256",
|
|
1895
|
+
}
|
|
1896
|
+
or artifact.selector_decision["policy_manifest_sha256"]
|
|
1897
|
+
!= policy_receipt["policy_manifest_file_sha256"]
|
|
1898
|
+
):
|
|
1899
|
+
raise ValidationError(
|
|
1900
|
+
f"policy arm {arm_spec.arm_id!r} decision differs from the "
|
|
1901
|
+
"independently hashed policy manifest"
|
|
1902
|
+
)
|
|
1903
|
+
selected = policy_receipt["selected_representative"]
|
|
1904
|
+
decision_family = artifact.selector_decision["selected_family"]
|
|
1905
|
+
if decision_family != SemanticFamily.NO_OP.value and (
|
|
1906
|
+
decision_family != selected["family"]
|
|
1907
|
+
or artifact.selector_decision["selected_release_id"] != selected["release_id"]
|
|
1908
|
+
):
|
|
1909
|
+
raise ValidationError(
|
|
1910
|
+
f"policy arm {arm_spec.arm_id!r} selected a release other "
|
|
1911
|
+
"than the frozen global representative"
|
|
1912
|
+
)
|
|
1913
|
+
if (
|
|
1914
|
+
decision_family == SemanticFamily.NO_OP.value
|
|
1915
|
+
and SemanticFamily(selected["family"]) in checkpoint.eligible_families
|
|
1916
|
+
):
|
|
1917
|
+
raise ValidationError(
|
|
1918
|
+
f"policy arm {arm_spec.arm_id!r} abstained even though the "
|
|
1919
|
+
"frozen global representative is eligible"
|
|
1920
|
+
)
|
|
1921
|
+
elif (
|
|
1922
|
+
set(artifact.selector_decision)
|
|
1923
|
+
!= {
|
|
1924
|
+
"selected_family",
|
|
1925
|
+
"selected_release_id",
|
|
1926
|
+
"abstained",
|
|
1927
|
+
"predicted_gain",
|
|
1928
|
+
"confidence",
|
|
1929
|
+
"selector_bundle_hash",
|
|
1930
|
+
}
|
|
1931
|
+
or artifact.selector_decision["selector_bundle_hash"]
|
|
1932
|
+
!= policy_receipt["selector_bundle"]["bundle_hash"]
|
|
1933
|
+
):
|
|
1934
|
+
raise ValidationError(
|
|
1935
|
+
f"policy arm {arm_spec.arm_id!r} selector bundle hash differs "
|
|
1936
|
+
"from the independently parsed frozen bundle"
|
|
1937
|
+
)
|
|
1938
|
+
elif artifact.selector_decision["selected_family"] != SemanticFamily.NO_OP.value:
|
|
1939
|
+
representatives = policy_receipt["representative_by_family"]
|
|
1940
|
+
if (
|
|
1941
|
+
representatives.get(artifact.selector_decision["selected_family"])
|
|
1942
|
+
!= (artifact.selector_decision["selected_release_id"])
|
|
1943
|
+
):
|
|
1944
|
+
raise ValidationError(
|
|
1945
|
+
f"policy arm {arm_spec.arm_id!r} selected a release outside "
|
|
1946
|
+
"the frozen selector representatives"
|
|
1947
|
+
)
|
|
1948
|
+
elif (
|
|
1949
|
+
artifact.selector_decision is not None
|
|
1950
|
+
or artifact.selector_input_state_hash is not None
|
|
1951
|
+
or artifact.selector_feature_hash is not None
|
|
1952
|
+
):
|
|
1953
|
+
raise ValidationError(f"release arm {arm_spec.arm_id!r} cannot carry selector evidence")
|
|
1954
|
+
return artifact
|
|
1955
|
+
|
|
1956
|
+
def _validate_artifact_set(
|
|
1957
|
+
self,
|
|
1958
|
+
cell: ExperimentCell,
|
|
1959
|
+
*,
|
|
1960
|
+
checkpoint: ExternalArenaCheckpoint,
|
|
1961
|
+
arm_specs: Mapping[str, ExperimentArmSpec],
|
|
1962
|
+
artifacts: Mapping[str, ExternalArmPayload],
|
|
1963
|
+
) -> None:
|
|
1964
|
+
if set(artifacts) != set(cell.arm_ids) or len(artifacts) != len(cell.arm_ids):
|
|
1965
|
+
raise ValidationError("external arm payload inventory is partial")
|
|
1966
|
+
eligible = set(checkpoint.eligible_families)
|
|
1967
|
+
configured_release_families = {
|
|
1968
|
+
binding.release_id: binding.family
|
|
1969
|
+
for binding in self._config.arms.values()
|
|
1970
|
+
if binding.arm_kind is ExperimentArmKind.RELEASE
|
|
1971
|
+
}
|
|
1972
|
+
for arm_id in cell.arm_ids:
|
|
1973
|
+
arm_spec = arm_specs[arm_id]
|
|
1974
|
+
artifact = artifacts[arm_id]
|
|
1975
|
+
if (
|
|
1976
|
+
arm_spec.arm_kind is ExperimentArmKind.RELEASE
|
|
1977
|
+
and arm_spec.family in _EXECUTABLE_RELEASE_FAMILIES
|
|
1978
|
+
and arm_spec.family not in eligible
|
|
1979
|
+
):
|
|
1980
|
+
raise ValidationError(
|
|
1981
|
+
f"checkpoint is not eligible for arm family {arm_spec.family.value}"
|
|
1982
|
+
)
|
|
1983
|
+
decision = artifact.selector_decision
|
|
1984
|
+
if decision is None:
|
|
1985
|
+
continue
|
|
1986
|
+
selected_family = SemanticFamily(decision["selected_family"])
|
|
1987
|
+
selected_release_id = decision["selected_release_id"]
|
|
1988
|
+
if selected_family is not SemanticFamily.NO_OP and selected_family not in eligible:
|
|
1989
|
+
raise ValidationError(f"policy arm {arm_id!r} selected an ineligible family")
|
|
1990
|
+
expected_family = configured_release_families.get(selected_release_id)
|
|
1991
|
+
if expected_family is None or expected_family is not selected_family:
|
|
1992
|
+
raise ValidationError(
|
|
1993
|
+
f"policy arm {arm_id!r} selected a release outside the locked release bindings"
|
|
1994
|
+
)
|
|
1995
|
+
|
|
1996
|
+
def _arm_outcomes(
|
|
1997
|
+
self,
|
|
1998
|
+
cell: ExperimentCell,
|
|
1999
|
+
*,
|
|
2000
|
+
arm_specs: Mapping[str, ExperimentArmSpec],
|
|
2001
|
+
artifacts: Mapping[str, ExternalArmPayload],
|
|
2002
|
+
pair_records: Sequence[tuple[str, AtomicReplayRequest, AtomicPairedReplayRecord]],
|
|
2003
|
+
target_binding: _TargetBinding,
|
|
2004
|
+
) -> tuple[Mapping[str, Any], ...]:
|
|
2005
|
+
baseline_id = self._config.baseline_arm_id
|
|
2006
|
+
baseline = pair_records[0][2].baseline
|
|
2007
|
+
receipt_hashes = tuple(
|
|
2008
|
+
content_hash(record.receipt) for _arm_id, _request, record in pair_records
|
|
2009
|
+
)
|
|
2010
|
+
records = {arm_id: record for arm_id, _request, record in pair_records}
|
|
2011
|
+
outcomes: list[Mapping[str, Any]] = []
|
|
2012
|
+
for arm_id in cell.arm_ids:
|
|
2013
|
+
spec = arm_specs[arm_id]
|
|
2014
|
+
artifact = artifacts[arm_id]
|
|
2015
|
+
if arm_id == baseline_id:
|
|
2016
|
+
result = baseline
|
|
2017
|
+
receipt_data: Mapping[str, Any] = {
|
|
2018
|
+
"atomic_pair_receipt_sha256s": receipt_hashes,
|
|
2019
|
+
}
|
|
2020
|
+
else:
|
|
2021
|
+
result = records[arm_id].treatment
|
|
2022
|
+
receipt_data = {
|
|
2023
|
+
"atomic_pair_receipt_sha256": content_hash(records[arm_id].receipt),
|
|
2024
|
+
}
|
|
2025
|
+
metrics, composition_receipt = _compose_valid_success(
|
|
2026
|
+
result.metrics,
|
|
2027
|
+
benchmark_id=cell.benchmark_id,
|
|
2028
|
+
arm_id=arm_id,
|
|
2029
|
+
trace_hash=result.trace_hash,
|
|
2030
|
+
bridge_lock=target_binding.bridge_lock,
|
|
2031
|
+
outcome_contract=target_binding.outcome_contract,
|
|
2032
|
+
)
|
|
2033
|
+
outcome: dict[str, Any] = {
|
|
2034
|
+
"arm_id": arm_id,
|
|
2035
|
+
"release_id": spec.release_id,
|
|
2036
|
+
"family": spec.family.value,
|
|
2037
|
+
"profile_kind": spec.profile_kind.value,
|
|
2038
|
+
"status": "succeeded",
|
|
2039
|
+
"metrics": metrics,
|
|
2040
|
+
"trace_hash": result.trace_hash,
|
|
2041
|
+
"usage": result.usage.as_dict(),
|
|
2042
|
+
"payload_sha256": artifact.payload_sha256,
|
|
2043
|
+
"valid_success_composition": composition_receipt,
|
|
2044
|
+
**receipt_data,
|
|
2045
|
+
}
|
|
2046
|
+
if artifact.selector_decision is not None:
|
|
2047
|
+
outcome["decision"] = artifact.selector_decision
|
|
2048
|
+
outcome["selector_decision_sha256"] = content_hash(artifact.selector_decision)
|
|
2049
|
+
outcome["selector_input_receipt"] = {
|
|
2050
|
+
"state_hash": artifact.selector_input_state_hash,
|
|
2051
|
+
"feature_hash": artifact.selector_feature_hash,
|
|
2052
|
+
}
|
|
2053
|
+
outcomes.append(freeze_mapping(outcome))
|
|
2054
|
+
return tuple(outcomes)
|
|
2055
|
+
|
|
2056
|
+
def _pre_action_receipt(self, state: FrozenAgentState) -> Mapping[str, Any]:
|
|
2057
|
+
observation = self._feature_extractor.extract(state)
|
|
2058
|
+
features = observation.features
|
|
2059
|
+
return freeze_mapping(
|
|
2060
|
+
{
|
|
2061
|
+
"state_hash": observation.state_hash,
|
|
2062
|
+
"feature_hash": observation.feature_hash,
|
|
2063
|
+
"feature_schema_id": self._feature_extractor.feature_schema_id,
|
|
2064
|
+
"feature_schema_hash": self._feature_extractor.feature_schema_hash,
|
|
2065
|
+
"provenance_hash": content_hash(observation.provenance),
|
|
2066
|
+
"defect_scores": {
|
|
2067
|
+
SemanticFamily.REFORMULATE.value: (features.reformulation_need_score),
|
|
2068
|
+
SemanticFamily.DECOMPOSE.value: features.multi_step_score,
|
|
2069
|
+
SemanticFamily.ACQUIRE.value: features.retrieval_need_score,
|
|
2070
|
+
SemanticFamily.CLARIFY.value: features.ambiguity_score,
|
|
2071
|
+
SemanticFamily.CURATE.value: features.context_pressure,
|
|
2072
|
+
},
|
|
2073
|
+
}
|
|
2074
|
+
)
|
|
2075
|
+
|
|
2076
|
+
|
|
2077
|
+
def factory() -> ExternalArenaExperimentDriver:
|
|
2078
|
+
"""Construct the driver from explicit path and exact-byte SHA-256 bindings."""
|
|
2079
|
+
|
|
2080
|
+
path = os.environ.get(CONFIG_PATH_ENV)
|
|
2081
|
+
expected_sha256 = os.environ.get(CONFIG_SHA256_ENV)
|
|
2082
|
+
if not path or not expected_sha256:
|
|
2083
|
+
raise ArtifactError(
|
|
2084
|
+
f"set both {CONFIG_PATH_ENV} and {CONFIG_SHA256_ENV} before loading "
|
|
2085
|
+
"the external arena driver"
|
|
2086
|
+
)
|
|
2087
|
+
return ExternalArenaExperimentDriver.from_file(
|
|
2088
|
+
path,
|
|
2089
|
+
expected_sha256=expected_sha256,
|
|
2090
|
+
)
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def _target_binding(value: Any, *, path: str) -> _TargetBinding:
|
|
2094
|
+
data = _strict_mapping(
|
|
2095
|
+
value,
|
|
2096
|
+
path=path,
|
|
2097
|
+
keys={
|
|
2098
|
+
"benchmark_id",
|
|
2099
|
+
"split",
|
|
2100
|
+
"source_revision",
|
|
2101
|
+
"checkpoint_builder",
|
|
2102
|
+
"bridge_builder",
|
|
2103
|
+
"bridge_lock",
|
|
2104
|
+
"outcome_contract",
|
|
2105
|
+
},
|
|
2106
|
+
required={
|
|
2107
|
+
"benchmark_id",
|
|
2108
|
+
"split",
|
|
2109
|
+
"source_revision",
|
|
2110
|
+
"checkpoint_builder",
|
|
2111
|
+
"bridge_builder",
|
|
2112
|
+
"bridge_lock",
|
|
2113
|
+
"outcome_contract",
|
|
2114
|
+
},
|
|
2115
|
+
)
|
|
2116
|
+
benchmark_id = _text(data["benchmark_id"], f"{path}.benchmark_id")
|
|
2117
|
+
source_revision = _text(data["source_revision"], f"{path}.source_revision")
|
|
2118
|
+
if benchmark_id not in _SUPPORTED_ARENAS:
|
|
2119
|
+
raise ValidationError(f"{path}.benchmark_id is not a supported external arena")
|
|
2120
|
+
official = {spec.cell_id: spec.source_revision for spec in interactive_benchmark_specs()}.get(
|
|
2121
|
+
benchmark_id
|
|
2122
|
+
)
|
|
2123
|
+
if benchmark_id == "tau3.retail":
|
|
2124
|
+
# Retail has its own checkpoint/provider path. Registering metadata is
|
|
2125
|
+
# not certification by the historical five-family interactive factory.
|
|
2126
|
+
official = BenchmarkRegistry.default().get(benchmark_id).source_revision
|
|
2127
|
+
if official is not None and source_revision != official:
|
|
2128
|
+
raise ValidationError(
|
|
2129
|
+
f"{path}.source_revision differs from the packaged official source lock"
|
|
2130
|
+
)
|
|
2131
|
+
outcome_contract = _OutcomeContract.from_mapping(
|
|
2132
|
+
data["outcome_contract"],
|
|
2133
|
+
path=f"{path}.outcome_contract",
|
|
2134
|
+
)
|
|
2135
|
+
expected_success_metric = _REGISTERED_OFFICIAL_SUCCESS_METRICS.get(benchmark_id)
|
|
2136
|
+
if (
|
|
2137
|
+
expected_success_metric is not None
|
|
2138
|
+
and outcome_contract.official_success_metric != expected_success_metric
|
|
2139
|
+
):
|
|
2140
|
+
raise ValidationError(
|
|
2141
|
+
f"{path}.outcome_contract.official_success_metric must match the "
|
|
2142
|
+
f"registered adapter metric {expected_success_metric!r}"
|
|
2143
|
+
)
|
|
2144
|
+
return _TargetBinding(
|
|
2145
|
+
benchmark_id=benchmark_id,
|
|
2146
|
+
split=_text(data["split"], f"{path}.split"),
|
|
2147
|
+
source_revision=source_revision,
|
|
2148
|
+
checkpoint_builder=_LockedBuilderSpec.from_mapping(
|
|
2149
|
+
data["checkpoint_builder"],
|
|
2150
|
+
path=f"{path}.checkpoint_builder",
|
|
2151
|
+
),
|
|
2152
|
+
bridge_builder=_LockedBuilderSpec.from_mapping(
|
|
2153
|
+
data["bridge_builder"],
|
|
2154
|
+
path=f"{path}.bridge_builder",
|
|
2155
|
+
),
|
|
2156
|
+
bridge_lock=_BridgeIdentityLock.from_mapping(
|
|
2157
|
+
data["bridge_lock"],
|
|
2158
|
+
path=f"{path}.bridge_lock",
|
|
2159
|
+
benchmark_id=benchmark_id,
|
|
2160
|
+
),
|
|
2161
|
+
outcome_contract=outcome_contract,
|
|
2162
|
+
)
|
|
2163
|
+
|
|
2164
|
+
|
|
2165
|
+
def _arm_binding(value: Any, *, path: str) -> _ArmBinding:
|
|
2166
|
+
data = _strict_mapping(
|
|
2167
|
+
value,
|
|
2168
|
+
path=path,
|
|
2169
|
+
keys={
|
|
2170
|
+
"release_id",
|
|
2171
|
+
"family",
|
|
2172
|
+
"profile_kind",
|
|
2173
|
+
"arm_kind",
|
|
2174
|
+
"payload_builder",
|
|
2175
|
+
"policy_kind",
|
|
2176
|
+
},
|
|
2177
|
+
required={"release_id", "family", "profile_kind", "payload_builder"},
|
|
2178
|
+
)
|
|
2179
|
+
try:
|
|
2180
|
+
family = SemanticFamily(_text(data["family"], f"{path}.family"))
|
|
2181
|
+
profile_kind = ProfileKind(_text(data["profile_kind"], f"{path}.profile_kind"))
|
|
2182
|
+
arm_kind = ExperimentArmKind(
|
|
2183
|
+
_text(
|
|
2184
|
+
data.get("arm_kind", ExperimentArmKind.RELEASE.value),
|
|
2185
|
+
f"{path}.arm_kind",
|
|
2186
|
+
)
|
|
2187
|
+
)
|
|
2188
|
+
except ValueError as exc:
|
|
2189
|
+
raise ValidationError(f"{path} contains an unknown enum value") from exc
|
|
2190
|
+
policy_kind = data.get("policy_kind")
|
|
2191
|
+
if policy_kind is not None:
|
|
2192
|
+
policy_kind = _text(policy_kind, f"{path}.policy_kind")
|
|
2193
|
+
return _ArmBinding(
|
|
2194
|
+
release_id=_text(data["release_id"], f"{path}.release_id"),
|
|
2195
|
+
family=family,
|
|
2196
|
+
profile_kind=profile_kind,
|
|
2197
|
+
arm_kind=arm_kind,
|
|
2198
|
+
payload_builder=_LockedBuilderSpec.from_mapping(
|
|
2199
|
+
data["payload_builder"],
|
|
2200
|
+
path=f"{path}.payload_builder",
|
|
2201
|
+
),
|
|
2202
|
+
policy_kind=policy_kind,
|
|
2203
|
+
)
|
|
2204
|
+
|
|
2205
|
+
|
|
2206
|
+
def _resolve_locked_builder(
|
|
2207
|
+
spec: _LockedBuilderSpec,
|
|
2208
|
+
*,
|
|
2209
|
+
expected_kind: str,
|
|
2210
|
+
) -> LockedExternalArenaBuilder:
|
|
2211
|
+
if expected_kind not in _BUILDER_KINDS: # pragma: no cover - internal invariant
|
|
2212
|
+
raise ValueError("unknown external builder kind")
|
|
2213
|
+
factory_candidate = resolve_driver_entrypoint(spec.entrypoint)
|
|
2214
|
+
if not callable(factory_candidate):
|
|
2215
|
+
raise ArtifactError("external builder entrypoint must be a zero-argument factory")
|
|
2216
|
+
try:
|
|
2217
|
+
builder = factory_candidate()
|
|
2218
|
+
except TypeError as exc:
|
|
2219
|
+
raise ArtifactError("external builder entrypoint must be a zero-argument factory") from exc
|
|
2220
|
+
if not isinstance(builder, LockedExternalArenaBuilder):
|
|
2221
|
+
raise ArtifactError("external component does not implement the locked builder contract")
|
|
2222
|
+
if builder.builder_kind != expected_kind:
|
|
2223
|
+
raise ArtifactError(
|
|
2224
|
+
f"external builder kind is {builder.builder_kind!r}, expected {expected_kind!r}"
|
|
2225
|
+
)
|
|
2226
|
+
identity = _builder_fingerprint(builder, label=f"{expected_kind} builder")
|
|
2227
|
+
if content_hash(identity) != spec.fingerprint_sha256:
|
|
2228
|
+
raise ArtifactError(f"{expected_kind} builder fingerprint checksum mismatch")
|
|
2229
|
+
return builder
|
|
2230
|
+
|
|
2231
|
+
|
|
2232
|
+
def _builder_fingerprint(
|
|
2233
|
+
builder: LockedExternalArenaBuilder,
|
|
2234
|
+
*,
|
|
2235
|
+
label: str,
|
|
2236
|
+
) -> Mapping[str, Any]:
|
|
2237
|
+
fingerprint = builder.fingerprint()
|
|
2238
|
+
if not isinstance(fingerprint, Mapping) or not fingerprint:
|
|
2239
|
+
raise ValidationError(f"{label} fingerprint must be a non-empty mapping")
|
|
2240
|
+
try:
|
|
2241
|
+
return freeze_mapping(fingerprint)
|
|
2242
|
+
except (TypeError, ValueError) as exc:
|
|
2243
|
+
raise ValidationError(f"{label} fingerprint must be canonical JSON") from exc
|
|
2244
|
+
|
|
2245
|
+
|
|
2246
|
+
def _builder_preflight(
|
|
2247
|
+
builder: LockedExternalArenaBuilder,
|
|
2248
|
+
parameters: Mapping[str, Any],
|
|
2249
|
+
) -> tuple[str, ...]:
|
|
2250
|
+
blockers = builder.preflight(parameters)
|
|
2251
|
+
if isinstance(blockers, (str, bytes)) or not isinstance(blockers, Sequence):
|
|
2252
|
+
raise ValidationError("external component preflight must return a sequence of strings")
|
|
2253
|
+
normalized = tuple(blockers)
|
|
2254
|
+
if any(not isinstance(item, str) or not item.strip() for item in normalized):
|
|
2255
|
+
raise ValidationError("external component preflight blockers must be non-empty strings")
|
|
2256
|
+
return normalized
|
|
2257
|
+
|
|
2258
|
+
|
|
2259
|
+
def _compose_valid_success(
|
|
2260
|
+
raw_metrics: Mapping[str, float],
|
|
2261
|
+
*,
|
|
2262
|
+
benchmark_id: str,
|
|
2263
|
+
arm_id: str,
|
|
2264
|
+
trace_hash: str,
|
|
2265
|
+
bridge_lock: _BridgeIdentityLock,
|
|
2266
|
+
outcome_contract: _OutcomeContract,
|
|
2267
|
+
) -> tuple[Mapping[str, float], Mapping[str, Any]]:
|
|
2268
|
+
"""Create the primary valid outcome only inside the trusted host boundary."""
|
|
2269
|
+
|
|
2270
|
+
if _VALID_SUCCESS_METRIC in raw_metrics:
|
|
2271
|
+
raise ValidationError(f"external bridge arm {arm_id!r} must not precompute valid_success")
|
|
2272
|
+
required = {
|
|
2273
|
+
outcome_contract.official_success_metric,
|
|
2274
|
+
_CONTRACT_PRESERVED_METRIC,
|
|
2275
|
+
_PRIVILEGED_LEAKAGE_METRIC,
|
|
2276
|
+
}
|
|
2277
|
+
missing = sorted(required - set(raw_metrics))
|
|
2278
|
+
if missing:
|
|
2279
|
+
raise ValidationError(
|
|
2280
|
+
f"external bridge arm {arm_id!r} is missing valid-success components: "
|
|
2281
|
+
+ ", ".join(missing)
|
|
2282
|
+
)
|
|
2283
|
+
official_success = _unit_interval_metric(
|
|
2284
|
+
raw_metrics[outcome_contract.official_success_metric],
|
|
2285
|
+
label=(
|
|
2286
|
+
f"external bridge arm {arm_id!r} metric {outcome_contract.official_success_metric!r}"
|
|
2287
|
+
),
|
|
2288
|
+
)
|
|
2289
|
+
contract_preserved = _unit_interval_metric(
|
|
2290
|
+
raw_metrics[_CONTRACT_PRESERVED_METRIC],
|
|
2291
|
+
label=f"external bridge arm {arm_id!r} metric {_CONTRACT_PRESERVED_METRIC!r}",
|
|
2292
|
+
)
|
|
2293
|
+
privileged_leakage = _unit_interval_metric(
|
|
2294
|
+
raw_metrics[_PRIVILEGED_LEAKAGE_METRIC],
|
|
2295
|
+
label=f"external bridge arm {arm_id!r} metric {_PRIVILEGED_LEAKAGE_METRIC!r}",
|
|
2296
|
+
)
|
|
2297
|
+
valid_success = official_success * contract_preserved * (1.0 - privileged_leakage)
|
|
2298
|
+
receipt_body = {
|
|
2299
|
+
"schema_version": 1,
|
|
2300
|
+
"formula_id": outcome_contract.formula_id,
|
|
2301
|
+
"benchmark_id": benchmark_id,
|
|
2302
|
+
"arm_id": arm_id,
|
|
2303
|
+
"official_success_metric": outcome_contract.official_success_metric,
|
|
2304
|
+
"components": {
|
|
2305
|
+
"official_success": official_success,
|
|
2306
|
+
_CONTRACT_PRESERVED_METRIC: contract_preserved,
|
|
2307
|
+
_PRIVILEGED_LEAKAGE_METRIC: privileged_leakage,
|
|
2308
|
+
},
|
|
2309
|
+
_VALID_SUCCESS_METRIC: valid_success,
|
|
2310
|
+
"outcome_contract_sha256": content_hash(outcome_contract),
|
|
2311
|
+
"scorer_id": bridge_lock.scorer_id,
|
|
2312
|
+
"scorer_revision": bridge_lock.scorer_revision,
|
|
2313
|
+
"trace_hash": trace_hash,
|
|
2314
|
+
}
|
|
2315
|
+
composition_receipt = freeze_mapping(
|
|
2316
|
+
{
|
|
2317
|
+
**receipt_body,
|
|
2318
|
+
"composition_receipt_sha256": content_hash(receipt_body),
|
|
2319
|
+
}
|
|
2320
|
+
)
|
|
2321
|
+
metrics = dict(raw_metrics)
|
|
2322
|
+
metrics[_VALID_SUCCESS_METRIC] = valid_success
|
|
2323
|
+
return freeze_mapping(metrics), composition_receipt
|
|
2324
|
+
|
|
2325
|
+
|
|
2326
|
+
def _unit_interval_metric(value: Any, *, label: str) -> float:
|
|
2327
|
+
numeric = _finite_number(value, label=label)
|
|
2328
|
+
if not 0.0 <= numeric <= 1.0:
|
|
2329
|
+
raise ValidationError(f"{label} must be in [0, 1]")
|
|
2330
|
+
return numeric
|
|
2331
|
+
|
|
2332
|
+
|
|
2333
|
+
def _baseline_consistency_hash(record: AtomicPairedReplayRecord) -> str:
|
|
2334
|
+
baseline_receipts = tuple(arm for arm in record.receipt.arms if arm.role == "baseline")
|
|
2335
|
+
if len(baseline_receipts) != 1: # pragma: no cover - typed receipt invariant
|
|
2336
|
+
raise ValidationError("atomic pair lacks one baseline receipt")
|
|
2337
|
+
baseline_receipt = baseline_receipts[0]
|
|
2338
|
+
return content_hash(
|
|
2339
|
+
{
|
|
2340
|
+
"metrics": record.baseline.metrics,
|
|
2341
|
+
"trace_hash": record.baseline.trace_hash,
|
|
2342
|
+
"usage": record.baseline.usage,
|
|
2343
|
+
"receipt": {
|
|
2344
|
+
"benchmark_id": record.receipt.benchmark_id,
|
|
2345
|
+
"checkpoint_id": record.receipt.checkpoint_id,
|
|
2346
|
+
"source_revision": record.receipt.source_revision,
|
|
2347
|
+
"bridge_id": record.receipt.bridge_id,
|
|
2348
|
+
"bridge_revision": record.receipt.bridge_revision,
|
|
2349
|
+
"scorer_id": record.receipt.scorer_id,
|
|
2350
|
+
"scorer_revision": record.receipt.scorer_revision,
|
|
2351
|
+
"origin_state_hash": record.receipt.origin_state_hash,
|
|
2352
|
+
"replay_identity": record.receipt.replay_identity,
|
|
2353
|
+
"seed": record.receipt.seed,
|
|
2354
|
+
"locked_dependency_hashes": (record.receipt.locked_dependency_hashes),
|
|
2355
|
+
"baseline_arm": {
|
|
2356
|
+
"role": baseline_receipt.role,
|
|
2357
|
+
"arm_id": baseline_receipt.arm_id,
|
|
2358
|
+
"origin_state_hash": baseline_receipt.origin_state_hash,
|
|
2359
|
+
"replay_identity": baseline_receipt.replay_identity,
|
|
2360
|
+
"seed": baseline_receipt.seed,
|
|
2361
|
+
"locked_dependency_hashes": (baseline_receipt.locked_dependency_hashes),
|
|
2362
|
+
"completed": baseline_receipt.completed,
|
|
2363
|
+
},
|
|
2364
|
+
},
|
|
2365
|
+
}
|
|
2366
|
+
)
|
|
2367
|
+
|
|
2368
|
+
|
|
2369
|
+
def _validated_policy_decision(value: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
2370
|
+
if not isinstance(value, Mapping):
|
|
2371
|
+
raise ValidationError("selector decision receipt must be an object")
|
|
2372
|
+
common = {
|
|
2373
|
+
"selected_family",
|
|
2374
|
+
"selected_release_id",
|
|
2375
|
+
"abstained",
|
|
2376
|
+
"predicted_gain",
|
|
2377
|
+
"confidence",
|
|
2378
|
+
}
|
|
2379
|
+
identity_fields = set(value) - common
|
|
2380
|
+
if identity_fields not in (
|
|
2381
|
+
{"selector_bundle_hash"},
|
|
2382
|
+
{"policy_manifest_sha256"},
|
|
2383
|
+
):
|
|
2384
|
+
raise ValidationError("selector decision receipt fields are invalid")
|
|
2385
|
+
try:
|
|
2386
|
+
family = SemanticFamily(value["selected_family"])
|
|
2387
|
+
except (TypeError, ValueError) as exc:
|
|
2388
|
+
raise ValidationError("selector decision selected_family is invalid") from exc
|
|
2389
|
+
if family not in {*_CORE_FAMILIES, SemanticFamily.NO_OP}:
|
|
2390
|
+
raise ValidationError("selector decision selected_family is not executable")
|
|
2391
|
+
release_id = value["selected_release_id"]
|
|
2392
|
+
_require_text(release_id, label="selector decision selected_release_id")
|
|
2393
|
+
abstained = value["abstained"]
|
|
2394
|
+
if not isinstance(abstained, bool):
|
|
2395
|
+
raise ValidationError("selector decision abstained must be boolean")
|
|
2396
|
+
if abstained is not (family is SemanticFamily.NO_OP):
|
|
2397
|
+
raise ValidationError("selector decision abstention and selected_family are inconsistent")
|
|
2398
|
+
if family is SemanticFamily.NO_OP and release_id != "baseline.no_op":
|
|
2399
|
+
raise ValidationError("selector NO_OP decision must select baseline.no_op")
|
|
2400
|
+
predicted_gain = _finite_number(
|
|
2401
|
+
value["predicted_gain"],
|
|
2402
|
+
label="selector decision predicted_gain",
|
|
2403
|
+
)
|
|
2404
|
+
confidence = _finite_number(
|
|
2405
|
+
value["confidence"],
|
|
2406
|
+
label="selector decision confidence",
|
|
2407
|
+
)
|
|
2408
|
+
if not 0 <= confidence <= 1:
|
|
2409
|
+
raise ValidationError("selector decision confidence must be in [0, 1]")
|
|
2410
|
+
identity_field = next(iter(identity_fields))
|
|
2411
|
+
artifact_hash = value[identity_field]
|
|
2412
|
+
_require_sha256(
|
|
2413
|
+
artifact_hash,
|
|
2414
|
+
label=f"selector decision {identity_field}",
|
|
2415
|
+
)
|
|
2416
|
+
return freeze_mapping(
|
|
2417
|
+
{
|
|
2418
|
+
"selected_family": family.value,
|
|
2419
|
+
"selected_release_id": release_id,
|
|
2420
|
+
"abstained": abstained,
|
|
2421
|
+
"predicted_gain": predicted_gain,
|
|
2422
|
+
"confidence": confidence,
|
|
2423
|
+
identity_field: artifact_hash,
|
|
2424
|
+
}
|
|
2425
|
+
)
|
|
2426
|
+
|
|
2427
|
+
|
|
2428
|
+
def _read_manifest_file_binding(
|
|
2429
|
+
manifest: ExperimentManifest,
|
|
2430
|
+
*,
|
|
2431
|
+
environment_name: Any,
|
|
2432
|
+
checksum_environment_name: Any,
|
|
2433
|
+
label: str,
|
|
2434
|
+
) -> tuple[bytes, str]:
|
|
2435
|
+
for value, field_label in (
|
|
2436
|
+
(environment_name, "file environment"),
|
|
2437
|
+
(checksum_environment_name, "checksum environment"),
|
|
2438
|
+
):
|
|
2439
|
+
if not isinstance(value, str) or _ENVIRONMENT_NAME.fullmatch(value) is None:
|
|
2440
|
+
raise ValidationError(f"{label} {field_label} is invalid")
|
|
2441
|
+
matches = tuple(
|
|
2442
|
+
prerequisite
|
|
2443
|
+
for prerequisite in manifest.prerequisites
|
|
2444
|
+
if prerequisite.kind is PrerequisiteKind.FILE_ENV
|
|
2445
|
+
and prerequisite.name == environment_name
|
|
2446
|
+
and prerequisite.sha256_env == checksum_environment_name
|
|
2447
|
+
)
|
|
2448
|
+
if len(matches) != 1:
|
|
2449
|
+
raise ValidationError(
|
|
2450
|
+
f"{label} is not declared as one exact checksum-bound FILE_ENV prerequisite"
|
|
2451
|
+
)
|
|
2452
|
+
raw_path = os.environ.get(environment_name)
|
|
2453
|
+
expected_sha256 = os.environ.get(checksum_environment_name)
|
|
2454
|
+
if not raw_path or not expected_sha256:
|
|
2455
|
+
raise ArtifactError(f"{label} environment binding is missing")
|
|
2456
|
+
_require_sha256(expected_sha256, label=f"{label} expected SHA-256")
|
|
2457
|
+
source = Path(raw_path).expanduser()
|
|
2458
|
+
try:
|
|
2459
|
+
if not source.is_file():
|
|
2460
|
+
raise ArtifactError(f"{label} does not identify an existing file")
|
|
2461
|
+
if source.stat().st_size > _MAX_POLICY_ARTIFACT_BYTES:
|
|
2462
|
+
raise ArtifactError(f"{label} exceeds the size limit")
|
|
2463
|
+
raw = source.read_bytes()
|
|
2464
|
+
except OSError as exc:
|
|
2465
|
+
raise ArtifactError(f"{label} cannot be read") from exc
|
|
2466
|
+
observed_sha256 = hashlib.sha256(raw).hexdigest()
|
|
2467
|
+
if observed_sha256 != expected_sha256:
|
|
2468
|
+
raise ArtifactError(f"{label} checksum differs from its declared checksum")
|
|
2469
|
+
return raw, observed_sha256
|
|
2470
|
+
|
|
2471
|
+
|
|
2472
|
+
def _load_frozen_selector_bundle(
|
|
2473
|
+
raw: bytes,
|
|
2474
|
+
*,
|
|
2475
|
+
identity_environment_name: Any,
|
|
2476
|
+
manifest: ExperimentManifest,
|
|
2477
|
+
label: str,
|
|
2478
|
+
) -> SelectorArtifactBundle:
|
|
2479
|
+
if (
|
|
2480
|
+
not isinstance(identity_environment_name, str)
|
|
2481
|
+
or _ENVIRONMENT_NAME.fullmatch(identity_environment_name) is None
|
|
2482
|
+
):
|
|
2483
|
+
raise ValidationError(f"{label} identity environment is invalid")
|
|
2484
|
+
identity_prerequisites = tuple(
|
|
2485
|
+
prerequisite
|
|
2486
|
+
for prerequisite in manifest.prerequisites
|
|
2487
|
+
if prerequisite.kind is PrerequisiteKind.ENV
|
|
2488
|
+
and prerequisite.name == identity_environment_name
|
|
2489
|
+
)
|
|
2490
|
+
if len(identity_prerequisites) != 1:
|
|
2491
|
+
raise ValidationError(f"{label} identity is not declared as one exact ENV prerequisite")
|
|
2492
|
+
identity_raw = os.environ.get(identity_environment_name)
|
|
2493
|
+
if not identity_raw:
|
|
2494
|
+
raise ArtifactError(f"{label} identity environment is missing")
|
|
2495
|
+
identity = _strict_mapping(
|
|
2496
|
+
_strict_json(identity_raw.encode("utf-8")),
|
|
2497
|
+
path=f"{label} identity",
|
|
2498
|
+
keys={"selector_id", "revision", "sha256"},
|
|
2499
|
+
required={"selector_id", "revision", "sha256"},
|
|
2500
|
+
)
|
|
2501
|
+
try:
|
|
2502
|
+
bundle = SelectorArtifactBundle.from_artifact(_strict_json(raw))
|
|
2503
|
+
except (TypeError, ValueError) as exc:
|
|
2504
|
+
raise ArtifactError(f"{label} failed strict SelectorArtifactBundle parsing") from exc
|
|
2505
|
+
if bundle.stage is not SelectorArtifactStage.FROZEN:
|
|
2506
|
+
raise ArtifactError(f"{label} stage must be frozen")
|
|
2507
|
+
selector_id = _text(identity["selector_id"], f"{label} identity.selector_id")
|
|
2508
|
+
revision = _text(identity["revision"], f"{label} identity.revision")
|
|
2509
|
+
sha256 = _text(identity["sha256"], f"{label} identity.sha256")
|
|
2510
|
+
_require_sha256(sha256, label=f"{label} identity.sha256")
|
|
2511
|
+
if (
|
|
2512
|
+
selector_id != bundle.selector_card_id
|
|
2513
|
+
or revision != bundle.revision
|
|
2514
|
+
or sha256 != bundle.bundle_hash
|
|
2515
|
+
):
|
|
2516
|
+
raise ArtifactError(f"{label} identity differs from its parsed frozen bundle")
|
|
2517
|
+
return bundle
|
|
2518
|
+
|
|
2519
|
+
|
|
2520
|
+
def _validate_selector_policy_provenance(
|
|
2521
|
+
bundle: SelectorArtifactBundle,
|
|
2522
|
+
*,
|
|
2523
|
+
policy_manifest: _SelectorPolicyManifest,
|
|
2524
|
+
label: str,
|
|
2525
|
+
) -> None:
|
|
2526
|
+
expected = {
|
|
2527
|
+
"training_table_sha256": policy_manifest.training_table_sha256,
|
|
2528
|
+
"split_provenance_hash": policy_manifest.split_provenance_hash,
|
|
2529
|
+
"comparison_profile_hash": policy_manifest.comparison_profile_hash,
|
|
2530
|
+
"representative_by_family": dict(policy_manifest.representative_by_family),
|
|
2531
|
+
}
|
|
2532
|
+
if any(getattr(bundle, field) != value for field, value in expected.items()):
|
|
2533
|
+
raise ValidationError(
|
|
2534
|
+
f"{label} selector bundle differs from policy-manifest "
|
|
2535
|
+
"training/profile/representative provenance"
|
|
2536
|
+
)
|
|
2537
|
+
discovery = bundle.fit_lineage.get("discovery")
|
|
2538
|
+
if (
|
|
2539
|
+
not isinstance(discovery, Mapping)
|
|
2540
|
+
or discovery.get("revision") != policy_manifest.discovery_revision
|
|
2541
|
+
or discovery.get("sha256") != policy_manifest.discovery_sha256
|
|
2542
|
+
):
|
|
2543
|
+
raise ValidationError(
|
|
2544
|
+
f"{label} selector bundle fit lineage differs from policy-manifest discovery"
|
|
2545
|
+
)
|
|
2546
|
+
if bundle.calibration_receipt.get("discovery_hash") != policy_manifest.discovery_sha256:
|
|
2547
|
+
raise ValidationError(
|
|
2548
|
+
f"{label} selector calibration receipt differs from policy-manifest discovery"
|
|
2549
|
+
)
|
|
2550
|
+
|
|
2551
|
+
|
|
2552
|
+
def _policy_representatives_from_mapping(
|
|
2553
|
+
value: Any,
|
|
2554
|
+
*,
|
|
2555
|
+
path: str,
|
|
2556
|
+
) -> Mapping[SemanticFamily, str]:
|
|
2557
|
+
raw = _nonempty_object(value, path=path)
|
|
2558
|
+
representatives: dict[SemanticFamily, str] = {}
|
|
2559
|
+
for name, release_id in raw.items():
|
|
2560
|
+
try:
|
|
2561
|
+
family = SemanticFamily(_text(name, f"{path} family"))
|
|
2562
|
+
except ValueError as exc:
|
|
2563
|
+
raise ValidationError(f"{path} contains an unknown family") from exc
|
|
2564
|
+
representatives[family] = _text(
|
|
2565
|
+
release_id,
|
|
2566
|
+
f"{path}[{name!r}]",
|
|
2567
|
+
)
|
|
2568
|
+
return _validated_policy_representatives(representatives)
|
|
2569
|
+
|
|
2570
|
+
|
|
2571
|
+
def _validated_policy_representatives(
|
|
2572
|
+
value: Mapping[SemanticFamily, str],
|
|
2573
|
+
) -> dict[SemanticFamily, str]:
|
|
2574
|
+
representatives = dict(value)
|
|
2575
|
+
if (
|
|
2576
|
+
not representatives
|
|
2577
|
+
or any(
|
|
2578
|
+
family not in _CORE_FAMILIES
|
|
2579
|
+
or not isinstance(release_id, str)
|
|
2580
|
+
or not release_id.strip()
|
|
2581
|
+
or release_id != release_id.strip()
|
|
2582
|
+
for family, release_id in representatives.items()
|
|
2583
|
+
)
|
|
2584
|
+
or len(set(representatives.values())) != len(representatives)
|
|
2585
|
+
):
|
|
2586
|
+
raise ValidationError("policy manifest representatives require unique core-family releases")
|
|
2587
|
+
return representatives
|
|
2588
|
+
|
|
2589
|
+
|
|
2590
|
+
def _validate_external_gate_receipt(
|
|
2591
|
+
value: Mapping[str, Any],
|
|
2592
|
+
*,
|
|
2593
|
+
benchmark_id: str,
|
|
2594
|
+
split: str,
|
|
2595
|
+
source_revision: str,
|
|
2596
|
+
manifest_hash: str,
|
|
2597
|
+
driver_config_sha256: str,
|
|
2598
|
+
unit_inventory_sha256: str,
|
|
2599
|
+
bridge_lock: _BridgeIdentityLock,
|
|
2600
|
+
reset_mode: str,
|
|
2601
|
+
) -> Mapping[str, Any]:
|
|
2602
|
+
path = f"external gate receipt for {benchmark_id}"
|
|
2603
|
+
data = _strict_mapping(
|
|
2604
|
+
value,
|
|
2605
|
+
path=path,
|
|
2606
|
+
keys={
|
|
2607
|
+
"schema_version",
|
|
2608
|
+
"artifact",
|
|
2609
|
+
"status",
|
|
2610
|
+
"benchmark_id",
|
|
2611
|
+
"split",
|
|
2612
|
+
"source_revision",
|
|
2613
|
+
"manifest_hash",
|
|
2614
|
+
"driver_config_sha256",
|
|
2615
|
+
"unit_inventory_sha256",
|
|
2616
|
+
"bridge",
|
|
2617
|
+
"scorer",
|
|
2618
|
+
"verification",
|
|
2619
|
+
"approval",
|
|
2620
|
+
},
|
|
2621
|
+
required={
|
|
2622
|
+
"schema_version",
|
|
2623
|
+
"artifact",
|
|
2624
|
+
"status",
|
|
2625
|
+
"benchmark_id",
|
|
2626
|
+
"split",
|
|
2627
|
+
"source_revision",
|
|
2628
|
+
"manifest_hash",
|
|
2629
|
+
"driver_config_sha256",
|
|
2630
|
+
"unit_inventory_sha256",
|
|
2631
|
+
"bridge",
|
|
2632
|
+
"scorer",
|
|
2633
|
+
"verification",
|
|
2634
|
+
"approval",
|
|
2635
|
+
},
|
|
2636
|
+
)
|
|
2637
|
+
if (
|
|
2638
|
+
data["schema_version"] != 1
|
|
2639
|
+
or data["artifact"] != "agentinputlab.external_arena_gate_receipt"
|
|
2640
|
+
or data["status"] != "approved"
|
|
2641
|
+
):
|
|
2642
|
+
raise ValidationError(f"{path} is not an approved schema-v1 gate receipt")
|
|
2643
|
+
expected_identity = {
|
|
2644
|
+
"benchmark_id": benchmark_id,
|
|
2645
|
+
"split": split,
|
|
2646
|
+
"source_revision": source_revision,
|
|
2647
|
+
"manifest_hash": manifest_hash,
|
|
2648
|
+
"driver_config_sha256": driver_config_sha256,
|
|
2649
|
+
"unit_inventory_sha256": unit_inventory_sha256,
|
|
2650
|
+
}
|
|
2651
|
+
for name in ("manifest_hash", "driver_config_sha256", "unit_inventory_sha256"):
|
|
2652
|
+
_require_sha256(expected_identity[name], label=f"expected {name}")
|
|
2653
|
+
for name, expected in expected_identity.items():
|
|
2654
|
+
if _text(data[name], f"{path}.{name}") != expected:
|
|
2655
|
+
raise ValidationError(f"{path}.{name} differs from the frozen experiment")
|
|
2656
|
+
bridge = _strict_mapping(
|
|
2657
|
+
data["bridge"],
|
|
2658
|
+
path=f"{path}.bridge",
|
|
2659
|
+
keys={"bridge_id", "bridge_revision"},
|
|
2660
|
+
required={"bridge_id", "bridge_revision"},
|
|
2661
|
+
)
|
|
2662
|
+
scorer = _strict_mapping(
|
|
2663
|
+
data["scorer"],
|
|
2664
|
+
path=f"{path}.scorer",
|
|
2665
|
+
keys={"scorer_id", "scorer_revision"},
|
|
2666
|
+
required={"scorer_id", "scorer_revision"},
|
|
2667
|
+
)
|
|
2668
|
+
if (
|
|
2669
|
+
bridge.get("bridge_id") != bridge_lock.bridge_id
|
|
2670
|
+
or bridge.get("bridge_revision") != bridge_lock.bridge_revision
|
|
2671
|
+
):
|
|
2672
|
+
raise ValidationError(f"{path}.bridge differs from the external driver lock")
|
|
2673
|
+
if (
|
|
2674
|
+
scorer.get("scorer_id") != bridge_lock.scorer_id
|
|
2675
|
+
or scorer.get("scorer_revision") != bridge_lock.scorer_revision
|
|
2676
|
+
):
|
|
2677
|
+
raise ValidationError(f"{path}.scorer differs from the external driver lock")
|
|
2678
|
+
verification = _strict_mapping(
|
|
2679
|
+
data["verification"],
|
|
2680
|
+
path=f"{path}.verification",
|
|
2681
|
+
keys={
|
|
2682
|
+
"atomic_replay_contract",
|
|
2683
|
+
"reset_mode",
|
|
2684
|
+
"reset_verified",
|
|
2685
|
+
"reset_receipt_sha256",
|
|
2686
|
+
"no_op_parity_verified",
|
|
2687
|
+
"no_op_parity_receipt_sha256",
|
|
2688
|
+
},
|
|
2689
|
+
required={
|
|
2690
|
+
"atomic_replay_contract",
|
|
2691
|
+
"reset_mode",
|
|
2692
|
+
"reset_verified",
|
|
2693
|
+
"reset_receipt_sha256",
|
|
2694
|
+
"no_op_parity_verified",
|
|
2695
|
+
"no_op_parity_receipt_sha256",
|
|
2696
|
+
},
|
|
2697
|
+
)
|
|
2698
|
+
if (
|
|
2699
|
+
verification["atomic_replay_contract"] != ATOMIC_EXTERNAL_PAIR_CONTRACT
|
|
2700
|
+
or verification["reset_mode"] != reset_mode
|
|
2701
|
+
or verification["reset_verified"] is not True
|
|
2702
|
+
or verification["no_op_parity_verified"] is not True
|
|
2703
|
+
):
|
|
2704
|
+
raise ValidationError(f"{path}.verification does not approve atomic reset and NO_OP parity")
|
|
2705
|
+
for name in ("reset_receipt_sha256", "no_op_parity_receipt_sha256"):
|
|
2706
|
+
_require_sha256(
|
|
2707
|
+
verification[name],
|
|
2708
|
+
label=f"{path}.verification.{name}",
|
|
2709
|
+
)
|
|
2710
|
+
approval = _strict_mapping(
|
|
2711
|
+
data["approval"],
|
|
2712
|
+
path=f"{path}.approval",
|
|
2713
|
+
keys={
|
|
2714
|
+
"approved",
|
|
2715
|
+
"approver_id",
|
|
2716
|
+
"approval_revision",
|
|
2717
|
+
"approved_at",
|
|
2718
|
+
"scope_sha256",
|
|
2719
|
+
},
|
|
2720
|
+
required={
|
|
2721
|
+
"approved",
|
|
2722
|
+
"approver_id",
|
|
2723
|
+
"approval_revision",
|
|
2724
|
+
"approved_at",
|
|
2725
|
+
"scope_sha256",
|
|
2726
|
+
},
|
|
2727
|
+
)
|
|
2728
|
+
if approval["approved"] is not True:
|
|
2729
|
+
raise ValidationError(f"{path}.approval.approved must be true")
|
|
2730
|
+
_require_text(approval["approver_id"], label=f"{path}.approval.approver_id")
|
|
2731
|
+
_require_text(approval["approved_at"], label=f"{path}.approval.approved_at")
|
|
2732
|
+
_require_sha256(
|
|
2733
|
+
approval["approval_revision"],
|
|
2734
|
+
label=f"{path}.approval.approval_revision",
|
|
2735
|
+
)
|
|
2736
|
+
_require_sha256(
|
|
2737
|
+
approval["scope_sha256"],
|
|
2738
|
+
label=f"{path}.approval.scope_sha256",
|
|
2739
|
+
)
|
|
2740
|
+
scope = {
|
|
2741
|
+
**expected_identity,
|
|
2742
|
+
"bridge": dict(bridge),
|
|
2743
|
+
"scorer": dict(scorer),
|
|
2744
|
+
"verification": dict(verification),
|
|
2745
|
+
}
|
|
2746
|
+
if approval["scope_sha256"] != content_hash(scope):
|
|
2747
|
+
raise ValidationError(f"{path}.approval.scope_sha256 differs from the approved scope")
|
|
2748
|
+
return freeze_mapping(data)
|
|
2749
|
+
|
|
2750
|
+
|
|
2751
|
+
def _validated_dependency_hashes(
|
|
2752
|
+
value: Mapping[str, str],
|
|
2753
|
+
*, benchmark_id: str = "",
|
|
2754
|
+
) -> Mapping[str, str]:
|
|
2755
|
+
# The common user simulator and NL judge are part of Retail's scientific
|
|
2756
|
+
# identity. Other arenas retain their existing exact ten-slot contract.
|
|
2757
|
+
required = _REQUIRED_SHARED_DEPENDENCY_SLOTS
|
|
2758
|
+
if benchmark_id == "tau3.retail":
|
|
2759
|
+
required = required | _RETAIL_ENVIRONMENT_DEPENDENCY_SLOTS
|
|
2760
|
+
if not isinstance(value, Mapping):
|
|
2761
|
+
raise ValidationError(
|
|
2762
|
+
"external locked dependency hashes must be an object containing "
|
|
2763
|
+
"the exact shared dependency slots"
|
|
2764
|
+
)
|
|
2765
|
+
if set(value) != required:
|
|
2766
|
+
missing = sorted(required - set(value))
|
|
2767
|
+
extra = sorted(set(value) - required)
|
|
2768
|
+
raise ValidationError(
|
|
2769
|
+
"external locked dependency hashes must contain the exact shared "
|
|
2770
|
+
f"dependency slots; missing={missing}, extra={extra}"
|
|
2771
|
+
)
|
|
2772
|
+
normalized: dict[str, str] = {}
|
|
2773
|
+
for name, digest in value.items():
|
|
2774
|
+
_require_text(name, label="external dependency name")
|
|
2775
|
+
_require_sha256(digest, label=f"external dependency {name!r}")
|
|
2776
|
+
normalized[name] = digest
|
|
2777
|
+
return freeze_mapping(normalized)
|
|
2778
|
+
|
|
2779
|
+
|
|
2780
|
+
def _anonymized_group_hash(benchmark_id: str, raw_group_id: str) -> str:
|
|
2781
|
+
_require_text(benchmark_id, label="group benchmark_id")
|
|
2782
|
+
_require_text(raw_group_id, label="group_id")
|
|
2783
|
+
return "group_" + content_hash({"benchmark_id": benchmark_id, "group_id": raw_group_id})
|
|
2784
|
+
|
|
2785
|
+
|
|
2786
|
+
def external_dependency_not_applicable_hash(slot: str) -> str:
|
|
2787
|
+
"""Return the canonical sentinel digest for an inapplicable shared slot."""
|
|
2788
|
+
|
|
2789
|
+
if slot not in _REQUIRED_SHARED_DEPENDENCY_SLOTS:
|
|
2790
|
+
raise ValidationError(f"unknown external dependency slot: {slot!r}")
|
|
2791
|
+
return _NOT_APPLICABLE_DEPENDENCY_HASHES[slot]
|
|
2792
|
+
|
|
2793
|
+
|
|
2794
|
+
def _target_key(target: ExperimentBenchmarkTarget) -> str:
|
|
2795
|
+
return f"{target.cell_id}:{target.split}"
|
|
2796
|
+
|
|
2797
|
+
|
|
2798
|
+
def _strict_json(raw: bytes) -> Mapping[str, Any]:
|
|
2799
|
+
try:
|
|
2800
|
+
parsed = json.loads(
|
|
2801
|
+
raw.decode("utf-8"),
|
|
2802
|
+
object_pairs_hook=_reject_duplicate_keys,
|
|
2803
|
+
parse_constant=lambda value: _reject_json_constant(value),
|
|
2804
|
+
)
|
|
2805
|
+
except (UnicodeError, json.JSONDecodeError) as exc:
|
|
2806
|
+
raise ArtifactError("external arena config is not strict UTF-8 JSON") from exc
|
|
2807
|
+
if not isinstance(parsed, Mapping):
|
|
2808
|
+
raise ValidationError("external arena config root must be an object")
|
|
2809
|
+
return parsed
|
|
2810
|
+
|
|
2811
|
+
|
|
2812
|
+
def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
2813
|
+
result: dict[str, Any] = {}
|
|
2814
|
+
for key, value in pairs:
|
|
2815
|
+
if key in result:
|
|
2816
|
+
raise ValidationError(f"duplicate JSON key: {key}")
|
|
2817
|
+
result[key] = value
|
|
2818
|
+
return result
|
|
2819
|
+
|
|
2820
|
+
|
|
2821
|
+
def _reject_json_constant(value: str) -> None:
|
|
2822
|
+
raise ValidationError(f"non-finite JSON constant is forbidden: {value}")
|
|
2823
|
+
|
|
2824
|
+
|
|
2825
|
+
def _strict_mapping(
|
|
2826
|
+
value: Any,
|
|
2827
|
+
*,
|
|
2828
|
+
path: str,
|
|
2829
|
+
keys: set[str],
|
|
2830
|
+
required: set[str],
|
|
2831
|
+
) -> Mapping[str, Any]:
|
|
2832
|
+
if not isinstance(value, Mapping):
|
|
2833
|
+
raise ValidationError(f"{path} must be an object")
|
|
2834
|
+
if any(not isinstance(key, str) for key in value):
|
|
2835
|
+
raise ValidationError(f"{path} keys must be strings")
|
|
2836
|
+
unknown = set(value) - keys
|
|
2837
|
+
missing = required - set(value)
|
|
2838
|
+
if unknown:
|
|
2839
|
+
raise ValidationError(f"{path} has unknown fields: {', '.join(sorted(unknown))}")
|
|
2840
|
+
if missing:
|
|
2841
|
+
raise ValidationError(f"{path} is missing fields: {', '.join(sorted(missing))}")
|
|
2842
|
+
return value
|
|
2843
|
+
|
|
2844
|
+
|
|
2845
|
+
def _nonempty_object(value: Any, *, path: str) -> Mapping[str, Any]:
|
|
2846
|
+
if not isinstance(value, Mapping) or not value:
|
|
2847
|
+
raise ValidationError(f"{path} must be a non-empty object")
|
|
2848
|
+
if any(not isinstance(key, str) or not key.strip() for key in value):
|
|
2849
|
+
raise ValidationError(f"{path} keys must be non-empty strings")
|
|
2850
|
+
return value
|
|
2851
|
+
|
|
2852
|
+
|
|
2853
|
+
def _text(value: Any, path: str) -> str:
|
|
2854
|
+
if not isinstance(value, str) or not value.strip() or value != value.strip():
|
|
2855
|
+
raise ValidationError(f"{path} must be trimmed non-empty text")
|
|
2856
|
+
return value
|
|
2857
|
+
|
|
2858
|
+
|
|
2859
|
+
def _require_text(value: Any, *, label: str) -> None:
|
|
2860
|
+
if (
|
|
2861
|
+
not isinstance(value, str)
|
|
2862
|
+
or not value.strip()
|
|
2863
|
+
or value != value.strip()
|
|
2864
|
+
or any(not character.isprintable() for character in value)
|
|
2865
|
+
):
|
|
2866
|
+
raise ValidationError(f"{label} must be trimmed non-empty printable text")
|
|
2867
|
+
|
|
2868
|
+
|
|
2869
|
+
def _require_sha256(value: Any, *, label: str) -> None:
|
|
2870
|
+
if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
|
|
2871
|
+
raise ValidationError(f"{label} must be a lowercase SHA-256")
|
|
2872
|
+
|
|
2873
|
+
|
|
2874
|
+
def _require_seed(value: Any) -> None:
|
|
2875
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
2876
|
+
raise ValidationError("external arena seed must be a non-negative integer")
|
|
2877
|
+
|
|
2878
|
+
|
|
2879
|
+
def _finite_number(value: Any, *, label: str) -> float:
|
|
2880
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
|
2881
|
+
raise ValidationError(f"{label} must be finite numeric")
|
|
2882
|
+
return float(value)
|
|
2883
|
+
|
|
2884
|
+
|
|
2885
|
+
def _safe_message(error: Exception) -> str:
|
|
2886
|
+
return f"{type(error).__name__}: {_safe_text(str(error))}"
|
|
2887
|
+
|
|
2888
|
+
|
|
2889
|
+
def _safe_text(value: str) -> str:
|
|
2890
|
+
result = value[:2048]
|
|
2891
|
+
for pattern in _SECRET_TEXT_PATTERNS:
|
|
2892
|
+
result = pattern.sub("<redacted>", result)
|
|
2893
|
+
return result
|
|
2894
|
+
|
|
2895
|
+
|
|
2896
|
+
__all__ = [
|
|
2897
|
+
"CONFIG_PATH_ENV",
|
|
2898
|
+
"CONFIG_SHA256_ENV",
|
|
2899
|
+
"ExternalArenaCheckpoint",
|
|
2900
|
+
"ExternalArenaExperimentDriver",
|
|
2901
|
+
"ExternalArmPayload",
|
|
2902
|
+
"ExternalArmPayloadBuildRequest",
|
|
2903
|
+
"ExternalBridgeBuildRequest",
|
|
2904
|
+
"ExternalCheckpointBuildRequest",
|
|
2905
|
+
"ExternalUnitPreflightRequest",
|
|
2906
|
+
"LockedExternalArenaBuilder",
|
|
2907
|
+
"factory",
|
|
2908
|
+
]
|