agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,1574 @@
|
|
|
1
|
+
"""Checksum-locked driver for the seven static production benchmark adapters.
|
|
2
|
+
|
|
3
|
+
This module is deliberately an assembly layer, not a model registry. Every
|
|
4
|
+
backend, downstream predictor, and intervention comes from an operator-owned
|
|
5
|
+
zero-argument builder named in one checksum-locked JSON file. The driver never
|
|
6
|
+
downloads an asset and never guesses a default deployment.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
from collections.abc import Mapping, Sequence
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from types import MappingProxyType
|
|
19
|
+
from typing import Any, Protocol, runtime_checkable
|
|
20
|
+
|
|
21
|
+
from agentinputlab.backends.retrieval import RetrievalResult, Retriever
|
|
22
|
+
from agentinputlab.benchmarks import (
|
|
23
|
+
BenchmarkAdapter,
|
|
24
|
+
BenchmarkBlockArm,
|
|
25
|
+
BenchmarkExample,
|
|
26
|
+
ControlledMultiArmBlockRunner,
|
|
27
|
+
DownstreamPredictor,
|
|
28
|
+
PreparedQABenchmarkAdapter,
|
|
29
|
+
PreparedRetrievalBenchmarkAdapter,
|
|
30
|
+
load_production_benchmark_adapter,
|
|
31
|
+
production_benchmark_specs,
|
|
32
|
+
)
|
|
33
|
+
from agentinputlab.contracts.dependency import DependencyBinding, DependencyRole
|
|
34
|
+
from agentinputlab.contracts.enums import ProfileKind, SemanticFamily
|
|
35
|
+
from agentinputlab.contracts.errors import ArtifactError, ValidationError
|
|
36
|
+
from agentinputlab.contracts.intent import IntentContract
|
|
37
|
+
from agentinputlab.contracts.lifecycle import Intervention, InterventionContext
|
|
38
|
+
from agentinputlab.contracts.serialization import (
|
|
39
|
+
content_hash,
|
|
40
|
+
freeze_mapping,
|
|
41
|
+
to_jsonable,
|
|
42
|
+
)
|
|
43
|
+
from agentinputlab.contracts.state import ContextBlock, ExecutionBudget, FrozenAgentState
|
|
44
|
+
from agentinputlab.experiments.executor import ExperimentCellResult
|
|
45
|
+
from agentinputlab.experiments.loader import resolve_driver_entrypoint
|
|
46
|
+
from agentinputlab.experiments.plan import ExperimentCell
|
|
47
|
+
from agentinputlab.experiments.schema import (
|
|
48
|
+
ExperimentArmSpec,
|
|
49
|
+
ExperimentBenchmarkTarget,
|
|
50
|
+
ExperimentManifest,
|
|
51
|
+
)
|
|
52
|
+
from agentinputlab.observability import PricingCatalog, ResourceReceipt, RunMonitor
|
|
53
|
+
from agentinputlab.runner import ComparisonArm, ComparisonProfile
|
|
54
|
+
from agentinputlab.selectors import VisibleStateFeatureExtractor
|
|
55
|
+
|
|
56
|
+
CONFIG_PATH_ENV = "AILAB_STATIC_PRODUCTION_DRIVER_CONFIG"
|
|
57
|
+
CONFIG_SHA256_ENV = "AILAB_STATIC_PRODUCTION_DRIVER_CONFIG_SHA256"
|
|
58
|
+
|
|
59
|
+
_CONFIG_SCHEMA_VERSION = 2
|
|
60
|
+
_DRIVER_IMPLEMENTATION_ID = "agentinputlab.static_production.multi_arm"
|
|
61
|
+
_PROFILE_ID = "static_production.multi_arm.v2"
|
|
62
|
+
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
63
|
+
_ENTRYPOINT = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*:[A-Za-z_][A-Za-z0-9_.]*$")
|
|
64
|
+
_MAX_CONFIG_BYTES = 4 * 1024 * 1024
|
|
65
|
+
_BUILDER_KINDS = frozenset({"backend", "predictor", "intervention"})
|
|
66
|
+
# Schema v2 is a frozen seven-cell campaign contract. Adding a standalone
|
|
67
|
+
# source-locked loader must not expand an existing campaign or dispatch new data.
|
|
68
|
+
_STATIC_V2_CELLS = frozenset(
|
|
69
|
+
{
|
|
70
|
+
"beir.arguana",
|
|
71
|
+
"bright.earth_science",
|
|
72
|
+
"bright.economics",
|
|
73
|
+
"musique.answerable",
|
|
74
|
+
"2wiki.multihop",
|
|
75
|
+
"hotpotqa.distractor",
|
|
76
|
+
"qasper.v0_3_dev",
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
_STATIC_PRODUCTION_SPECS = tuple(
|
|
80
|
+
spec for spec in production_benchmark_specs() if spec.cell_id in _STATIC_V2_CELLS
|
|
81
|
+
)
|
|
82
|
+
_STATIC_SPLITS = MappingProxyType({spec.cell_id: spec.split for spec in _STATIC_PRODUCTION_SPECS})
|
|
83
|
+
_SECRET_TEXT_PATTERNS = (
|
|
84
|
+
re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{8,}"),
|
|
85
|
+
re.compile(r"\bsk-[A-Za-z0-9_-]{8,}"),
|
|
86
|
+
re.compile(r"\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\b"),
|
|
87
|
+
re.compile(
|
|
88
|
+
r"(?i)(\b(?:[a-z0-9]+[_-])*(?:api[_-]?key|x[_-]?api[_-]?key|authorization)"
|
|
89
|
+
r"\b\s*[:=]\s*)['\"]?(?:bearer\s+)?[^\s,;'\"]{4,}"
|
|
90
|
+
),
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@runtime_checkable
|
|
95
|
+
class LockedComponentBuilder(Protocol):
|
|
96
|
+
"""Operator builder contract resolved only during explicit preflight/run."""
|
|
97
|
+
|
|
98
|
+
builder_kind: str
|
|
99
|
+
|
|
100
|
+
def fingerprint(self) -> Mapping[str, Any]: ...
|
|
101
|
+
|
|
102
|
+
def preflight(self, parameters: Mapping[str, Any]) -> Sequence[str]: ...
|
|
103
|
+
|
|
104
|
+
def build(self, request: Any) -> Any: ...
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass(frozen=True, slots=True)
|
|
108
|
+
class LockedBuilderSpec:
|
|
109
|
+
"""One builder entrypoint plus its construction and optional product locks."""
|
|
110
|
+
|
|
111
|
+
entrypoint: str
|
|
112
|
+
fingerprint_sha256: str
|
|
113
|
+
parameters: Mapping[str, Any] = field(default_factory=dict)
|
|
114
|
+
expected_product_sha256: str | None = None
|
|
115
|
+
|
|
116
|
+
def __post_init__(self) -> None:
|
|
117
|
+
if not isinstance(self.entrypoint, str) or _ENTRYPOINT.fullmatch(self.entrypoint) is None:
|
|
118
|
+
raise ValidationError("builder entrypoint must use module:factory syntax")
|
|
119
|
+
_require_sha256(self.fingerprint_sha256, label="builder fingerprint_sha256")
|
|
120
|
+
if self.expected_product_sha256 is not None:
|
|
121
|
+
_require_sha256(
|
|
122
|
+
self.expected_product_sha256,
|
|
123
|
+
label="builder expected_product_sha256",
|
|
124
|
+
)
|
|
125
|
+
try:
|
|
126
|
+
object.__setattr__(self, "parameters", freeze_mapping(self.parameters))
|
|
127
|
+
except (TypeError, ValueError) as exc:
|
|
128
|
+
raise ValidationError("builder parameters must be canonical JSON") from exc
|
|
129
|
+
|
|
130
|
+
@classmethod
|
|
131
|
+
def from_mapping(cls, value: Any, *, path: str) -> LockedBuilderSpec:
|
|
132
|
+
data = _strict_mapping(
|
|
133
|
+
value,
|
|
134
|
+
path=path,
|
|
135
|
+
keys={
|
|
136
|
+
"entrypoint",
|
|
137
|
+
"fingerprint_sha256",
|
|
138
|
+
"expected_product_sha256",
|
|
139
|
+
"parameters",
|
|
140
|
+
},
|
|
141
|
+
required={"entrypoint", "fingerprint_sha256"},
|
|
142
|
+
)
|
|
143
|
+
parameters = data.get("parameters", {})
|
|
144
|
+
if not isinstance(parameters, Mapping):
|
|
145
|
+
raise ValidationError(f"{path}.parameters must be an object")
|
|
146
|
+
return cls(
|
|
147
|
+
entrypoint=_text(data["entrypoint"], f"{path}.entrypoint"),
|
|
148
|
+
fingerprint_sha256=_text(
|
|
149
|
+
data["fingerprint_sha256"],
|
|
150
|
+
f"{path}.fingerprint_sha256",
|
|
151
|
+
),
|
|
152
|
+
expected_product_sha256=(
|
|
153
|
+
_text(
|
|
154
|
+
data["expected_product_sha256"],
|
|
155
|
+
f"{path}.expected_product_sha256",
|
|
156
|
+
)
|
|
157
|
+
if "expected_product_sha256" in data
|
|
158
|
+
else None
|
|
159
|
+
),
|
|
160
|
+
parameters=parameters,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def binding_identity(self) -> Mapping[str, Any]:
|
|
165
|
+
"""Public identity without copying raw operator parameters into results."""
|
|
166
|
+
|
|
167
|
+
return freeze_mapping(
|
|
168
|
+
{
|
|
169
|
+
"entrypoint": self.entrypoint,
|
|
170
|
+
"fingerprint_sha256": self.fingerprint_sha256,
|
|
171
|
+
"expected_product_sha256": self.expected_product_sha256,
|
|
172
|
+
"parameters_sha256": content_hash(self.parameters),
|
|
173
|
+
}
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass(frozen=True, slots=True)
|
|
178
|
+
class StaticRuntimeComponents:
|
|
179
|
+
"""Fresh runtime graph returned by a backend builder for one arm."""
|
|
180
|
+
|
|
181
|
+
backend: Any
|
|
182
|
+
dense_embedder: Any = None
|
|
183
|
+
retriever: Any = None
|
|
184
|
+
vector_searcher: Any = None
|
|
185
|
+
tools: Any = None
|
|
186
|
+
external_runtime: Any = None
|
|
187
|
+
artifact_manager: Any = None
|
|
188
|
+
run_config: Mapping[str, Any] = field(default_factory=dict)
|
|
189
|
+
intent_contract: IntentContract | None = None
|
|
190
|
+
pricing_catalog: PricingCatalog | None = None
|
|
191
|
+
strict_intent: bool = True
|
|
192
|
+
|
|
193
|
+
def __post_init__(self) -> None:
|
|
194
|
+
if self.backend is None:
|
|
195
|
+
raise ValidationError("static production runtime requires an explicit backend")
|
|
196
|
+
fingerprint = getattr(self.backend, "fingerprint", None)
|
|
197
|
+
if not callable(fingerprint):
|
|
198
|
+
raise ValidationError("static production backend must expose fingerprint()")
|
|
199
|
+
if not isinstance(self.strict_intent, bool):
|
|
200
|
+
raise ValidationError("static production strict_intent must be boolean")
|
|
201
|
+
try:
|
|
202
|
+
object.__setattr__(self, "run_config", freeze_mapping(self.run_config))
|
|
203
|
+
except (TypeError, ValueError) as exc:
|
|
204
|
+
raise ValidationError("static production run_config must be canonical JSON") from exc
|
|
205
|
+
if self.intent_contract is not None and not isinstance(
|
|
206
|
+
self.intent_contract,
|
|
207
|
+
IntentContract,
|
|
208
|
+
):
|
|
209
|
+
raise TypeError("static production intent_contract must be IntentContract")
|
|
210
|
+
if self.pricing_catalog is not None and not isinstance(
|
|
211
|
+
self.pricing_catalog,
|
|
212
|
+
PricingCatalog,
|
|
213
|
+
):
|
|
214
|
+
raise TypeError("static production pricing_catalog must be PricingCatalog")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@dataclass(frozen=True, slots=True)
|
|
218
|
+
class StaticBackendBuildRequest:
|
|
219
|
+
model_slot: str
|
|
220
|
+
seed: int
|
|
221
|
+
builder_parameters: Mapping[str, Any]
|
|
222
|
+
|
|
223
|
+
def __post_init__(self) -> None:
|
|
224
|
+
_require_nonempty(self.model_slot, label="model_slot")
|
|
225
|
+
_require_seed(self.seed)
|
|
226
|
+
object.__setattr__(
|
|
227
|
+
self,
|
|
228
|
+
"builder_parameters",
|
|
229
|
+
freeze_mapping(self.builder_parameters),
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@dataclass(frozen=True, slots=True)
|
|
234
|
+
class StaticPredictorBuildRequest:
|
|
235
|
+
benchmark_id: str
|
|
236
|
+
split: str
|
|
237
|
+
model_slot: str
|
|
238
|
+
seed: int
|
|
239
|
+
builder_parameters: Mapping[str, Any]
|
|
240
|
+
runtime: StaticRuntimeComponents
|
|
241
|
+
|
|
242
|
+
def __post_init__(self) -> None:
|
|
243
|
+
for label, value in (
|
|
244
|
+
("benchmark_id", self.benchmark_id),
|
|
245
|
+
("split", self.split),
|
|
246
|
+
("model_slot", self.model_slot),
|
|
247
|
+
):
|
|
248
|
+
_require_nonempty(value, label=label)
|
|
249
|
+
_require_seed(self.seed)
|
|
250
|
+
if not isinstance(self.runtime, StaticRuntimeComponents):
|
|
251
|
+
raise TypeError("predictor request runtime must be StaticRuntimeComponents")
|
|
252
|
+
object.__setattr__(
|
|
253
|
+
self,
|
|
254
|
+
"builder_parameters",
|
|
255
|
+
freeze_mapping(self.builder_parameters),
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@dataclass(frozen=True, slots=True)
|
|
260
|
+
class StaticArmBuildRequest:
|
|
261
|
+
arm_id: str
|
|
262
|
+
release_id: str
|
|
263
|
+
family: SemanticFamily
|
|
264
|
+
profile_kind: ProfileKind
|
|
265
|
+
model_slot: str
|
|
266
|
+
seed: int
|
|
267
|
+
builder_parameters: Mapping[str, Any]
|
|
268
|
+
arm_parameters: Mapping[str, Any]
|
|
269
|
+
runtime: StaticRuntimeComponents
|
|
270
|
+
|
|
271
|
+
def __post_init__(self) -> None:
|
|
272
|
+
for label, value in (
|
|
273
|
+
("arm_id", self.arm_id),
|
|
274
|
+
("release_id", self.release_id),
|
|
275
|
+
("model_slot", self.model_slot),
|
|
276
|
+
):
|
|
277
|
+
_require_nonempty(value, label=label)
|
|
278
|
+
if not isinstance(self.family, SemanticFamily):
|
|
279
|
+
raise TypeError("arm request family must be SemanticFamily")
|
|
280
|
+
if not isinstance(self.profile_kind, ProfileKind):
|
|
281
|
+
raise TypeError("arm request profile_kind must be ProfileKind")
|
|
282
|
+
_require_seed(self.seed)
|
|
283
|
+
if not isinstance(self.runtime, StaticRuntimeComponents):
|
|
284
|
+
raise TypeError("arm request runtime must be StaticRuntimeComponents")
|
|
285
|
+
object.__setattr__(
|
|
286
|
+
self,
|
|
287
|
+
"builder_parameters",
|
|
288
|
+
freeze_mapping(self.builder_parameters),
|
|
289
|
+
)
|
|
290
|
+
object.__setattr__(self, "arm_parameters", freeze_mapping(self.arm_parameters))
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
@dataclass(frozen=True, slots=True)
|
|
294
|
+
class _BenchmarkBinding:
|
|
295
|
+
split: str
|
|
296
|
+
predictor_builder: LockedBuilderSpec
|
|
297
|
+
|
|
298
|
+
def __post_init__(self) -> None:
|
|
299
|
+
_require_nonempty(self.split, label="benchmark split")
|
|
300
|
+
if not isinstance(self.predictor_builder, LockedBuilderSpec):
|
|
301
|
+
raise TypeError("benchmark predictor_builder must be LockedBuilderSpec")
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
@dataclass(frozen=True, slots=True)
|
|
305
|
+
class _ArmBinding:
|
|
306
|
+
release_id: str
|
|
307
|
+
intervention_builder: LockedBuilderSpec
|
|
308
|
+
|
|
309
|
+
def __post_init__(self) -> None:
|
|
310
|
+
_require_nonempty(self.release_id, label="arm release_id")
|
|
311
|
+
if not isinstance(self.intervention_builder, LockedBuilderSpec):
|
|
312
|
+
raise TypeError("arm intervention_builder must be LockedBuilderSpec")
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@dataclass(frozen=True, slots=True)
|
|
316
|
+
class _DriverConfig:
|
|
317
|
+
schema_version: int
|
|
318
|
+
driver_id: str
|
|
319
|
+
driver_revision_sha256: str
|
|
320
|
+
prepared_data_root: Path
|
|
321
|
+
baseline_arm_id: str
|
|
322
|
+
execution_budget: ExecutionBudget
|
|
323
|
+
model_slots: Mapping[str, LockedBuilderSpec]
|
|
324
|
+
benchmark_cells: Mapping[str, _BenchmarkBinding]
|
|
325
|
+
arms: Mapping[str, _ArmBinding]
|
|
326
|
+
|
|
327
|
+
def __post_init__(self) -> None:
|
|
328
|
+
if self.schema_version != _CONFIG_SCHEMA_VERSION:
|
|
329
|
+
raise ValidationError("unsupported static production config schema_version")
|
|
330
|
+
_require_nonempty(self.driver_id, label="driver_id")
|
|
331
|
+
_require_sha256(
|
|
332
|
+
self.driver_revision_sha256,
|
|
333
|
+
label="driver_revision_sha256",
|
|
334
|
+
)
|
|
335
|
+
root = self.prepared_data_root
|
|
336
|
+
if not isinstance(root, Path) or not root.is_absolute():
|
|
337
|
+
raise ValidationError("prepared_data_root must be an absolute path")
|
|
338
|
+
_require_nonempty(self.baseline_arm_id, label="baseline_arm_id")
|
|
339
|
+
if not isinstance(self.execution_budget, ExecutionBudget):
|
|
340
|
+
raise TypeError("execution_budget must be ExecutionBudget")
|
|
341
|
+
model_slots = dict(self.model_slots)
|
|
342
|
+
benchmark_cells = dict(self.benchmark_cells)
|
|
343
|
+
arms = dict(self.arms)
|
|
344
|
+
if not model_slots or not all(
|
|
345
|
+
isinstance(name, str) and name.strip() and isinstance(spec, LockedBuilderSpec)
|
|
346
|
+
for name, spec in model_slots.items()
|
|
347
|
+
):
|
|
348
|
+
raise ValidationError("model_slots requires named LockedBuilderSpec values")
|
|
349
|
+
for name, spec in model_slots.items():
|
|
350
|
+
if spec.expected_product_sha256 is None:
|
|
351
|
+
raise ValidationError(f"model_slots.{name} requires expected_product_sha256")
|
|
352
|
+
if set(benchmark_cells) != set(_STATIC_SPLITS):
|
|
353
|
+
missing = sorted(set(_STATIC_SPLITS) - set(benchmark_cells))
|
|
354
|
+
extra = sorted(set(benchmark_cells) - set(_STATIC_SPLITS))
|
|
355
|
+
raise ValidationError(
|
|
356
|
+
"benchmark_cells must bind exactly the seven static production cells; "
|
|
357
|
+
f"missing={missing}, extra={extra}"
|
|
358
|
+
)
|
|
359
|
+
for cell_id, binding in benchmark_cells.items():
|
|
360
|
+
if not isinstance(binding, _BenchmarkBinding):
|
|
361
|
+
raise TypeError("benchmark_cells values must be benchmark bindings")
|
|
362
|
+
if binding.split != _STATIC_SPLITS[cell_id]:
|
|
363
|
+
raise ValidationError(
|
|
364
|
+
f"benchmark_cells.{cell_id}.split differs from the production lock"
|
|
365
|
+
)
|
|
366
|
+
if binding.predictor_builder.expected_product_sha256 is None:
|
|
367
|
+
raise ValidationError(
|
|
368
|
+
f"benchmark_cells.{cell_id}.predictor_builder requires expected_product_sha256"
|
|
369
|
+
)
|
|
370
|
+
if len(arms) < 2 or not all(
|
|
371
|
+
isinstance(arm_id, str) and arm_id.strip() and isinstance(binding, _ArmBinding)
|
|
372
|
+
for arm_id, binding in arms.items()
|
|
373
|
+
):
|
|
374
|
+
raise ValidationError("arms requires at least two named arm bindings")
|
|
375
|
+
if self.baseline_arm_id not in arms:
|
|
376
|
+
raise ValidationError("baseline_arm_id is not present in configured arms")
|
|
377
|
+
for arm_id, binding in arms.items():
|
|
378
|
+
if binding.intervention_builder.expected_product_sha256 is not None:
|
|
379
|
+
raise ValidationError(
|
|
380
|
+
f"arms.{arm_id}.intervention_builder must not declare expected_product_sha256"
|
|
381
|
+
)
|
|
382
|
+
object.__setattr__(self, "model_slots", MappingProxyType(model_slots))
|
|
383
|
+
object.__setattr__(self, "benchmark_cells", MappingProxyType(benchmark_cells))
|
|
384
|
+
object.__setattr__(self, "arms", MappingProxyType(arms))
|
|
385
|
+
|
|
386
|
+
@classmethod
|
|
387
|
+
def from_mapping(cls, value: Any) -> _DriverConfig:
|
|
388
|
+
data = _strict_mapping(
|
|
389
|
+
value,
|
|
390
|
+
path="config",
|
|
391
|
+
keys={
|
|
392
|
+
"schema_version",
|
|
393
|
+
"driver_id",
|
|
394
|
+
"driver_revision_sha256",
|
|
395
|
+
"prepared_data_root",
|
|
396
|
+
"baseline_arm_id",
|
|
397
|
+
"execution_budget",
|
|
398
|
+
"model_slots",
|
|
399
|
+
"benchmark_cells",
|
|
400
|
+
"arms",
|
|
401
|
+
},
|
|
402
|
+
required={
|
|
403
|
+
"schema_version",
|
|
404
|
+
"driver_id",
|
|
405
|
+
"driver_revision_sha256",
|
|
406
|
+
"prepared_data_root",
|
|
407
|
+
"baseline_arm_id",
|
|
408
|
+
"model_slots",
|
|
409
|
+
"benchmark_cells",
|
|
410
|
+
"arms",
|
|
411
|
+
},
|
|
412
|
+
)
|
|
413
|
+
schema_version = data["schema_version"]
|
|
414
|
+
if isinstance(schema_version, bool) or not isinstance(schema_version, int):
|
|
415
|
+
raise ValidationError("config.schema_version must be an integer")
|
|
416
|
+
root_text = _text(data["prepared_data_root"], "config.prepared_data_root")
|
|
417
|
+
root = Path(root_text).expanduser()
|
|
418
|
+
if not root.is_absolute():
|
|
419
|
+
raise ValidationError("config.prepared_data_root must be an absolute path")
|
|
420
|
+
models = _strict_object(data["model_slots"], path="config.model_slots")
|
|
421
|
+
benchmarks = _strict_object(
|
|
422
|
+
data["benchmark_cells"],
|
|
423
|
+
path="config.benchmark_cells",
|
|
424
|
+
)
|
|
425
|
+
arms = _strict_object(data["arms"], path="config.arms")
|
|
426
|
+
return cls(
|
|
427
|
+
schema_version=schema_version,
|
|
428
|
+
driver_id=_text(data["driver_id"], "config.driver_id"),
|
|
429
|
+
driver_revision_sha256=_text(
|
|
430
|
+
data["driver_revision_sha256"],
|
|
431
|
+
"config.driver_revision_sha256",
|
|
432
|
+
),
|
|
433
|
+
prepared_data_root=root,
|
|
434
|
+
baseline_arm_id=_text(
|
|
435
|
+
data["baseline_arm_id"],
|
|
436
|
+
"config.baseline_arm_id",
|
|
437
|
+
),
|
|
438
|
+
execution_budget=_execution_budget(
|
|
439
|
+
data.get("execution_budget", {}),
|
|
440
|
+
path="config.execution_budget",
|
|
441
|
+
),
|
|
442
|
+
model_slots={
|
|
443
|
+
model_slot: LockedBuilderSpec.from_mapping(
|
|
444
|
+
raw,
|
|
445
|
+
path=f"config.model_slots.{model_slot}",
|
|
446
|
+
)
|
|
447
|
+
for model_slot, raw in models.items()
|
|
448
|
+
},
|
|
449
|
+
benchmark_cells={
|
|
450
|
+
cell_id: _benchmark_binding(
|
|
451
|
+
raw,
|
|
452
|
+
path=f"config.benchmark_cells.{cell_id}",
|
|
453
|
+
)
|
|
454
|
+
for cell_id, raw in benchmarks.items()
|
|
455
|
+
},
|
|
456
|
+
arms={
|
|
457
|
+
arm_id: _arm_binding(raw, path=f"config.arms.{arm_id}")
|
|
458
|
+
for arm_id, raw in arms.items()
|
|
459
|
+
},
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
class StaticProductionExperimentDriver:
|
|
464
|
+
"""Execute exact per-unit multi-arm blocks over verified static adapters."""
|
|
465
|
+
|
|
466
|
+
def __init__(self, config: _DriverConfig, *, config_sha256: str) -> None:
|
|
467
|
+
if not isinstance(config, _DriverConfig):
|
|
468
|
+
raise TypeError("config must be a static production driver config")
|
|
469
|
+
_require_sha256(config_sha256, label="config_sha256")
|
|
470
|
+
self._config = config
|
|
471
|
+
self._config_sha256 = config_sha256
|
|
472
|
+
self._adapters: dict[str, BenchmarkAdapter] = {}
|
|
473
|
+
self._preflight_manifest_hash: str | None = None
|
|
474
|
+
self._feature_extractor = VisibleStateFeatureExtractor()
|
|
475
|
+
|
|
476
|
+
@classmethod
|
|
477
|
+
def from_file(
|
|
478
|
+
cls,
|
|
479
|
+
path: str | Path,
|
|
480
|
+
*,
|
|
481
|
+
expected_sha256: str,
|
|
482
|
+
) -> StaticProductionExperimentDriver:
|
|
483
|
+
"""Load a strict UTF-8 JSON config after verifying its exact bytes."""
|
|
484
|
+
|
|
485
|
+
_require_sha256(expected_sha256, label="expected config SHA-256")
|
|
486
|
+
source = Path(path).expanduser()
|
|
487
|
+
if not source.is_absolute():
|
|
488
|
+
raise ArtifactError("static production config path must be absolute")
|
|
489
|
+
try:
|
|
490
|
+
size = source.stat().st_size
|
|
491
|
+
if size > _MAX_CONFIG_BYTES:
|
|
492
|
+
raise ArtifactError("static production config exceeds the size limit")
|
|
493
|
+
raw_bytes = source.read_bytes()
|
|
494
|
+
except OSError as exc:
|
|
495
|
+
raise ArtifactError("cannot read static production driver config") from exc
|
|
496
|
+
observed = hashlib.sha256(raw_bytes).hexdigest()
|
|
497
|
+
if observed != expected_sha256:
|
|
498
|
+
raise ArtifactError("static production driver config checksum mismatch")
|
|
499
|
+
raw = _strict_json(raw_bytes)
|
|
500
|
+
return cls(_DriverConfig.from_mapping(raw), config_sha256=observed)
|
|
501
|
+
|
|
502
|
+
def fingerprint(self) -> Mapping[str, Any]:
|
|
503
|
+
return freeze_mapping(
|
|
504
|
+
{
|
|
505
|
+
"implementation_id": _DRIVER_IMPLEMENTATION_ID,
|
|
506
|
+
"schema_version": _CONFIG_SCHEMA_VERSION,
|
|
507
|
+
"driver_id": self._config.driver_id,
|
|
508
|
+
"driver_revision_sha256": self._config.driver_revision_sha256,
|
|
509
|
+
"config_sha256": self._config_sha256,
|
|
510
|
+
"baseline_arm_id": self._config.baseline_arm_id,
|
|
511
|
+
"execution_budget_sha256": content_hash(self._config.execution_budget),
|
|
512
|
+
"model_slots": {
|
|
513
|
+
name: spec.binding_identity for name, spec in self._config.model_slots.items()
|
|
514
|
+
},
|
|
515
|
+
"benchmark_cells": {
|
|
516
|
+
name: {
|
|
517
|
+
"split": binding.split,
|
|
518
|
+
"predictor_builder": binding.predictor_builder.binding_identity,
|
|
519
|
+
}
|
|
520
|
+
for name, binding in self._config.benchmark_cells.items()
|
|
521
|
+
},
|
|
522
|
+
"arms": {
|
|
523
|
+
name: {
|
|
524
|
+
"release_id": binding.release_id,
|
|
525
|
+
"intervention_builder": binding.intervention_builder.binding_identity,
|
|
526
|
+
}
|
|
527
|
+
for name, binding in self._config.arms.items()
|
|
528
|
+
},
|
|
529
|
+
"feature_schema_id": self._feature_extractor.feature_schema_id,
|
|
530
|
+
"feature_schema_hash": self._feature_extractor.feature_schema_hash,
|
|
531
|
+
}
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
def preflight(self, manifest: ExperimentManifest) -> Sequence[str]:
|
|
535
|
+
"""Verify every selected binding and prepared cache without executing a cell."""
|
|
536
|
+
|
|
537
|
+
self._adapters.clear()
|
|
538
|
+
self._preflight_manifest_hash = None
|
|
539
|
+
blockers = self._manifest_blockers(manifest)
|
|
540
|
+
if blockers:
|
|
541
|
+
return tuple(blockers)
|
|
542
|
+
|
|
543
|
+
used_cells = tuple(target.cell_id for target in manifest.benchmarks)
|
|
544
|
+
used_arms = tuple(arm.arm_id for arm in manifest.arms)
|
|
545
|
+
builder_checks: list[tuple[str, LockedBuilderSpec, str]] = []
|
|
546
|
+
builder_checks.extend(
|
|
547
|
+
(
|
|
548
|
+
f"model slot {model_slot}",
|
|
549
|
+
self._config.model_slots[model_slot],
|
|
550
|
+
"backend",
|
|
551
|
+
)
|
|
552
|
+
for model_slot in manifest.model_slots
|
|
553
|
+
)
|
|
554
|
+
builder_checks.extend(
|
|
555
|
+
(
|
|
556
|
+
f"benchmark {cell_id}",
|
|
557
|
+
self._config.benchmark_cells[cell_id].predictor_builder,
|
|
558
|
+
"predictor",
|
|
559
|
+
)
|
|
560
|
+
for cell_id in used_cells
|
|
561
|
+
)
|
|
562
|
+
builder_checks.extend(
|
|
563
|
+
(
|
|
564
|
+
f"arm {arm_id}",
|
|
565
|
+
self._config.arms[arm_id].intervention_builder,
|
|
566
|
+
"intervention",
|
|
567
|
+
)
|
|
568
|
+
for arm_id in used_arms
|
|
569
|
+
)
|
|
570
|
+
for label, spec, kind in builder_checks:
|
|
571
|
+
try:
|
|
572
|
+
builder = _resolve_locked_builder(spec, expected_kind=kind)
|
|
573
|
+
messages = _builder_preflight(builder, spec.parameters)
|
|
574
|
+
except Exception as exc:
|
|
575
|
+
blockers.append(
|
|
576
|
+
f"{label}: "
|
|
577
|
+
+ _safe_blocker_message(
|
|
578
|
+
exc,
|
|
579
|
+
prepared_root=self._config.prepared_data_root,
|
|
580
|
+
)
|
|
581
|
+
)
|
|
582
|
+
continue
|
|
583
|
+
blockers.extend(
|
|
584
|
+
f"{label}: "
|
|
585
|
+
+ _safe_operator_message(
|
|
586
|
+
message,
|
|
587
|
+
prepared_root=self._config.prepared_data_root,
|
|
588
|
+
)
|
|
589
|
+
for message in messages
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
for target in manifest.benchmarks:
|
|
593
|
+
try:
|
|
594
|
+
adapter = load_production_benchmark_adapter(
|
|
595
|
+
target.cell_id,
|
|
596
|
+
self._config.prepared_data_root,
|
|
597
|
+
)
|
|
598
|
+
if not isinstance(
|
|
599
|
+
adapter,
|
|
600
|
+
(PreparedQABenchmarkAdapter, PreparedRetrievalBenchmarkAdapter),
|
|
601
|
+
):
|
|
602
|
+
raise ArtifactError("driver loaded a non-static production adapter")
|
|
603
|
+
examples = tuple(adapter.examples(target.split))
|
|
604
|
+
identifiers = {example.example_id for example in examples}
|
|
605
|
+
if len(identifiers) != len(examples):
|
|
606
|
+
raise ArtifactError("production adapter contains duplicate unit IDs")
|
|
607
|
+
missing = sorted(set(target.unit_ids) - identifiers)
|
|
608
|
+
if missing:
|
|
609
|
+
raise ArtifactError(
|
|
610
|
+
"frozen manifest units are absent from the prepared adapter: "
|
|
611
|
+
+ ", ".join(missing[:5])
|
|
612
|
+
)
|
|
613
|
+
self._adapters[target.cell_id] = adapter
|
|
614
|
+
except Exception as exc:
|
|
615
|
+
blockers.append(
|
|
616
|
+
f"benchmark {target.cell_id}: "
|
|
617
|
+
+ _safe_blocker_message(
|
|
618
|
+
exc,
|
|
619
|
+
prepared_root=self._config.prepared_data_root,
|
|
620
|
+
)
|
|
621
|
+
)
|
|
622
|
+
|
|
623
|
+
if blockers:
|
|
624
|
+
self._adapters.clear()
|
|
625
|
+
return tuple(blockers)
|
|
626
|
+
self._preflight_manifest_hash = manifest.manifest_hash
|
|
627
|
+
return ()
|
|
628
|
+
|
|
629
|
+
def preflight_units(
|
|
630
|
+
self,
|
|
631
|
+
manifest: ExperimentManifest,
|
|
632
|
+
resolved_units: Mapping[str, tuple[str, ...]],
|
|
633
|
+
) -> Sequence[str]:
|
|
634
|
+
"""Prove every planned unit exists before the plan is accepted."""
|
|
635
|
+
|
|
636
|
+
if self._preflight_manifest_hash != manifest.manifest_hash:
|
|
637
|
+
return ("unit inventory validation requires a successful driver preflight",)
|
|
638
|
+
expected_keys = {f"{target.cell_id}:{target.split}" for target in manifest.benchmarks}
|
|
639
|
+
if set(resolved_units) != expected_keys:
|
|
640
|
+
return ("resolved unit inventories differ from manifest benchmark targets",)
|
|
641
|
+
blockers: list[str] = []
|
|
642
|
+
for target in manifest.benchmarks:
|
|
643
|
+
key = f"{target.cell_id}:{target.split}"
|
|
644
|
+
try:
|
|
645
|
+
adapter = self._adapters[target.cell_id]
|
|
646
|
+
identifiers = {example.example_id for example in adapter.examples(target.split)}
|
|
647
|
+
except Exception as exc:
|
|
648
|
+
blockers.append(
|
|
649
|
+
f"benchmark {target.cell_id}: "
|
|
650
|
+
+ _safe_blocker_message(
|
|
651
|
+
exc,
|
|
652
|
+
prepared_root=self._config.prepared_data_root,
|
|
653
|
+
)
|
|
654
|
+
)
|
|
655
|
+
continue
|
|
656
|
+
missing = sorted(set(resolved_units[key]) - identifiers)
|
|
657
|
+
if missing:
|
|
658
|
+
blockers.append(
|
|
659
|
+
f"benchmark {target.cell_id}: resolved units are absent from the "
|
|
660
|
+
"prepared adapter: " + ", ".join(missing[:5])
|
|
661
|
+
)
|
|
662
|
+
return tuple(blockers)
|
|
663
|
+
|
|
664
|
+
def execute_cell(
|
|
665
|
+
self,
|
|
666
|
+
manifest: ExperimentManifest,
|
|
667
|
+
cell: ExperimentCell,
|
|
668
|
+
) -> ExperimentCellResult:
|
|
669
|
+
if self._preflight_manifest_hash != manifest.manifest_hash:
|
|
670
|
+
raise ArtifactError(
|
|
671
|
+
"static production driver requires successful preflight for this manifest"
|
|
672
|
+
)
|
|
673
|
+
target = self._validate_cell(manifest, cell)
|
|
674
|
+
try:
|
|
675
|
+
adapter = self._adapters[cell.benchmark_id]
|
|
676
|
+
except KeyError as exc: # pragma: no cover - preflight invariant
|
|
677
|
+
raise ArtifactError("preflight did not retain the production adapter") from exc
|
|
678
|
+
|
|
679
|
+
arm_specs = {arm.arm_id: arm for arm in manifest.arms}
|
|
680
|
+
dataset_identity = _dataset_identity(adapter, split=cell.split)
|
|
681
|
+
benchmark_binding = self._config.benchmark_cells[cell.benchmark_id]
|
|
682
|
+
built_arms: list[BenchmarkBlockArm] = []
|
|
683
|
+
contexts: list[InterventionContext] = []
|
|
684
|
+
for arm_id in cell.arm_ids:
|
|
685
|
+
arm_spec = arm_specs[arm_id]
|
|
686
|
+
runtime = self._build_runtime(cell)
|
|
687
|
+
predictor = self._build_predictor(
|
|
688
|
+
cell,
|
|
689
|
+
runtime=runtime,
|
|
690
|
+
binding=benchmark_binding,
|
|
691
|
+
)
|
|
692
|
+
intervention = self._build_intervention(
|
|
693
|
+
cell,
|
|
694
|
+
arm_spec=arm_spec,
|
|
695
|
+
runtime=runtime,
|
|
696
|
+
)
|
|
697
|
+
runtime_product_sha256 = _require_product_sha256(
|
|
698
|
+
static_runtime_product_sha256(runtime),
|
|
699
|
+
expected=self._config.model_slots[cell.model_slot].expected_product_sha256,
|
|
700
|
+
label="static runtime product",
|
|
701
|
+
)
|
|
702
|
+
predictor_product_sha256 = _require_product_sha256(
|
|
703
|
+
static_predictor_product_sha256(predictor),
|
|
704
|
+
expected=benchmark_binding.predictor_builder.expected_product_sha256,
|
|
705
|
+
label="static predictor product",
|
|
706
|
+
)
|
|
707
|
+
model_identity = freeze_mapping({"runtime_product_sha256": runtime_product_sha256})
|
|
708
|
+
predictor_identity = freeze_mapping(
|
|
709
|
+
{"predictor_product_sha256": predictor_product_sha256}
|
|
710
|
+
)
|
|
711
|
+
bindings = _comparison_bindings(
|
|
712
|
+
dataset_identity=dataset_identity,
|
|
713
|
+
model_identity=model_identity,
|
|
714
|
+
predictor_identity=predictor_identity,
|
|
715
|
+
arm_spec=arm_spec,
|
|
716
|
+
builder_spec=self._config.arms[arm_id].intervention_builder,
|
|
717
|
+
intervention=intervention,
|
|
718
|
+
driver_config_sha256=self._config_sha256,
|
|
719
|
+
)
|
|
720
|
+
run_config = _context_run_config(
|
|
721
|
+
runtime.run_config,
|
|
722
|
+
dataset_identity=dataset_identity,
|
|
723
|
+
model_identity=model_identity,
|
|
724
|
+
)
|
|
725
|
+
context = InterventionContext(
|
|
726
|
+
backend=runtime.backend,
|
|
727
|
+
dense_embedder=runtime.dense_embedder,
|
|
728
|
+
retriever=runtime.retriever,
|
|
729
|
+
vector_searcher=runtime.vector_searcher,
|
|
730
|
+
tools=runtime.tools,
|
|
731
|
+
external_runtime=runtime.external_runtime,
|
|
732
|
+
artifact_manager=runtime.artifact_manager,
|
|
733
|
+
seed=cell.seed,
|
|
734
|
+
run_config=run_config,
|
|
735
|
+
intent_contract=runtime.intent_contract,
|
|
736
|
+
monitor=RunMonitor(),
|
|
737
|
+
pricing_catalog=runtime.pricing_catalog,
|
|
738
|
+
strict_intent=runtime.strict_intent,
|
|
739
|
+
)
|
|
740
|
+
contexts.append(context)
|
|
741
|
+
built_arms.append(
|
|
742
|
+
BenchmarkBlockArm(
|
|
743
|
+
comparison_arm=ComparisonArm(
|
|
744
|
+
arm_id=arm_id,
|
|
745
|
+
release_id=arm_spec.release_id,
|
|
746
|
+
bindings=bindings,
|
|
747
|
+
),
|
|
748
|
+
intervention=intervention,
|
|
749
|
+
context=context,
|
|
750
|
+
predictor=predictor,
|
|
751
|
+
)
|
|
752
|
+
)
|
|
753
|
+
|
|
754
|
+
profile = ComparisonProfile(
|
|
755
|
+
profile_id=_PROFILE_ID,
|
|
756
|
+
kind=manifest.estimand_profile_kind,
|
|
757
|
+
required_slots=(
|
|
758
|
+
"dataset",
|
|
759
|
+
"model",
|
|
760
|
+
"downstream_predictor",
|
|
761
|
+
"driver_config",
|
|
762
|
+
),
|
|
763
|
+
varied_slots=("intervention",),
|
|
764
|
+
seed_policy="fixed",
|
|
765
|
+
cache_policy="frozen",
|
|
766
|
+
)
|
|
767
|
+
execution_adapter: BenchmarkAdapter = adapter
|
|
768
|
+
if isinstance(adapter, PreparedRetrievalBenchmarkAdapter) and any(
|
|
769
|
+
arm.family is SemanticFamily.CURATE
|
|
770
|
+
for arm in manifest.arms
|
|
771
|
+
if arm.arm_id != self._config.baseline_arm_id
|
|
772
|
+
):
|
|
773
|
+
execution_adapter = _preload_retrieval_candidates(
|
|
774
|
+
adapter,
|
|
775
|
+
split=cell.split,
|
|
776
|
+
example_id=cell.unit_id,
|
|
777
|
+
retriever=built_arms[0].context.retriever,
|
|
778
|
+
top_k=_retrieval_top_k(benchmark_binding.predictor_builder.parameters),
|
|
779
|
+
)
|
|
780
|
+
block = ControlledMultiArmBlockRunner().run_one(
|
|
781
|
+
execution_adapter,
|
|
782
|
+
cell.split,
|
|
783
|
+
cell.unit_id,
|
|
784
|
+
profile,
|
|
785
|
+
tuple(built_arms),
|
|
786
|
+
planned_arm_ids=cell.arm_ids,
|
|
787
|
+
baseline_arm_id=self._config.baseline_arm_id,
|
|
788
|
+
budget=self._config.execution_budget,
|
|
789
|
+
pre_action_observer=self._pre_action_receipt,
|
|
790
|
+
)
|
|
791
|
+
outcomes = tuple(
|
|
792
|
+
_arm_payload(record, arm_spec=arm_specs[record.arm_id]) for record in block.arms
|
|
793
|
+
)
|
|
794
|
+
if tuple(item["arm_id"] for item in outcomes) != cell.arm_ids:
|
|
795
|
+
raise ValidationError("static production driver produced a partial arm inventory")
|
|
796
|
+
receipts = (
|
|
797
|
+
*_resource_receipts(contexts),
|
|
798
|
+
*(receipt for record in block.arms for receipt in record.downstream_resource_receipts),
|
|
799
|
+
)
|
|
800
|
+
return ExperimentCellResult(
|
|
801
|
+
cell_id=cell.cell_id,
|
|
802
|
+
status="succeeded",
|
|
803
|
+
payload={
|
|
804
|
+
"benchmark_id": block.benchmark_id,
|
|
805
|
+
"split": block.split,
|
|
806
|
+
"unit_id": block.example_id,
|
|
807
|
+
"state_hash": block.state_hash,
|
|
808
|
+
"profile_id": block.profile_id,
|
|
809
|
+
"baseline_arm_id": block.baseline_arm_id,
|
|
810
|
+
"planned_arm_ids": block.planned_arm_ids,
|
|
811
|
+
"pre_action": block.pre_action,
|
|
812
|
+
"arms": outcomes,
|
|
813
|
+
"metric_deltas": block.metric_deltas,
|
|
814
|
+
"balance": to_jsonable(block.balance),
|
|
815
|
+
"observed_dependency_hashes": block.observed_dependency_hashes,
|
|
816
|
+
"dataset_identity_sha256": content_hash(dataset_identity),
|
|
817
|
+
"driver_config_sha256": self._config_sha256,
|
|
818
|
+
"target_source_revision": target.source_revision,
|
|
819
|
+
},
|
|
820
|
+
resource_receipts=receipts,
|
|
821
|
+
)
|
|
822
|
+
|
|
823
|
+
def _manifest_blockers(self, manifest: ExperimentManifest) -> list[str]:
|
|
824
|
+
if not isinstance(manifest, ExperimentManifest):
|
|
825
|
+
return ["manifest must be an ExperimentManifest"]
|
|
826
|
+
blockers: list[str] = []
|
|
827
|
+
if set(manifest.model_slots) != set(self._config.model_slots):
|
|
828
|
+
blockers.append("manifest model_slots differ from the locked driver config")
|
|
829
|
+
manifest_arms = {arm.arm_id: arm for arm in manifest.arms}
|
|
830
|
+
if set(manifest_arms) != set(self._config.arms):
|
|
831
|
+
blockers.append("manifest arms differ from the locked driver config")
|
|
832
|
+
else:
|
|
833
|
+
for arm_id, arm in manifest_arms.items():
|
|
834
|
+
configured = self._config.arms[arm_id]
|
|
835
|
+
if configured.release_id != arm.release_id:
|
|
836
|
+
blockers.append(
|
|
837
|
+
f"arm {arm_id}: manifest release differs from the locked config"
|
|
838
|
+
)
|
|
839
|
+
no_op_ids = tuple(arm.arm_id for arm in manifest.arms if arm.family is SemanticFamily.NO_OP)
|
|
840
|
+
if no_op_ids != (self._config.baseline_arm_id,):
|
|
841
|
+
blockers.append(
|
|
842
|
+
"manifest requires exactly one NO_OP arm matching configured baseline_arm_id"
|
|
843
|
+
)
|
|
844
|
+
seen_cells: set[str] = set()
|
|
845
|
+
for target in manifest.benchmarks:
|
|
846
|
+
if target.external:
|
|
847
|
+
blockers.append(
|
|
848
|
+
f"benchmark {target.cell_id}: stateful/external targets require "
|
|
849
|
+
"an atomic replay driver"
|
|
850
|
+
)
|
|
851
|
+
continue
|
|
852
|
+
if target.cell_id not in _STATIC_SPLITS:
|
|
853
|
+
blockers.append(
|
|
854
|
+
f"benchmark {target.cell_id}: target is not one of the seven static "
|
|
855
|
+
"production adapters"
|
|
856
|
+
)
|
|
857
|
+
continue
|
|
858
|
+
if target.cell_id in seen_cells:
|
|
859
|
+
blockers.append(f"benchmark {target.cell_id}: duplicate target")
|
|
860
|
+
seen_cells.add(target.cell_id)
|
|
861
|
+
if target.split != _STATIC_SPLITS[target.cell_id]:
|
|
862
|
+
blockers.append(
|
|
863
|
+
f"benchmark {target.cell_id}: split differs from the production lock"
|
|
864
|
+
)
|
|
865
|
+
selected = target.arm_ids or tuple(manifest_arms)
|
|
866
|
+
if self._config.baseline_arm_id not in selected:
|
|
867
|
+
blockers.append(
|
|
868
|
+
f"benchmark {target.cell_id}: selected arm block omits the NO_OP baseline"
|
|
869
|
+
)
|
|
870
|
+
return blockers
|
|
871
|
+
|
|
872
|
+
def _validate_cell(
|
|
873
|
+
self,
|
|
874
|
+
manifest: ExperimentManifest,
|
|
875
|
+
cell: ExperimentCell,
|
|
876
|
+
) -> ExperimentBenchmarkTarget:
|
|
877
|
+
if not isinstance(cell, ExperimentCell):
|
|
878
|
+
raise TypeError("cell must be ExperimentCell")
|
|
879
|
+
if cell.study_id != manifest.study_id or cell.manifest_hash != manifest.manifest_hash:
|
|
880
|
+
raise ValidationError("experiment cell does not belong to this manifest")
|
|
881
|
+
matches = tuple(
|
|
882
|
+
target
|
|
883
|
+
for target in manifest.benchmarks
|
|
884
|
+
if target.cell_id == cell.benchmark_id and target.split == cell.split
|
|
885
|
+
)
|
|
886
|
+
if len(matches) != 1:
|
|
887
|
+
raise ValidationError("experiment cell target is absent or ambiguous")
|
|
888
|
+
target = matches[0]
|
|
889
|
+
expected_arms = target.arm_ids or tuple(arm.arm_id for arm in manifest.arms)
|
|
890
|
+
if cell.arm_ids != expected_arms:
|
|
891
|
+
raise ValidationError("experiment cell arms differ from the frozen target block")
|
|
892
|
+
if cell.model_slot not in manifest.model_slots:
|
|
893
|
+
raise ValidationError("experiment cell model_slot is outside the manifest")
|
|
894
|
+
if cell.seed not in manifest.seeds:
|
|
895
|
+
raise ValidationError("experiment cell seed is outside the manifest")
|
|
896
|
+
if target.unit_ids and cell.unit_id not in target.unit_ids:
|
|
897
|
+
raise ValidationError("experiment cell unit is outside the frozen target inventory")
|
|
898
|
+
return target
|
|
899
|
+
|
|
900
|
+
def _build_runtime(self, cell: ExperimentCell) -> StaticRuntimeComponents:
|
|
901
|
+
spec = self._config.model_slots[cell.model_slot]
|
|
902
|
+
builder = _resolve_locked_builder(spec, expected_kind="backend")
|
|
903
|
+
runtime = builder.build(
|
|
904
|
+
StaticBackendBuildRequest(
|
|
905
|
+
model_slot=cell.model_slot,
|
|
906
|
+
seed=cell.seed,
|
|
907
|
+
builder_parameters=spec.parameters,
|
|
908
|
+
)
|
|
909
|
+
)
|
|
910
|
+
if not isinstance(runtime, StaticRuntimeComponents):
|
|
911
|
+
raise ValidationError(
|
|
912
|
+
f"backend builder must return StaticRuntimeComponents, got {type(runtime).__name__}"
|
|
913
|
+
)
|
|
914
|
+
_require_product_sha256(
|
|
915
|
+
static_runtime_product_sha256(runtime),
|
|
916
|
+
expected=spec.expected_product_sha256,
|
|
917
|
+
label="static runtime product",
|
|
918
|
+
)
|
|
919
|
+
return runtime
|
|
920
|
+
|
|
921
|
+
def _build_predictor(
|
|
922
|
+
self,
|
|
923
|
+
cell: ExperimentCell,
|
|
924
|
+
*,
|
|
925
|
+
runtime: StaticRuntimeComponents,
|
|
926
|
+
binding: _BenchmarkBinding,
|
|
927
|
+
) -> DownstreamPredictor:
|
|
928
|
+
spec = binding.predictor_builder
|
|
929
|
+
builder = _resolve_locked_builder(spec, expected_kind="predictor")
|
|
930
|
+
predictor = builder.build(
|
|
931
|
+
StaticPredictorBuildRequest(
|
|
932
|
+
benchmark_id=cell.benchmark_id,
|
|
933
|
+
split=cell.split,
|
|
934
|
+
model_slot=cell.model_slot,
|
|
935
|
+
seed=cell.seed,
|
|
936
|
+
builder_parameters=spec.parameters,
|
|
937
|
+
runtime=runtime,
|
|
938
|
+
)
|
|
939
|
+
)
|
|
940
|
+
if not isinstance(predictor, DownstreamPredictor):
|
|
941
|
+
raise ValidationError(
|
|
942
|
+
"predictor builder must return a DownstreamPredictor, "
|
|
943
|
+
f"got {type(predictor).__name__}"
|
|
944
|
+
)
|
|
945
|
+
_require_product_sha256(
|
|
946
|
+
static_predictor_product_sha256(predictor),
|
|
947
|
+
expected=spec.expected_product_sha256,
|
|
948
|
+
label="static predictor product",
|
|
949
|
+
)
|
|
950
|
+
return predictor
|
|
951
|
+
|
|
952
|
+
def _build_intervention(
|
|
953
|
+
self,
|
|
954
|
+
cell: ExperimentCell,
|
|
955
|
+
*,
|
|
956
|
+
arm_spec: ExperimentArmSpec,
|
|
957
|
+
runtime: StaticRuntimeComponents,
|
|
958
|
+
) -> Intervention:
|
|
959
|
+
binding = self._config.arms[arm_spec.arm_id]
|
|
960
|
+
spec = binding.intervention_builder
|
|
961
|
+
builder = _resolve_locked_builder(spec, expected_kind="intervention")
|
|
962
|
+
intervention = builder.build(
|
|
963
|
+
StaticArmBuildRequest(
|
|
964
|
+
arm_id=arm_spec.arm_id,
|
|
965
|
+
release_id=arm_spec.release_id,
|
|
966
|
+
family=arm_spec.family,
|
|
967
|
+
profile_kind=arm_spec.profile_kind,
|
|
968
|
+
model_slot=cell.model_slot,
|
|
969
|
+
seed=cell.seed,
|
|
970
|
+
builder_parameters=spec.parameters,
|
|
971
|
+
arm_parameters=arm_spec.parameters,
|
|
972
|
+
runtime=runtime,
|
|
973
|
+
)
|
|
974
|
+
)
|
|
975
|
+
if not isinstance(intervention, Intervention):
|
|
976
|
+
raise ValidationError(
|
|
977
|
+
"intervention builder must return an Intervention, "
|
|
978
|
+
f"got {type(intervention).__name__}"
|
|
979
|
+
)
|
|
980
|
+
if intervention.release_id != arm_spec.release_id:
|
|
981
|
+
raise ValidationError("intervention builder returned another release_id")
|
|
982
|
+
if intervention.family is not arm_spec.family:
|
|
983
|
+
raise ValidationError("intervention builder returned another semantic family")
|
|
984
|
+
return intervention
|
|
985
|
+
|
|
986
|
+
def _pre_action_receipt(self, state: FrozenAgentState) -> Mapping[str, Any]:
|
|
987
|
+
observation = self._feature_extractor.extract(state)
|
|
988
|
+
features = observation.features
|
|
989
|
+
return {
|
|
990
|
+
"feature_schema_id": self._feature_extractor.feature_schema_id,
|
|
991
|
+
"feature_schema_hash": self._feature_extractor.feature_schema_hash,
|
|
992
|
+
"state_hash": observation.state_hash,
|
|
993
|
+
"feature_hash": observation.feature_hash,
|
|
994
|
+
"provenance_hash": content_hash(observation.provenance),
|
|
995
|
+
"defect_scores": {
|
|
996
|
+
SemanticFamily.REFORMULATE.value: features.reformulation_need_score,
|
|
997
|
+
SemanticFamily.DECOMPOSE.value: features.multi_step_score,
|
|
998
|
+
SemanticFamily.ACQUIRE.value: features.retrieval_need_score,
|
|
999
|
+
SemanticFamily.CLARIFY.value: features.ambiguity_score,
|
|
1000
|
+
SemanticFamily.CURATE.value: features.context_pressure,
|
|
1001
|
+
},
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
def factory() -> StaticProductionExperimentDriver:
|
|
1006
|
+
"""Zero-argument factory driven only by two explicit environment bindings."""
|
|
1007
|
+
|
|
1008
|
+
path = os.environ.get(CONFIG_PATH_ENV)
|
|
1009
|
+
expected_sha256 = os.environ.get(CONFIG_SHA256_ENV)
|
|
1010
|
+
if not path or not expected_sha256:
|
|
1011
|
+
raise ArtifactError(
|
|
1012
|
+
f"set both {CONFIG_PATH_ENV} and {CONFIG_SHA256_ENV} before loading "
|
|
1013
|
+
"the static production driver"
|
|
1014
|
+
)
|
|
1015
|
+
return StaticProductionExperimentDriver.from_file(
|
|
1016
|
+
path,
|
|
1017
|
+
expected_sha256=expected_sha256,
|
|
1018
|
+
)
|
|
1019
|
+
|
|
1020
|
+
|
|
1021
|
+
def _resolve_locked_builder(
|
|
1022
|
+
spec: LockedBuilderSpec,
|
|
1023
|
+
*,
|
|
1024
|
+
expected_kind: str,
|
|
1025
|
+
) -> LockedComponentBuilder:
|
|
1026
|
+
if expected_kind not in _BUILDER_KINDS: # pragma: no cover - internal invariant
|
|
1027
|
+
raise ValueError("unknown builder kind")
|
|
1028
|
+
candidate = resolve_driver_entrypoint(spec.entrypoint)
|
|
1029
|
+
if not callable(candidate):
|
|
1030
|
+
raise ArtifactError("component builder entrypoint must be a zero-argument factory")
|
|
1031
|
+
try:
|
|
1032
|
+
builder = candidate()
|
|
1033
|
+
except TypeError as exc:
|
|
1034
|
+
raise ArtifactError("component builder entrypoint must be a zero-argument factory") from exc
|
|
1035
|
+
if not isinstance(builder, LockedComponentBuilder):
|
|
1036
|
+
raise ArtifactError("component builder does not implement the locked builder contract")
|
|
1037
|
+
if builder.builder_kind != expected_kind:
|
|
1038
|
+
raise ArtifactError(
|
|
1039
|
+
f"component builder kind is {builder.builder_kind!r}, expected {expected_kind!r}"
|
|
1040
|
+
)
|
|
1041
|
+
identity = _runtime_fingerprint(builder, label=f"{expected_kind} builder")
|
|
1042
|
+
if content_hash(identity) != spec.fingerprint_sha256:
|
|
1043
|
+
raise ArtifactError(f"{expected_kind} builder fingerprint checksum mismatch")
|
|
1044
|
+
return builder
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
def _builder_preflight(
|
|
1048
|
+
builder: LockedComponentBuilder,
|
|
1049
|
+
parameters: Mapping[str, Any],
|
|
1050
|
+
) -> tuple[str, ...]:
|
|
1051
|
+
blockers = builder.preflight(parameters)
|
|
1052
|
+
if isinstance(blockers, (str, bytes)) or not isinstance(blockers, Sequence):
|
|
1053
|
+
raise ValidationError("component builder preflight must return a sequence of strings")
|
|
1054
|
+
normalized = tuple(blockers)
|
|
1055
|
+
if any(not isinstance(item, str) or not item.strip() for item in normalized):
|
|
1056
|
+
raise ValidationError("component builder preflight blockers must be non-empty strings")
|
|
1057
|
+
return normalized
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
def _comparison_bindings(
|
|
1061
|
+
*,
|
|
1062
|
+
dataset_identity: Mapping[str, Any],
|
|
1063
|
+
model_identity: Mapping[str, Any],
|
|
1064
|
+
predictor_identity: Mapping[str, Any],
|
|
1065
|
+
arm_spec: ExperimentArmSpec,
|
|
1066
|
+
builder_spec: LockedBuilderSpec,
|
|
1067
|
+
intervention: Intervention,
|
|
1068
|
+
driver_config_sha256: str,
|
|
1069
|
+
) -> tuple[DependencyBinding, ...]:
|
|
1070
|
+
return (
|
|
1071
|
+
DependencyBinding(
|
|
1072
|
+
slot="dataset",
|
|
1073
|
+
role=DependencyRole.NUISANCE,
|
|
1074
|
+
implementation_id=str(dataset_identity["benchmark_id"]),
|
|
1075
|
+
revision=content_hash(dataset_identity),
|
|
1076
|
+
config={"identity": dataset_identity},
|
|
1077
|
+
),
|
|
1078
|
+
DependencyBinding(
|
|
1079
|
+
slot="model",
|
|
1080
|
+
role=DependencyRole.NUISANCE,
|
|
1081
|
+
implementation_id="operator_backend",
|
|
1082
|
+
revision=content_hash(model_identity),
|
|
1083
|
+
config={"fingerprint": model_identity},
|
|
1084
|
+
),
|
|
1085
|
+
DependencyBinding(
|
|
1086
|
+
slot="downstream_predictor",
|
|
1087
|
+
role=DependencyRole.NUISANCE,
|
|
1088
|
+
implementation_id="operator_predictor",
|
|
1089
|
+
revision=content_hash(predictor_identity),
|
|
1090
|
+
config={"fingerprint": predictor_identity},
|
|
1091
|
+
),
|
|
1092
|
+
DependencyBinding(
|
|
1093
|
+
slot="driver_config",
|
|
1094
|
+
role=DependencyRole.NUISANCE,
|
|
1095
|
+
implementation_id=_DRIVER_IMPLEMENTATION_ID,
|
|
1096
|
+
revision=driver_config_sha256,
|
|
1097
|
+
config={
|
|
1098
|
+
"schema_version": _CONFIG_SCHEMA_VERSION,
|
|
1099
|
+
"config_sha256": driver_config_sha256,
|
|
1100
|
+
},
|
|
1101
|
+
),
|
|
1102
|
+
DependencyBinding(
|
|
1103
|
+
slot="intervention",
|
|
1104
|
+
role=DependencyRole.TREATMENT,
|
|
1105
|
+
implementation_id=intervention.method_id,
|
|
1106
|
+
revision=builder_spec.fingerprint_sha256,
|
|
1107
|
+
config={
|
|
1108
|
+
"release_id": arm_spec.release_id,
|
|
1109
|
+
"family": arm_spec.family.value,
|
|
1110
|
+
"profile_kind": arm_spec.profile_kind.value,
|
|
1111
|
+
"arm_parameters": arm_spec.parameters,
|
|
1112
|
+
"builder": builder_spec.binding_identity,
|
|
1113
|
+
},
|
|
1114
|
+
),
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def _context_run_config(
|
|
1119
|
+
value: Mapping[str, Any],
|
|
1120
|
+
*,
|
|
1121
|
+
dataset_identity: Mapping[str, Any],
|
|
1122
|
+
model_identity: Mapping[str, Any],
|
|
1123
|
+
) -> Mapping[str, Any]:
|
|
1124
|
+
run_config = dict(value)
|
|
1125
|
+
declared = run_config.pop("dependency_fingerprints", {})
|
|
1126
|
+
if not isinstance(declared, Mapping):
|
|
1127
|
+
raise ValidationError("runtime dependency_fingerprints must be an object")
|
|
1128
|
+
dependencies = dict(declared)
|
|
1129
|
+
collisions = {"dataset", "model"} & set(dependencies)
|
|
1130
|
+
if collisions:
|
|
1131
|
+
raise ValidationError(
|
|
1132
|
+
"backend builder cannot override driver-owned dependency slots: "
|
|
1133
|
+
+ ", ".join(sorted(collisions))
|
|
1134
|
+
)
|
|
1135
|
+
dependencies["dataset"] = dataset_identity
|
|
1136
|
+
dependencies["model"] = model_identity
|
|
1137
|
+
run_config["dependency_fingerprints"] = dependencies
|
|
1138
|
+
return freeze_mapping(run_config)
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
def _dataset_identity(
|
|
1142
|
+
adapter: BenchmarkAdapter,
|
|
1143
|
+
*,
|
|
1144
|
+
split: str,
|
|
1145
|
+
) -> Mapping[str, Any]:
|
|
1146
|
+
if isinstance(adapter, PreparedQABenchmarkAdapter):
|
|
1147
|
+
assets = {"query": _prepared_identity(adapter.prepared)}
|
|
1148
|
+
elif isinstance(adapter, PreparedRetrievalBenchmarkAdapter):
|
|
1149
|
+
assets = {
|
|
1150
|
+
"query": _prepared_identity(adapter.query_prepared),
|
|
1151
|
+
"corpus": _prepared_identity(adapter.corpus_prepared),
|
|
1152
|
+
}
|
|
1153
|
+
else:
|
|
1154
|
+
raise ValidationError("static production driver received an unsupported adapter")
|
|
1155
|
+
return freeze_mapping(
|
|
1156
|
+
{
|
|
1157
|
+
"benchmark_id": adapter.benchmark_id,
|
|
1158
|
+
"split": split,
|
|
1159
|
+
"assets": assets,
|
|
1160
|
+
}
|
|
1161
|
+
)
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
def _prepared_identity(prepared: Any) -> Mapping[str, Any]:
|
|
1165
|
+
required = (
|
|
1166
|
+
"asset_id",
|
|
1167
|
+
"cell_id",
|
|
1168
|
+
"source_revision",
|
|
1169
|
+
"source_sha256",
|
|
1170
|
+
"processor",
|
|
1171
|
+
"processor_revision",
|
|
1172
|
+
"counts",
|
|
1173
|
+
"file_sha256",
|
|
1174
|
+
)
|
|
1175
|
+
if any(not hasattr(prepared, name) for name in required):
|
|
1176
|
+
raise ValidationError("prepared benchmark does not expose its audit identity")
|
|
1177
|
+
return freeze_mapping({name: getattr(prepared, name) for name in required})
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
def _runtime_fingerprint(value: Any, *, label: str) -> Mapping[str, Any]:
|
|
1181
|
+
fingerprint = getattr(value, "fingerprint", None)
|
|
1182
|
+
if not callable(fingerprint):
|
|
1183
|
+
raise ValidationError(f"{label} must expose fingerprint()")
|
|
1184
|
+
identity = fingerprint()
|
|
1185
|
+
if not isinstance(identity, Mapping) or not identity:
|
|
1186
|
+
raise ValidationError(f"{label} fingerprint must be a non-empty mapping")
|
|
1187
|
+
try:
|
|
1188
|
+
return freeze_mapping(identity)
|
|
1189
|
+
except (TypeError, ValueError) as exc:
|
|
1190
|
+
raise ValidationError(f"{label} fingerprint must be canonical JSON") from exc
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
def static_runtime_product_sha256(runtime: StaticRuntimeComponents) -> str:
|
|
1194
|
+
"""Hash every object and option that can affect a static arm execution.
|
|
1195
|
+
|
|
1196
|
+
This is intentionally stricter than locking the backend builder. A builder
|
|
1197
|
+
can stay byte-for-byte identical while its endpoint routing, model, index,
|
|
1198
|
+
tools, runtime assets, prompt defaults, intent policy, or pricing inputs
|
|
1199
|
+
drift. Operators place this returned hash in ``expected_product_sha256``;
|
|
1200
|
+
every fresh product is checked before it can execute.
|
|
1201
|
+
"""
|
|
1202
|
+
|
|
1203
|
+
if not isinstance(runtime, StaticRuntimeComponents):
|
|
1204
|
+
raise TypeError("runtime product hashing requires StaticRuntimeComponents")
|
|
1205
|
+
component_slots = (
|
|
1206
|
+
"backend",
|
|
1207
|
+
"dense_embedder",
|
|
1208
|
+
"retriever",
|
|
1209
|
+
"vector_searcher",
|
|
1210
|
+
"tools",
|
|
1211
|
+
"external_runtime",
|
|
1212
|
+
"artifact_manager",
|
|
1213
|
+
)
|
|
1214
|
+
components: dict[str, Any] = {}
|
|
1215
|
+
for slot in component_slots:
|
|
1216
|
+
value = getattr(runtime, slot)
|
|
1217
|
+
components[slot] = (
|
|
1218
|
+
None if value is None else _runtime_fingerprint(value, label=f"runtime {slot}")
|
|
1219
|
+
)
|
|
1220
|
+
product_identity = {
|
|
1221
|
+
"components": components,
|
|
1222
|
+
"run_config": runtime.run_config,
|
|
1223
|
+
"intent_contract": runtime.intent_contract,
|
|
1224
|
+
"pricing_catalog": runtime.pricing_catalog,
|
|
1225
|
+
"strict_intent": runtime.strict_intent,
|
|
1226
|
+
}
|
|
1227
|
+
try:
|
|
1228
|
+
return content_hash(product_identity)
|
|
1229
|
+
except (TypeError, ValueError) as exc:
|
|
1230
|
+
raise ValidationError("static runtime product identity must be canonical JSON") from exc
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
def static_predictor_product_sha256(predictor: DownstreamPredictor) -> str:
|
|
1234
|
+
"""Hash the actual downstream predictor returned by its locked builder."""
|
|
1235
|
+
|
|
1236
|
+
if not isinstance(predictor, DownstreamPredictor):
|
|
1237
|
+
raise TypeError("predictor product hashing requires a DownstreamPredictor")
|
|
1238
|
+
return content_hash(_runtime_fingerprint(predictor, label="downstream predictor"))
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
def _require_product_sha256(
|
|
1242
|
+
observed: str,
|
|
1243
|
+
*,
|
|
1244
|
+
expected: str | None,
|
|
1245
|
+
label: str,
|
|
1246
|
+
) -> str:
|
|
1247
|
+
if expected is None: # pragma: no cover - rejected by config construction
|
|
1248
|
+
raise ArtifactError(f"{label} has no configured checksum")
|
|
1249
|
+
if observed != expected:
|
|
1250
|
+
raise ArtifactError(f"{label} checksum mismatch")
|
|
1251
|
+
return observed
|
|
1252
|
+
|
|
1253
|
+
|
|
1254
|
+
def _arm_payload(record: Any, *, arm_spec: ExperimentArmSpec) -> Mapping[str, Any]:
|
|
1255
|
+
base: dict[str, Any] = {
|
|
1256
|
+
"arm_id": record.arm_id,
|
|
1257
|
+
"release_id": record.release_id,
|
|
1258
|
+
"family": arm_spec.family.value,
|
|
1259
|
+
"profile_kind": arm_spec.profile_kind.value,
|
|
1260
|
+
"status": record.status,
|
|
1261
|
+
"usage": record.total_usage.as_dict(),
|
|
1262
|
+
"downstream_duration_ms": record.downstream_duration_ms,
|
|
1263
|
+
"downstream_resource_receipt_hashes": tuple(
|
|
1264
|
+
content_hash(receipt) for receipt in record.downstream_resource_receipts
|
|
1265
|
+
),
|
|
1266
|
+
}
|
|
1267
|
+
intervention_record = record.intervention_record
|
|
1268
|
+
if intervention_record is not None:
|
|
1269
|
+
patch = intervention_record.patch
|
|
1270
|
+
if patch.profile_kind != arm_spec.profile_kind.value:
|
|
1271
|
+
raise ValidationError(
|
|
1272
|
+
f"arm {record.arm_id!r} patch profile_kind differs from the manifest"
|
|
1273
|
+
)
|
|
1274
|
+
base.update(
|
|
1275
|
+
{
|
|
1276
|
+
"intervention_status": intervention_record.status.value,
|
|
1277
|
+
"intervention_duration_ms": intervention_record.duration_ms,
|
|
1278
|
+
"monitor_run_id": intervention_record.monitor_run_id,
|
|
1279
|
+
"patch_hash": patch.patch_hash,
|
|
1280
|
+
"trace_hash": patch.trace.trace_hash,
|
|
1281
|
+
}
|
|
1282
|
+
)
|
|
1283
|
+
if record.status == "succeeded":
|
|
1284
|
+
if record.benchmark_record is None: # pragma: no cover - record invariant
|
|
1285
|
+
raise RuntimeError("successful arm lost its benchmark record")
|
|
1286
|
+
base["metrics"] = record.benchmark_record.metrics
|
|
1287
|
+
base["evaluation_provenance"] = record.benchmark_record.evaluation_provenance
|
|
1288
|
+
else:
|
|
1289
|
+
if record.failure is None: # pragma: no cover - record invariant
|
|
1290
|
+
raise RuntimeError("failed arm lost its failure")
|
|
1291
|
+
base["failure"] = {
|
|
1292
|
+
"scope": "algorithm",
|
|
1293
|
+
"category": record.failure.category,
|
|
1294
|
+
"message": _redact_secret_text(record.failure.message),
|
|
1295
|
+
"monitor_run_id": record.failure.monitor_run_id,
|
|
1296
|
+
}
|
|
1297
|
+
return freeze_mapping(base)
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
def _resource_receipts(
|
|
1301
|
+
contexts: Sequence[InterventionContext],
|
|
1302
|
+
) -> tuple[ResourceReceipt, ...]:
|
|
1303
|
+
receipts: list[ResourceReceipt] = []
|
|
1304
|
+
for context in contexts:
|
|
1305
|
+
monitor = context.monitor
|
|
1306
|
+
if not isinstance(monitor, RunMonitor): # pragma: no cover - construction invariant
|
|
1307
|
+
continue
|
|
1308
|
+
for snapshot in monitor.list_snapshots():
|
|
1309
|
+
receipts.extend(snapshot.resource_receipts)
|
|
1310
|
+
return tuple(receipts)
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
class _PreloadedRetrievalAdapter:
|
|
1314
|
+
"""Single-unit view that exposes one shared ranked candidate context.
|
|
1315
|
+
|
|
1316
|
+
Gold remains owned by the wrapped production adapter. Both arms receive
|
|
1317
|
+
the exact same visible candidate list; the wrapper forwards scoring to the
|
|
1318
|
+
original example so evaluator isolation and official metrics are unchanged.
|
|
1319
|
+
"""
|
|
1320
|
+
|
|
1321
|
+
pairing_mode = "stateless_score_twice"
|
|
1322
|
+
|
|
1323
|
+
def __init__(
|
|
1324
|
+
self,
|
|
1325
|
+
base: PreparedRetrievalBenchmarkAdapter,
|
|
1326
|
+
original: BenchmarkExample,
|
|
1327
|
+
visible: BenchmarkExample,
|
|
1328
|
+
) -> None:
|
|
1329
|
+
self.benchmark_id = base.benchmark_id
|
|
1330
|
+
self._base = base
|
|
1331
|
+
self._original = original
|
|
1332
|
+
self._visible = visible
|
|
1333
|
+
|
|
1334
|
+
def examples(self, split: str) -> tuple[BenchmarkExample, ...]:
|
|
1335
|
+
if split != self._base.split:
|
|
1336
|
+
raise ValidationError("preloaded retrieval adapter split differs")
|
|
1337
|
+
return (self._visible,)
|
|
1338
|
+
|
|
1339
|
+
def score(self, example: BenchmarkExample, prediction: Any) -> Mapping[str, float]:
|
|
1340
|
+
if example is not self._visible:
|
|
1341
|
+
raise ValidationError("preloaded retrieval example differs from the frozen unit")
|
|
1342
|
+
return self._base.score(self._original, prediction)
|
|
1343
|
+
|
|
1344
|
+
|
|
1345
|
+
def _preload_retrieval_candidates(
|
|
1346
|
+
adapter: PreparedRetrievalBenchmarkAdapter,
|
|
1347
|
+
*,
|
|
1348
|
+
split: str,
|
|
1349
|
+
example_id: str,
|
|
1350
|
+
retriever: Any,
|
|
1351
|
+
top_k: int,
|
|
1352
|
+
) -> BenchmarkAdapter:
|
|
1353
|
+
if not isinstance(retriever, Retriever):
|
|
1354
|
+
raise ValidationError(
|
|
1355
|
+
"curation on a retrieval benchmark requires the shared locked retriever"
|
|
1356
|
+
)
|
|
1357
|
+
original = adapter.example(split, example_id)
|
|
1358
|
+
results = tuple(retriever.retrieve(original.query, top_k=top_k))
|
|
1359
|
+
if not results or any(not isinstance(item, RetrievalResult) for item in results):
|
|
1360
|
+
raise ValidationError("shared retriever returned no valid curation candidates")
|
|
1361
|
+
expected_ranks = tuple(range(1, len(results) + 1))
|
|
1362
|
+
if tuple(item.rank for item in results) != expected_ranks:
|
|
1363
|
+
raise ValidationError("curation candidates require contiguous ranks from one")
|
|
1364
|
+
document_ids = tuple(item.document.doc_id for item in results)
|
|
1365
|
+
if len(document_ids) != len(set(document_ids)):
|
|
1366
|
+
raise ValidationError("curation candidates contain duplicate document IDs")
|
|
1367
|
+
context = tuple(
|
|
1368
|
+
ContextBlock(
|
|
1369
|
+
block_id=item.document.doc_id,
|
|
1370
|
+
text="\n".join(
|
|
1371
|
+
part for part in (item.document.title, item.document.text) if part.strip()
|
|
1372
|
+
),
|
|
1373
|
+
source="shared_locked_retriever",
|
|
1374
|
+
score=item.score,
|
|
1375
|
+
metadata={
|
|
1376
|
+
"document_id": item.document.doc_id,
|
|
1377
|
+
"retrieval_rank": item.rank,
|
|
1378
|
+
"retrieval_score": item.score,
|
|
1379
|
+
"corpus_revision": item.document.corpus_revision,
|
|
1380
|
+
"candidate_set_role": "pre_action_shared_context",
|
|
1381
|
+
},
|
|
1382
|
+
)
|
|
1383
|
+
for item in results
|
|
1384
|
+
)
|
|
1385
|
+
visible = BenchmarkExample(
|
|
1386
|
+
example_id=original.example_id,
|
|
1387
|
+
query=original.query,
|
|
1388
|
+
instruction=original.instruction,
|
|
1389
|
+
visible_context=context,
|
|
1390
|
+
visible_messages=original.visible_messages,
|
|
1391
|
+
visible_metadata={
|
|
1392
|
+
**dict(original.visible_metadata),
|
|
1393
|
+
"preloaded_candidate_count": len(context),
|
|
1394
|
+
"preloaded_candidate_ids_sha256": content_hash(document_ids),
|
|
1395
|
+
},
|
|
1396
|
+
evaluator_gold={},
|
|
1397
|
+
visible_tool_names=original.visible_tool_names,
|
|
1398
|
+
optimizer_visible_id=original.optimizer_visible_id,
|
|
1399
|
+
)
|
|
1400
|
+
return _PreloadedRetrievalAdapter(adapter, original, visible)
|
|
1401
|
+
|
|
1402
|
+
|
|
1403
|
+
def _retrieval_top_k(parameters: Mapping[str, Any]) -> int:
|
|
1404
|
+
retrieval = parameters.get("retrieval", {})
|
|
1405
|
+
if not isinstance(retrieval, Mapping):
|
|
1406
|
+
raise ValidationError("retrieval predictor parameters are invalid")
|
|
1407
|
+
value = retrieval.get("top_k", 100)
|
|
1408
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|
1409
|
+
raise ValidationError("retrieval predictor top_k must be positive")
|
|
1410
|
+
return value
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def _benchmark_binding(value: Any, *, path: str) -> _BenchmarkBinding:
|
|
1414
|
+
data = _strict_mapping(
|
|
1415
|
+
value,
|
|
1416
|
+
path=path,
|
|
1417
|
+
keys={"split", "predictor_builder"},
|
|
1418
|
+
required={"split", "predictor_builder"},
|
|
1419
|
+
)
|
|
1420
|
+
return _BenchmarkBinding(
|
|
1421
|
+
split=_text(data["split"], f"{path}.split"),
|
|
1422
|
+
predictor_builder=LockedBuilderSpec.from_mapping(
|
|
1423
|
+
data["predictor_builder"],
|
|
1424
|
+
path=f"{path}.predictor_builder",
|
|
1425
|
+
),
|
|
1426
|
+
)
|
|
1427
|
+
|
|
1428
|
+
|
|
1429
|
+
def _arm_binding(value: Any, *, path: str) -> _ArmBinding:
|
|
1430
|
+
data = _strict_mapping(
|
|
1431
|
+
value,
|
|
1432
|
+
path=path,
|
|
1433
|
+
keys={"release_id", "intervention_builder"},
|
|
1434
|
+
required={"release_id", "intervention_builder"},
|
|
1435
|
+
)
|
|
1436
|
+
return _ArmBinding(
|
|
1437
|
+
release_id=_text(data["release_id"], f"{path}.release_id"),
|
|
1438
|
+
intervention_builder=LockedBuilderSpec.from_mapping(
|
|
1439
|
+
data["intervention_builder"],
|
|
1440
|
+
path=f"{path}.intervention_builder",
|
|
1441
|
+
),
|
|
1442
|
+
)
|
|
1443
|
+
|
|
1444
|
+
|
|
1445
|
+
def _execution_budget(value: Any, *, path: str) -> ExecutionBudget:
|
|
1446
|
+
data = _strict_mapping(
|
|
1447
|
+
value,
|
|
1448
|
+
path=path,
|
|
1449
|
+
keys={
|
|
1450
|
+
"max_model_calls",
|
|
1451
|
+
"max_retrieval_calls",
|
|
1452
|
+
"max_tool_calls",
|
|
1453
|
+
"max_output_tokens",
|
|
1454
|
+
"max_context_tokens",
|
|
1455
|
+
"max_cost_usd",
|
|
1456
|
+
},
|
|
1457
|
+
required=set(),
|
|
1458
|
+
)
|
|
1459
|
+
try:
|
|
1460
|
+
return ExecutionBudget(**data)
|
|
1461
|
+
except (TypeError, ValueError) as exc:
|
|
1462
|
+
raise ValidationError(f"{path} is invalid: {exc}") from exc
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
def _strict_json(raw: bytes) -> Mapping[str, Any]:
|
|
1466
|
+
try:
|
|
1467
|
+
text = raw.decode("utf-8")
|
|
1468
|
+
parsed = json.loads(
|
|
1469
|
+
text,
|
|
1470
|
+
object_pairs_hook=_reject_duplicate_keys,
|
|
1471
|
+
parse_constant=lambda value: _reject_json_constant(value),
|
|
1472
|
+
)
|
|
1473
|
+
except (UnicodeError, json.JSONDecodeError) as exc:
|
|
1474
|
+
raise ArtifactError("static production config is not strict UTF-8 JSON") from exc
|
|
1475
|
+
if not isinstance(parsed, Mapping):
|
|
1476
|
+
raise ValidationError("static production config root must be an object")
|
|
1477
|
+
return parsed
|
|
1478
|
+
|
|
1479
|
+
|
|
1480
|
+
def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
1481
|
+
result: dict[str, Any] = {}
|
|
1482
|
+
for key, value in pairs:
|
|
1483
|
+
if key in result:
|
|
1484
|
+
raise ValidationError(f"duplicate JSON key: {key}")
|
|
1485
|
+
result[key] = value
|
|
1486
|
+
return result
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def _reject_json_constant(value: str) -> None:
|
|
1490
|
+
raise ValidationError(f"non-finite JSON constant is forbidden: {value}")
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def _strict_mapping(
|
|
1494
|
+
value: Any,
|
|
1495
|
+
*,
|
|
1496
|
+
path: str,
|
|
1497
|
+
keys: set[str],
|
|
1498
|
+
required: set[str],
|
|
1499
|
+
) -> Mapping[str, Any]:
|
|
1500
|
+
if not isinstance(value, Mapping):
|
|
1501
|
+
raise ValidationError(f"{path} must be an object")
|
|
1502
|
+
if any(not isinstance(key, str) for key in value):
|
|
1503
|
+
raise ValidationError(f"{path} keys must be strings")
|
|
1504
|
+
unknown = set(value) - keys
|
|
1505
|
+
missing = required - set(value)
|
|
1506
|
+
if unknown:
|
|
1507
|
+
raise ValidationError(f"{path} has unknown fields: {', '.join(sorted(unknown))}")
|
|
1508
|
+
if missing:
|
|
1509
|
+
raise ValidationError(f"{path} is missing fields: {', '.join(sorted(missing))}")
|
|
1510
|
+
return value
|
|
1511
|
+
|
|
1512
|
+
|
|
1513
|
+
def _strict_object(value: Any, *, path: str) -> Mapping[str, Any]:
|
|
1514
|
+
if not isinstance(value, Mapping) or not value:
|
|
1515
|
+
raise ValidationError(f"{path} must be a non-empty object")
|
|
1516
|
+
if any(not isinstance(key, str) or not key.strip() for key in value):
|
|
1517
|
+
raise ValidationError(f"{path} keys must be non-empty strings")
|
|
1518
|
+
return value
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
def _text(value: Any, path: str) -> str:
|
|
1522
|
+
if not isinstance(value, str) or not value.strip():
|
|
1523
|
+
raise ValidationError(f"{path} must be a non-empty string")
|
|
1524
|
+
return value
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
def _require_nonempty(value: str, *, label: str) -> None:
|
|
1528
|
+
if not isinstance(value, str) or not value.strip():
|
|
1529
|
+
raise ValidationError(f"{label} must be a non-empty string")
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
def _require_seed(value: int) -> None:
|
|
1533
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
1534
|
+
raise ValidationError("seed must be a non-negative integer")
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
def _require_sha256(value: str, *, label: str) -> None:
|
|
1538
|
+
if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
|
|
1539
|
+
raise ValidationError(f"{label} must be a lowercase SHA-256")
|
|
1540
|
+
|
|
1541
|
+
|
|
1542
|
+
def _safe_blocker_message(error: Exception, *, prepared_root: Path) -> str:
|
|
1543
|
+
return f"{type(error).__name__}: " + _safe_operator_message(
|
|
1544
|
+
str(error),
|
|
1545
|
+
prepared_root=prepared_root,
|
|
1546
|
+
)
|
|
1547
|
+
|
|
1548
|
+
|
|
1549
|
+
def _safe_operator_message(value: str, *, prepared_root: Path) -> str:
|
|
1550
|
+
message = value[:2048].replace(str(prepared_root), "<prepared_data_root>")
|
|
1551
|
+
return _redact_secret_text(message)
|
|
1552
|
+
|
|
1553
|
+
|
|
1554
|
+
def _redact_secret_text(value: str) -> str:
|
|
1555
|
+
redacted = value
|
|
1556
|
+
for pattern in _SECRET_TEXT_PATTERNS:
|
|
1557
|
+
redacted = pattern.sub("<redacted>", redacted)
|
|
1558
|
+
return redacted
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
__all__ = [
|
|
1562
|
+
"CONFIG_PATH_ENV",
|
|
1563
|
+
"CONFIG_SHA256_ENV",
|
|
1564
|
+
"LockedBuilderSpec",
|
|
1565
|
+
"LockedComponentBuilder",
|
|
1566
|
+
"StaticArmBuildRequest",
|
|
1567
|
+
"StaticBackendBuildRequest",
|
|
1568
|
+
"StaticPredictorBuildRequest",
|
|
1569
|
+
"StaticProductionExperimentDriver",
|
|
1570
|
+
"StaticRuntimeComponents",
|
|
1571
|
+
"factory",
|
|
1572
|
+
"static_predictor_product_sha256",
|
|
1573
|
+
"static_runtime_product_sha256",
|
|
1574
|
+
]
|