agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Materialize compact study blueprints against one content-bound arm catalog."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from agentinputlab.contracts.errors import ArtifactError, ValidationError
|
|
13
|
+
from agentinputlab.contracts.serialization import content_hash
|
|
14
|
+
|
|
15
|
+
from .schema import ExperimentManifest
|
|
16
|
+
|
|
17
|
+
_ENVIRONMENT_NAME = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
|
18
|
+
_GIT_OBJECT_ID_LENGTHS = frozenset({40, 64})
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def load_experiment_blueprint(
|
|
22
|
+
blueprint_path: str | Path,
|
|
23
|
+
*,
|
|
24
|
+
arm_catalog_path: str | Path,
|
|
25
|
+
environment: Mapping[str, str] | None = None,
|
|
26
|
+
) -> ExperimentManifest:
|
|
27
|
+
blueprint = _read_object(blueprint_path, label="experiment blueprint")
|
|
28
|
+
catalog = _read_object(arm_catalog_path, label="experiment arm catalog")
|
|
29
|
+
if set(blueprint) != {"schema_version", "arm_set", "manifest"}:
|
|
30
|
+
raise ValidationError("experiment blueprint fields are invalid")
|
|
31
|
+
if blueprint["schema_version"] != 1:
|
|
32
|
+
raise ValidationError("unsupported experiment blueprint schema")
|
|
33
|
+
arm_set_id = blueprint["arm_set"]
|
|
34
|
+
if not isinstance(arm_set_id, str) or not arm_set_id.strip():
|
|
35
|
+
raise ValidationError("experiment blueprint arm_set must be non-empty")
|
|
36
|
+
if set(catalog) != {"schema_version", "arm_sets"} or catalog["schema_version"] != 1:
|
|
37
|
+
raise ValidationError("unsupported experiment arm catalog schema")
|
|
38
|
+
arm_sets = catalog["arm_sets"]
|
|
39
|
+
if not isinstance(arm_sets, Mapping):
|
|
40
|
+
raise ValidationError("experiment arm_sets must be an object")
|
|
41
|
+
try:
|
|
42
|
+
arms = arm_sets[arm_set_id]
|
|
43
|
+
except KeyError as exc:
|
|
44
|
+
raise ValidationError(f"unknown experiment arm set: {arm_set_id}") from exc
|
|
45
|
+
manifest = blueprint["manifest"]
|
|
46
|
+
if not isinstance(manifest, Mapping):
|
|
47
|
+
raise ValidationError("experiment blueprint manifest must be an object")
|
|
48
|
+
if "arms" in manifest:
|
|
49
|
+
raise ValidationError("experiment blueprint cannot override catalog arms")
|
|
50
|
+
metadata = manifest.get("metadata", {})
|
|
51
|
+
if not isinstance(metadata, Mapping):
|
|
52
|
+
raise ValidationError("experiment blueprint metadata must be an object")
|
|
53
|
+
metadata = dict(metadata)
|
|
54
|
+
manifest_fields = dict(manifest)
|
|
55
|
+
benchmarks = manifest_fields.get("benchmarks")
|
|
56
|
+
env = os.environ if environment is None else environment
|
|
57
|
+
revision_environment = metadata.get("source_revision_placeholder_must_match_env")
|
|
58
|
+
if revision_environment is not None:
|
|
59
|
+
if not isinstance(revision_environment, str) or not revision_environment:
|
|
60
|
+
raise ValidationError(
|
|
61
|
+
"source_revision_placeholder_must_match_env must be a non-empty "
|
|
62
|
+
"environment-variable name"
|
|
63
|
+
)
|
|
64
|
+
revision = env.get(revision_environment)
|
|
65
|
+
if revision:
|
|
66
|
+
if not isinstance(benchmarks, list):
|
|
67
|
+
raise ValidationError("experiment blueprint benchmarks must be an array")
|
|
68
|
+
bound_benchmarks: list[Any] = []
|
|
69
|
+
for target in benchmarks:
|
|
70
|
+
if not isinstance(target, Mapping):
|
|
71
|
+
raise ValidationError("experiment blueprint benchmark targets must be objects")
|
|
72
|
+
bound_target = dict(target)
|
|
73
|
+
if bound_target.get("external") is True:
|
|
74
|
+
bound_target["source_revision"] = revision
|
|
75
|
+
bound_benchmarks.append(bound_target)
|
|
76
|
+
manifest_fields["benchmarks"] = bound_benchmarks
|
|
77
|
+
|
|
78
|
+
revision_bindings = metadata.get("checkout_expected_revision_from_env")
|
|
79
|
+
if revision_bindings is not None:
|
|
80
|
+
if not isinstance(revision_bindings, Mapping) or not revision_bindings:
|
|
81
|
+
raise ValidationError("checkout_expected_revision_from_env must be a non-empty object")
|
|
82
|
+
literal_expectations = metadata.get("checkout_expected_revisions", {})
|
|
83
|
+
if not isinstance(literal_expectations, Mapping):
|
|
84
|
+
raise ValidationError("checkout_expected_revisions must be an object")
|
|
85
|
+
materialized_expectations = dict(literal_expectations)
|
|
86
|
+
for checkout_environment, source_revision_environment in revision_bindings.items():
|
|
87
|
+
if (
|
|
88
|
+
not isinstance(checkout_environment, str)
|
|
89
|
+
or _ENVIRONMENT_NAME.fullmatch(checkout_environment) is None
|
|
90
|
+
or not isinstance(source_revision_environment, str)
|
|
91
|
+
or _ENVIRONMENT_NAME.fullmatch(source_revision_environment) is None
|
|
92
|
+
):
|
|
93
|
+
raise ValidationError(
|
|
94
|
+
"checkout_expected_revision_from_env must map environment-variable "
|
|
95
|
+
"names to environment-variable names"
|
|
96
|
+
)
|
|
97
|
+
revision = env.get(source_revision_environment)
|
|
98
|
+
if not revision:
|
|
99
|
+
continue
|
|
100
|
+
_require_git_object_id(
|
|
101
|
+
revision,
|
|
102
|
+
label=f"environment {source_revision_environment}",
|
|
103
|
+
)
|
|
104
|
+
existing = materialized_expectations.get(checkout_environment)
|
|
105
|
+
if existing is not None and existing != revision:
|
|
106
|
+
raise ValidationError(
|
|
107
|
+
f"checkout revision for {checkout_environment} conflicts with "
|
|
108
|
+
"checkout_expected_revisions"
|
|
109
|
+
)
|
|
110
|
+
materialized_expectations[checkout_environment] = revision
|
|
111
|
+
if materialized_expectations:
|
|
112
|
+
metadata["checkout_expected_revisions"] = materialized_expectations
|
|
113
|
+
|
|
114
|
+
merged: dict[str, Any] = {
|
|
115
|
+
**manifest_fields,
|
|
116
|
+
"arms": arms,
|
|
117
|
+
"metadata": {
|
|
118
|
+
**metadata,
|
|
119
|
+
"blueprint_hash": content_hash(blueprint),
|
|
120
|
+
"arm_set_id": arm_set_id,
|
|
121
|
+
"arm_set_hash": content_hash({"arm_set_id": arm_set_id, "arms": arms}),
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
return ExperimentManifest.from_mapping(merged)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _require_git_object_id(value: str, *, label: str) -> None:
|
|
128
|
+
if (
|
|
129
|
+
not isinstance(value, str)
|
|
130
|
+
or len(value) not in _GIT_OBJECT_ID_LENGTHS
|
|
131
|
+
or any(character not in "0123456789abcdef" for character in value)
|
|
132
|
+
):
|
|
133
|
+
raise ValidationError(f"{label} must be a full lowercase Git object ID")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _read_object(path: str | Path, *, label: str) -> Mapping[str, Any]:
|
|
137
|
+
source = Path(path).expanduser().resolve()
|
|
138
|
+
try:
|
|
139
|
+
value = json.loads(source.read_text(encoding="utf-8"))
|
|
140
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
141
|
+
raise ArtifactError(f"{label} is not valid UTF-8 JSON: {source}") from exc
|
|
142
|
+
if not isinstance(value, Mapping):
|
|
143
|
+
raise ValidationError(f"{label} must be an object")
|
|
144
|
+
return value
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
__all__ = ["load_experiment_blueprint"]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Reviewed, checksum-lockable component builders for production experiments."""
|
|
2
|
+
|
|
3
|
+
from .executable_external_payload import (
|
|
4
|
+
EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
|
|
5
|
+
ExecutableReleaseArmPayloadBuilder,
|
|
6
|
+
executable_release_arm_payload_builder_factory,
|
|
7
|
+
)
|
|
8
|
+
from .external_process_arena import (
|
|
9
|
+
ATOMIC_PROCESS_BRIDGE_PROTOCOL,
|
|
10
|
+
ATOMIC_PROCESS_REQUEST_PROTOCOL,
|
|
11
|
+
ATOMIC_PROCESS_RESPONSE_PROTOCOL,
|
|
12
|
+
CHECKPOINT_INVENTORY_PROTOCOL,
|
|
13
|
+
RELEASE_PAYLOAD_PROTOCOL,
|
|
14
|
+
AtomicProcessBridgeBuilder,
|
|
15
|
+
FrozenCheckpointInventoryBuilder,
|
|
16
|
+
ReleaseArmPayloadBuilder,
|
|
17
|
+
atomic_process_bridge_builder_factory,
|
|
18
|
+
frozen_checkpoint_inventory_builder_factory,
|
|
19
|
+
release_arm_payload_builder_factory,
|
|
20
|
+
)
|
|
21
|
+
from .registry_intervention import (
|
|
22
|
+
REGISTRY_INTERVENTION_BUILDER_PROTOCOL,
|
|
23
|
+
RegistryInterventionBuilder,
|
|
24
|
+
registry_intervention_builder_factory,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"ATOMIC_PROCESS_BRIDGE_PROTOCOL",
|
|
29
|
+
"ATOMIC_PROCESS_REQUEST_PROTOCOL",
|
|
30
|
+
"ATOMIC_PROCESS_RESPONSE_PROTOCOL",
|
|
31
|
+
"CHECKPOINT_INVENTORY_PROTOCOL",
|
|
32
|
+
"EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL",
|
|
33
|
+
"REGISTRY_INTERVENTION_BUILDER_PROTOCOL",
|
|
34
|
+
"RELEASE_PAYLOAD_PROTOCOL",
|
|
35
|
+
"AtomicProcessBridgeBuilder",
|
|
36
|
+
"ExecutableReleaseArmPayloadBuilder",
|
|
37
|
+
"FrozenCheckpointInventoryBuilder",
|
|
38
|
+
"RegistryInterventionBuilder",
|
|
39
|
+
"ReleaseArmPayloadBuilder",
|
|
40
|
+
"atomic_process_bridge_builder_factory",
|
|
41
|
+
"executable_release_arm_payload_builder_factory",
|
|
42
|
+
"frozen_checkpoint_inventory_builder_factory",
|
|
43
|
+
"registry_intervention_builder_factory",
|
|
44
|
+
"release_arm_payload_builder_factory",
|
|
45
|
+
]
|
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
"""Execute a registered intervention before entering an official external arena.
|
|
2
|
+
|
|
3
|
+
The external-arena transport deliberately keeps the protected benchmark
|
|
4
|
+
runtime in a separate process. A release name alone is not an intervention,
|
|
5
|
+
however: the selected algorithm must first run against the exact frozen,
|
|
6
|
+
optimizer-visible state. This builder performs that missing production step.
|
|
7
|
+
|
|
8
|
+
It reuses the same checksum-locked backend and registry-intervention builders
|
|
9
|
+
as the static production driver. The resulting payload contains the complete
|
|
10
|
+
``InputPatch``, its materialized post-intervention state, and measured
|
|
11
|
+
intervention usage. The official arena worker can therefore inject the
|
|
12
|
+
materialized state without reimplementing or guessing any algorithm logic.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from collections.abc import Mapping, Sequence
|
|
18
|
+
from types import MappingProxyType
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from agentinputlab.contracts.enums import SemanticFamily
|
|
22
|
+
from agentinputlab.contracts.errors import (
|
|
23
|
+
ArmExecutionError,
|
|
24
|
+
ArtifactError,
|
|
25
|
+
BudgetExceededError,
|
|
26
|
+
IntentViolationError,
|
|
27
|
+
ValidationError,
|
|
28
|
+
)
|
|
29
|
+
from agentinputlab.contracts.lifecycle import Intervention, InterventionContext
|
|
30
|
+
from agentinputlab.contracts.serialization import content_hash, freeze_mapping, to_jsonable
|
|
31
|
+
from agentinputlab.experiments.drivers.static_production import (
|
|
32
|
+
LockedBuilderSpec,
|
|
33
|
+
StaticArmBuildRequest,
|
|
34
|
+
StaticBackendBuildRequest,
|
|
35
|
+
StaticRuntimeComponents,
|
|
36
|
+
static_runtime_product_sha256,
|
|
37
|
+
)
|
|
38
|
+
from agentinputlab.experiments.loader import resolve_driver_entrypoint
|
|
39
|
+
from agentinputlab.observability import RunMonitor
|
|
40
|
+
from agentinputlab.runner import InterventionRunner
|
|
41
|
+
from agentinputlab.selectors import NoOpIntervention
|
|
42
|
+
|
|
43
|
+
from .external_process_arena import (
|
|
44
|
+
ExternalArmPayload,
|
|
45
|
+
ExternalArmPayloadBuildRequest,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL = "agentinputlab.executable-release-payload.v1"
|
|
49
|
+
|
|
50
|
+
_SCHEMA_VERSION = 1
|
|
51
|
+
_FIELDS = frozenset(
|
|
52
|
+
{
|
|
53
|
+
"schema_version",
|
|
54
|
+
"protocol",
|
|
55
|
+
"allowed_release_ids",
|
|
56
|
+
"runtime_by_model_slot",
|
|
57
|
+
"intervention_builder",
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ExecutableReleaseArmPayloadBuilder:
|
|
63
|
+
"""Run one unchanged registered release and serialize its materialized patch."""
|
|
64
|
+
|
|
65
|
+
builder_kind = "arm_payload"
|
|
66
|
+
|
|
67
|
+
def fingerprint(self) -> Mapping[str, Any]:
|
|
68
|
+
return freeze_mapping(
|
|
69
|
+
{
|
|
70
|
+
"builder_id": EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
|
|
71
|
+
"schema_version": _SCHEMA_VERSION,
|
|
72
|
+
"runtime_source": "checksum_locked_backend_builder",
|
|
73
|
+
"intervention_source": "checksum_locked_intervention_builder",
|
|
74
|
+
"execution": "agentinputlab.InterventionRunner",
|
|
75
|
+
"payload": "patch_plus_materialized_visible_state_plus_usage",
|
|
76
|
+
"protected_runtime_access": False,
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def preflight(self, parameters: Mapping[str, Any]) -> Sequence[str]:
|
|
81
|
+
try:
|
|
82
|
+
parsed = _parse_parameters(parameters)
|
|
83
|
+
for model_slot, runtime_spec in parsed["runtime_by_model_slot"].items():
|
|
84
|
+
builder = _resolve_builder(runtime_spec, expected_kind="backend")
|
|
85
|
+
_require_clean_preflight(
|
|
86
|
+
builder,
|
|
87
|
+
runtime_spec.parameters,
|
|
88
|
+
label=f"runtime_by_model_slot.{model_slot}",
|
|
89
|
+
)
|
|
90
|
+
intervention_spec = parsed["intervention_builder"]
|
|
91
|
+
intervention_builder = _resolve_builder(
|
|
92
|
+
intervention_spec,
|
|
93
|
+
expected_kind="intervention",
|
|
94
|
+
)
|
|
95
|
+
_require_clean_preflight(
|
|
96
|
+
intervention_builder,
|
|
97
|
+
intervention_spec.parameters,
|
|
98
|
+
label="intervention_builder",
|
|
99
|
+
)
|
|
100
|
+
except (ArtifactError, ValidationError, TypeError, ValueError) as exc:
|
|
101
|
+
return (str(exc),)
|
|
102
|
+
return ()
|
|
103
|
+
|
|
104
|
+
def build(self, request: ExternalArmPayloadBuildRequest) -> ExternalArmPayload:
|
|
105
|
+
if not isinstance(request, ExternalArmPayloadBuildRequest):
|
|
106
|
+
raise TypeError(
|
|
107
|
+
"executable release payload builder requires ExternalArmPayloadBuildRequest"
|
|
108
|
+
)
|
|
109
|
+
parsed = _parse_parameters(request.builder_parameters)
|
|
110
|
+
if request.release_id not in parsed["allowed_release_ids"]:
|
|
111
|
+
raise ValidationError("release is outside the executable payload allowlist")
|
|
112
|
+
if request.arm_kind.value != "release":
|
|
113
|
+
raise ValidationError(
|
|
114
|
+
"executable release payload builder does not construct selector policy arms"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
state = request.checkpoint.visible_state
|
|
118
|
+
if request.release_id == "baseline.no_op":
|
|
119
|
+
if request.family is not SemanticFamily.NO_OP:
|
|
120
|
+
raise ValidationError("baseline.no_op requires the no_op family")
|
|
121
|
+
runtime = None
|
|
122
|
+
intervention: Intervention = NoOpIntervention()
|
|
123
|
+
runtime_product_sha256 = content_hash(
|
|
124
|
+
{
|
|
125
|
+
"schema_version": 1,
|
|
126
|
+
"runtime": "not_used_by_no_op_intervention",
|
|
127
|
+
"model_slot": request.model_slot,
|
|
128
|
+
}
|
|
129
|
+
)
|
|
130
|
+
context = InterventionContext(
|
|
131
|
+
seed=request.seed,
|
|
132
|
+
run_config={
|
|
133
|
+
"benchmark_id": request.checkpoint.benchmark_id,
|
|
134
|
+
"checkpoint_id": request.checkpoint.checkpoint_id,
|
|
135
|
+
"model_slot": request.model_slot,
|
|
136
|
+
},
|
|
137
|
+
monitor=RunMonitor(),
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
if request.family is SemanticFamily.NO_OP:
|
|
141
|
+
raise ValidationError("non-baseline release cannot use the no_op family")
|
|
142
|
+
runtime_spec = _runtime_spec(parsed, request.model_slot)
|
|
143
|
+
runtime_builder = _resolve_builder(runtime_spec, expected_kind="backend")
|
|
144
|
+
runtime = runtime_builder.build(
|
|
145
|
+
StaticBackendBuildRequest(
|
|
146
|
+
model_slot=request.model_slot,
|
|
147
|
+
seed=request.seed,
|
|
148
|
+
builder_parameters=runtime_spec.parameters,
|
|
149
|
+
)
|
|
150
|
+
)
|
|
151
|
+
if not isinstance(runtime, StaticRuntimeComponents):
|
|
152
|
+
raise ValidationError(
|
|
153
|
+
"external payload backend builder must return StaticRuntimeComponents"
|
|
154
|
+
)
|
|
155
|
+
runtime_product_sha256 = static_runtime_product_sha256(runtime)
|
|
156
|
+
if runtime_product_sha256 != runtime_spec.expected_product_sha256:
|
|
157
|
+
raise ArtifactError("external payload runtime product checksum mismatch")
|
|
158
|
+
_bind_benchmark_retriever(runtime, request.checkpoint.benchmark_id)
|
|
159
|
+
|
|
160
|
+
intervention_spec = parsed["intervention_builder"]
|
|
161
|
+
intervention_builder = _resolve_builder(
|
|
162
|
+
intervention_spec,
|
|
163
|
+
expected_kind="intervention",
|
|
164
|
+
)
|
|
165
|
+
intervention = intervention_builder.build(
|
|
166
|
+
StaticArmBuildRequest(
|
|
167
|
+
arm_id=request.arm_id,
|
|
168
|
+
release_id=request.release_id,
|
|
169
|
+
family=request.family,
|
|
170
|
+
profile_kind=request.profile_kind,
|
|
171
|
+
model_slot=request.model_slot,
|
|
172
|
+
seed=request.seed,
|
|
173
|
+
builder_parameters=intervention_spec.parameters,
|
|
174
|
+
arm_parameters=request.arm_parameters,
|
|
175
|
+
runtime=runtime,
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
if not isinstance(intervention, Intervention):
|
|
179
|
+
raise ValidationError(
|
|
180
|
+
"external payload intervention builder did not return an Intervention"
|
|
181
|
+
)
|
|
182
|
+
if (
|
|
183
|
+
intervention.release_id != request.release_id
|
|
184
|
+
or intervention.family is not request.family
|
|
185
|
+
):
|
|
186
|
+
raise ValidationError(
|
|
187
|
+
"external payload intervention identity differs from the requested arm"
|
|
188
|
+
)
|
|
189
|
+
context = InterventionContext(
|
|
190
|
+
backend=runtime.backend,
|
|
191
|
+
dense_embedder=runtime.dense_embedder,
|
|
192
|
+
retriever=runtime.retriever,
|
|
193
|
+
vector_searcher=runtime.vector_searcher,
|
|
194
|
+
tools=runtime.tools,
|
|
195
|
+
external_runtime=runtime.external_runtime,
|
|
196
|
+
artifact_manager=runtime.artifact_manager,
|
|
197
|
+
seed=request.seed,
|
|
198
|
+
run_config={
|
|
199
|
+
**dict(runtime.run_config),
|
|
200
|
+
"benchmark_id": request.checkpoint.benchmark_id,
|
|
201
|
+
"checkpoint_id": request.checkpoint.checkpoint_id,
|
|
202
|
+
"model_slot": request.model_slot,
|
|
203
|
+
"runtime_product_sha256": runtime_product_sha256,
|
|
204
|
+
},
|
|
205
|
+
intent_contract=runtime.intent_contract,
|
|
206
|
+
monitor=RunMonitor(),
|
|
207
|
+
pricing_catalog=runtime.pricing_catalog,
|
|
208
|
+
strict_intent=runtime.strict_intent,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
monitor = context.monitor
|
|
212
|
+
if not isinstance(monitor, RunMonitor): # pragma: no cover - construction invariant
|
|
213
|
+
raise TypeError("external payload context monitor must be a RunMonitor")
|
|
214
|
+
intervention_builder_sha256 = (
|
|
215
|
+
content_hash(parsed["intervention_builder"].binding_identity)
|
|
216
|
+
if request.release_id != "baseline.no_op"
|
|
217
|
+
else content_hash({"builder": "builtin.no_op"})
|
|
218
|
+
)
|
|
219
|
+
try:
|
|
220
|
+
record = InterventionRunner().run(intervention, state, context)
|
|
221
|
+
except (ArmExecutionError, BudgetExceededError, IntentViolationError):
|
|
222
|
+
snapshots = monitor.list_snapshots()
|
|
223
|
+
if len(snapshots) != 1 or snapshots[0].failure is None:
|
|
224
|
+
raise ValidationError(
|
|
225
|
+
"arm-local intervention failure has no unique monitored failure receipt"
|
|
226
|
+
) from None
|
|
227
|
+
snapshot = snapshots[0]
|
|
228
|
+
execution = {
|
|
229
|
+
"status": "failed",
|
|
230
|
+
"input_state_hash": state.state_hash,
|
|
231
|
+
"output_state_hash": state.state_hash,
|
|
232
|
+
"patch": None,
|
|
233
|
+
"patch_hash": None,
|
|
234
|
+
# The official arena may still execute the unchanged state to
|
|
235
|
+
# preserve paired runtime/scorer evidence. Downstream
|
|
236
|
+
# normalization assigns the preregistered algorithm-failure
|
|
237
|
+
# utility; it never mistakes this continuation for success.
|
|
238
|
+
"materialized_state": to_jsonable(state),
|
|
239
|
+
"runtime_product_sha256": runtime_product_sha256,
|
|
240
|
+
"intervention_builder_sha256": intervention_builder_sha256,
|
|
241
|
+
"usage": to_jsonable(snapshot.usage),
|
|
242
|
+
"duration_ms": snapshot.duration_ms,
|
|
243
|
+
"intent_check": None,
|
|
244
|
+
"failure": {
|
|
245
|
+
"scope": "algorithm",
|
|
246
|
+
**to_jsonable(snapshot.failure),
|
|
247
|
+
"monitor_run_id": snapshot.run_id,
|
|
248
|
+
},
|
|
249
|
+
}
|
|
250
|
+
else:
|
|
251
|
+
materialized = record.patch.apply(state)
|
|
252
|
+
if materialized.state_hash != record.output_state_hash:
|
|
253
|
+
raise ValidationError("materialized external payload state hash is inconsistent")
|
|
254
|
+
execution = {
|
|
255
|
+
"status": record.status.value,
|
|
256
|
+
"input_state_hash": record.input_state_hash,
|
|
257
|
+
"output_state_hash": record.output_state_hash,
|
|
258
|
+
"patch": record.patch.as_dict(),
|
|
259
|
+
"patch_hash": record.patch.patch_hash,
|
|
260
|
+
"materialized_state": to_jsonable(materialized),
|
|
261
|
+
"runtime_product_sha256": runtime_product_sha256,
|
|
262
|
+
"intervention_builder_sha256": intervention_builder_sha256,
|
|
263
|
+
"usage": to_jsonable(record.usage),
|
|
264
|
+
"duration_ms": record.duration_ms,
|
|
265
|
+
"intent_check": to_jsonable(record.intent_check),
|
|
266
|
+
"failure": None,
|
|
267
|
+
}
|
|
268
|
+
payload = {
|
|
269
|
+
"schema_version": _SCHEMA_VERSION,
|
|
270
|
+
"protocol": EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
|
|
271
|
+
"arm_id": request.arm_id,
|
|
272
|
+
"release_id": request.release_id,
|
|
273
|
+
"family": request.family.value,
|
|
274
|
+
"profile_kind": request.profile_kind.value,
|
|
275
|
+
"model_slot": request.model_slot,
|
|
276
|
+
"seed": request.seed,
|
|
277
|
+
"checkpoint_identity_hash": request.checkpoint.identity_hash,
|
|
278
|
+
"visible_state_hash": state.state_hash,
|
|
279
|
+
"pre_action": request.pre_action,
|
|
280
|
+
"arm_parameters": request.arm_parameters,
|
|
281
|
+
"intervention_execution": execution,
|
|
282
|
+
}
|
|
283
|
+
return ExternalArmPayload(
|
|
284
|
+
arm_id=request.arm_id,
|
|
285
|
+
release_id=request.release_id,
|
|
286
|
+
family=request.family,
|
|
287
|
+
profile_kind=request.profile_kind,
|
|
288
|
+
payload=payload,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def executable_release_arm_payload_builder_factory() -> ExecutableReleaseArmPayloadBuilder:
|
|
293
|
+
"""Return the zero-argument production payload builder."""
|
|
294
|
+
|
|
295
|
+
return ExecutableReleaseArmPayloadBuilder()
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _bind_benchmark_retriever(
|
|
299
|
+
runtime: StaticRuntimeComponents,
|
|
300
|
+
benchmark_id: str,
|
|
301
|
+
) -> None:
|
|
302
|
+
"""Bind the production retriever router to the audited checkpoint cell.
|
|
303
|
+
|
|
304
|
+
Static predictors perform this binding themselves. External-arena payloads
|
|
305
|
+
execute the intervention before a downstream predictor exists, so methods
|
|
306
|
+
such as Self-Ask and IRCoT need the same benchmark-validated route here.
|
|
307
|
+
The router fingerprint is intentionally stable across binding.
|
|
308
|
+
"""
|
|
309
|
+
|
|
310
|
+
from agentinputlab.experiments.production_builders import ProductionRetrieverRouter
|
|
311
|
+
|
|
312
|
+
retriever = runtime.retriever
|
|
313
|
+
if isinstance(retriever, ProductionRetrieverRouter):
|
|
314
|
+
retriever.bind(benchmark_id)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _parse_parameters(parameters: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
318
|
+
if not isinstance(parameters, Mapping) or set(parameters) != _FIELDS:
|
|
319
|
+
raise ValidationError("executable release payload parameters are invalid")
|
|
320
|
+
if (
|
|
321
|
+
parameters["schema_version"] != _SCHEMA_VERSION
|
|
322
|
+
or parameters["protocol"] != EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL
|
|
323
|
+
):
|
|
324
|
+
raise ValidationError("unsupported executable release payload protocol")
|
|
325
|
+
raw_ids = parameters["allowed_release_ids"]
|
|
326
|
+
if (
|
|
327
|
+
isinstance(raw_ids, (str, bytes))
|
|
328
|
+
or not isinstance(raw_ids, Sequence)
|
|
329
|
+
or not raw_ids
|
|
330
|
+
or len(raw_ids) != len(set(raw_ids))
|
|
331
|
+
or any(not isinstance(item, str) or not item.strip() for item in raw_ids)
|
|
332
|
+
):
|
|
333
|
+
raise ValidationError(
|
|
334
|
+
"executable payload allowed_release_ids must be unique non-empty strings"
|
|
335
|
+
)
|
|
336
|
+
raw_runtimes = parameters["runtime_by_model_slot"]
|
|
337
|
+
if not isinstance(raw_runtimes, Mapping) or not raw_runtimes:
|
|
338
|
+
raise ValidationError("runtime_by_model_slot must be a non-empty object")
|
|
339
|
+
runtimes: dict[str, LockedBuilderSpec] = {}
|
|
340
|
+
for model_slot, value in raw_runtimes.items():
|
|
341
|
+
if not isinstance(model_slot, str) or not model_slot.strip():
|
|
342
|
+
raise ValidationError("runtime_by_model_slot keys must be non-empty strings")
|
|
343
|
+
spec = LockedBuilderSpec.from_mapping(
|
|
344
|
+
value,
|
|
345
|
+
path=f"runtime_by_model_slot.{model_slot}",
|
|
346
|
+
)
|
|
347
|
+
if spec.expected_product_sha256 is None:
|
|
348
|
+
raise ValidationError(
|
|
349
|
+
f"runtime_by_model_slot.{model_slot} requires expected_product_sha256"
|
|
350
|
+
)
|
|
351
|
+
runtimes[model_slot] = spec
|
|
352
|
+
intervention = LockedBuilderSpec.from_mapping(
|
|
353
|
+
parameters["intervention_builder"],
|
|
354
|
+
path="intervention_builder",
|
|
355
|
+
)
|
|
356
|
+
if intervention.expected_product_sha256 is not None:
|
|
357
|
+
raise ValidationError(
|
|
358
|
+
"executable payload intervention_builder must not declare a product checksum"
|
|
359
|
+
)
|
|
360
|
+
return MappingProxyType(
|
|
361
|
+
{
|
|
362
|
+
"schema_version": _SCHEMA_VERSION,
|
|
363
|
+
"protocol": EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL,
|
|
364
|
+
"allowed_release_ids": tuple(raw_ids),
|
|
365
|
+
"runtime_by_model_slot": MappingProxyType(runtimes),
|
|
366
|
+
"intervention_builder": intervention,
|
|
367
|
+
}
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _runtime_spec(
|
|
372
|
+
parsed: Mapping[str, Any],
|
|
373
|
+
model_slot: str,
|
|
374
|
+
) -> LockedBuilderSpec:
|
|
375
|
+
try:
|
|
376
|
+
return parsed["runtime_by_model_slot"][model_slot]
|
|
377
|
+
except KeyError as exc:
|
|
378
|
+
raise ArtifactError(
|
|
379
|
+
f"executable payload has no runtime binding for model slot {model_slot!r}"
|
|
380
|
+
) from exc
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _resolve_builder(spec: LockedBuilderSpec, *, expected_kind: str) -> Any:
|
|
384
|
+
factory = resolve_driver_entrypoint(spec.entrypoint)
|
|
385
|
+
if not callable(factory):
|
|
386
|
+
raise ArtifactError("executable payload builder entrypoint is not callable")
|
|
387
|
+
try:
|
|
388
|
+
builder = factory()
|
|
389
|
+
except TypeError as exc:
|
|
390
|
+
raise ArtifactError(
|
|
391
|
+
"executable payload builder entrypoint must be a zero-argument factory"
|
|
392
|
+
) from exc
|
|
393
|
+
if getattr(builder, "builder_kind", None) != expected_kind:
|
|
394
|
+
raise ArtifactError(f"executable payload component kind must be {expected_kind!r}")
|
|
395
|
+
fingerprint = builder.fingerprint()
|
|
396
|
+
if not isinstance(fingerprint, Mapping) or not fingerprint:
|
|
397
|
+
raise ArtifactError("executable payload component fingerprint is invalid")
|
|
398
|
+
if content_hash(fingerprint) != spec.fingerprint_sha256:
|
|
399
|
+
raise ArtifactError("executable payload component fingerprint checksum mismatch")
|
|
400
|
+
return builder
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _require_clean_preflight(
|
|
404
|
+
builder: Any,
|
|
405
|
+
parameters: Mapping[str, Any],
|
|
406
|
+
*,
|
|
407
|
+
label: str,
|
|
408
|
+
) -> None:
|
|
409
|
+
blockers = builder.preflight(parameters)
|
|
410
|
+
if isinstance(blockers, (str, bytes)) or not isinstance(blockers, Sequence):
|
|
411
|
+
raise ValidationError(f"{label} preflight must return a sequence")
|
|
412
|
+
messages = tuple(blockers)
|
|
413
|
+
if any(not isinstance(message, str) or not message.strip() for message in messages):
|
|
414
|
+
raise ValidationError(f"{label} preflight returned an invalid blocker")
|
|
415
|
+
if messages:
|
|
416
|
+
raise ArtifactError(f"{label}: " + "; ".join(messages))
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
__all__ = [
|
|
420
|
+
"EXECUTABLE_RELEASE_PAYLOAD_PROTOCOL",
|
|
421
|
+
"ExecutableReleaseArmPayloadBuilder",
|
|
422
|
+
"executable_release_arm_payload_builder_factory",
|
|
423
|
+
]
|