agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,605 @@
|
|
|
1
|
+
"""Fail-closed implementation identities for upstream-vs-library parity runs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import TYPE_CHECKING, Any
|
|
8
|
+
|
|
9
|
+
from agentinputlab.contracts.errors import ArtifactError, ValidationError
|
|
10
|
+
from agentinputlab.contracts.serialization import content_hash, freeze_mapping, to_jsonable
|
|
11
|
+
|
|
12
|
+
from .bindings import FrozenBindingReceipt
|
|
13
|
+
from .schema import ExperimentManifest, ExperimentStage, require_sha256
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from .executor import ExperimentCellResult
|
|
17
|
+
from .plan import ExperimentCell
|
|
18
|
+
|
|
19
|
+
_UPSTREAM_ARM = "upstream_reference"
|
|
20
|
+
_CANDIDATE_ARM = "agentinputlab_candidate"
|
|
21
|
+
_IMPLEMENTATION_KINDS = {
|
|
22
|
+
_UPSTREAM_ARM: "upstream_reference",
|
|
23
|
+
_CANDIDATE_ARM: "agentinputlab_candidate",
|
|
24
|
+
}
|
|
25
|
+
_GIT_OBJECT_ID_LENGTHS = frozenset({40, 64})
|
|
26
|
+
HOST_EXECUTION_RECEIPT_PROTOCOL = "agentinputlab.host-process-execution-receipt.v1"
|
|
27
|
+
_HOST_RECEIPT_FIELDS = {
|
|
28
|
+
"schema_version",
|
|
29
|
+
"protocol",
|
|
30
|
+
"arm_id",
|
|
31
|
+
"manifest_hash",
|
|
32
|
+
"cell_id",
|
|
33
|
+
"request_hash",
|
|
34
|
+
"config_sha256",
|
|
35
|
+
"checkout_environment",
|
|
36
|
+
"checkout_binding_hash",
|
|
37
|
+
"checkout",
|
|
38
|
+
"adapter_checkout_environment",
|
|
39
|
+
"adapter_checkout_binding_hash",
|
|
40
|
+
"adapter_checkout",
|
|
41
|
+
"command",
|
|
42
|
+
"process",
|
|
43
|
+
"receipt_hash",
|
|
44
|
+
}
|
|
45
|
+
_HOST_CHECKOUT_FIELDS = {
|
|
46
|
+
"resolved_path_sha256",
|
|
47
|
+
"git_head",
|
|
48
|
+
"git_tree",
|
|
49
|
+
"git_clean_before",
|
|
50
|
+
"git_clean_after",
|
|
51
|
+
}
|
|
52
|
+
_HOST_PROCESS_FIELDS = {
|
|
53
|
+
"resource_receipt_sha256",
|
|
54
|
+
"stdout_sha256",
|
|
55
|
+
"stderr_sha256",
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True, slots=True)
|
|
60
|
+
class ParityImplementationIdentity:
|
|
61
|
+
"""Execution identity for exactly one parity arm."""
|
|
62
|
+
|
|
63
|
+
arm_id: str
|
|
64
|
+
implementation_kind: str
|
|
65
|
+
implementation_id: str
|
|
66
|
+
runtime_entrypoint: str
|
|
67
|
+
source_revision: str
|
|
68
|
+
source_tree: str
|
|
69
|
+
release_id: str
|
|
70
|
+
checkout_binding_hash: str | None
|
|
71
|
+
schema_version: int = 1
|
|
72
|
+
|
|
73
|
+
def __post_init__(self) -> None:
|
|
74
|
+
if self.schema_version != 1:
|
|
75
|
+
raise ValidationError("unsupported parity implementation identity schema")
|
|
76
|
+
expected_kind = _IMPLEMENTATION_KINDS.get(self.arm_id)
|
|
77
|
+
if expected_kind is None or self.implementation_kind != expected_kind:
|
|
78
|
+
raise ValidationError("parity arm and implementation kind differ")
|
|
79
|
+
for label in (
|
|
80
|
+
"implementation_id",
|
|
81
|
+
"runtime_entrypoint",
|
|
82
|
+
"source_revision",
|
|
83
|
+
"source_tree",
|
|
84
|
+
"release_id",
|
|
85
|
+
):
|
|
86
|
+
value = getattr(self, label)
|
|
87
|
+
if not isinstance(value, str) or not value.strip():
|
|
88
|
+
raise ValidationError(f"parity implementation {label} must be non-empty")
|
|
89
|
+
if len(self.source_tree) not in _GIT_OBJECT_ID_LENGTHS or any(
|
|
90
|
+
character not in "0123456789abcdef" for character in self.source_tree
|
|
91
|
+
):
|
|
92
|
+
raise ValidationError(
|
|
93
|
+
"parity implementation source_tree must be a full lowercase object identity"
|
|
94
|
+
)
|
|
95
|
+
if self.checkout_binding_hash is not None:
|
|
96
|
+
require_sha256(
|
|
97
|
+
self.checkout_binding_hash,
|
|
98
|
+
label="parity checkout_binding_hash",
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def identity_hash(self) -> str:
|
|
103
|
+
return content_hash(self)
|
|
104
|
+
|
|
105
|
+
def to_dict(self) -> dict[str, Any]:
|
|
106
|
+
payload = to_jsonable(self)
|
|
107
|
+
payload["identity_hash"] = self.identity_hash
|
|
108
|
+
return payload
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass(frozen=True, slots=True)
|
|
112
|
+
class ParityImplementationContract:
|
|
113
|
+
"""Exact two-implementation comparison contract bound to one manifest."""
|
|
114
|
+
|
|
115
|
+
method_id: str
|
|
116
|
+
manifest_hash: str
|
|
117
|
+
binding_receipt_hash: str
|
|
118
|
+
implementations: tuple[ParityImplementationIdentity, ...]
|
|
119
|
+
schema_version: int = 1
|
|
120
|
+
|
|
121
|
+
def __post_init__(self) -> None:
|
|
122
|
+
if self.schema_version != 1:
|
|
123
|
+
raise ValidationError("unsupported parity implementation contract schema")
|
|
124
|
+
if not isinstance(self.method_id, str) or not self.method_id.strip():
|
|
125
|
+
raise ValidationError("parity contract method_id must be non-empty")
|
|
126
|
+
require_sha256(self.manifest_hash, label="parity contract manifest_hash")
|
|
127
|
+
require_sha256(
|
|
128
|
+
self.binding_receipt_hash,
|
|
129
|
+
label="parity contract binding_receipt_hash",
|
|
130
|
+
)
|
|
131
|
+
implementations = tuple(self.implementations)
|
|
132
|
+
if not all(isinstance(item, ParityImplementationIdentity) for item in implementations):
|
|
133
|
+
raise ValidationError("parity contract implementation identities must be typed")
|
|
134
|
+
by_arm = {item.arm_id: item for item in implementations}
|
|
135
|
+
if set(by_arm) != set(_IMPLEMENTATION_KINDS) or len(by_arm) != len(implementations):
|
|
136
|
+
raise ValidationError("parity contract requires exact upstream and candidate arms")
|
|
137
|
+
upstream = by_arm[_UPSTREAM_ARM]
|
|
138
|
+
candidate = by_arm[_CANDIDATE_ARM]
|
|
139
|
+
if upstream.implementation_id == candidate.implementation_id:
|
|
140
|
+
raise ValidationError("parity implementations require distinct implementation_id")
|
|
141
|
+
if upstream.runtime_entrypoint == candidate.runtime_entrypoint:
|
|
142
|
+
raise ValidationError("parity implementations require distinct runtime entrypoints")
|
|
143
|
+
if (
|
|
144
|
+
upstream.checkout_binding_hash is None
|
|
145
|
+
or candidate.checkout_binding_hash is None
|
|
146
|
+
or upstream.checkout_binding_hash == candidate.checkout_binding_hash
|
|
147
|
+
):
|
|
148
|
+
raise ValidationError("parity implementations require distinct host-attested checkouts")
|
|
149
|
+
if upstream.identity_hash == candidate.identity_hash:
|
|
150
|
+
raise ValidationError("parity implementations resolved to the same identity")
|
|
151
|
+
canonical = tuple(sorted(implementations, key=lambda item: item.arm_id))
|
|
152
|
+
object.__setattr__(self, "implementations", canonical)
|
|
153
|
+
|
|
154
|
+
@property
|
|
155
|
+
def contract_hash(self) -> str:
|
|
156
|
+
return content_hash(
|
|
157
|
+
{
|
|
158
|
+
"schema_version": self.schema_version,
|
|
159
|
+
"method_id": self.method_id,
|
|
160
|
+
"manifest_hash": self.manifest_hash,
|
|
161
|
+
"binding_receipt_hash": self.binding_receipt_hash,
|
|
162
|
+
"implementations": [
|
|
163
|
+
{
|
|
164
|
+
"schema_version": item.schema_version,
|
|
165
|
+
"arm_id": item.arm_id,
|
|
166
|
+
"implementation_kind": item.implementation_kind,
|
|
167
|
+
"implementation_id": item.implementation_id,
|
|
168
|
+
"runtime_entrypoint": item.runtime_entrypoint,
|
|
169
|
+
"source_revision": item.source_revision,
|
|
170
|
+
"source_tree": item.source_tree,
|
|
171
|
+
"release_id": item.release_id,
|
|
172
|
+
"checkout_binding_hash": item.checkout_binding_hash,
|
|
173
|
+
}
|
|
174
|
+
for item in self.implementations
|
|
175
|
+
],
|
|
176
|
+
}
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
def implementation(self, arm_id: str) -> ParityImplementationIdentity:
|
|
180
|
+
for item in self.implementations:
|
|
181
|
+
if item.arm_id == arm_id:
|
|
182
|
+
return item
|
|
183
|
+
raise ArtifactError(f"parity contract has no implementation for arm {arm_id!r}")
|
|
184
|
+
|
|
185
|
+
def to_dict(self) -> dict[str, Any]:
|
|
186
|
+
return {
|
|
187
|
+
"schema_version": self.schema_version,
|
|
188
|
+
"method_id": self.method_id,
|
|
189
|
+
"manifest_hash": self.manifest_hash,
|
|
190
|
+
"binding_receipt_hash": self.binding_receipt_hash,
|
|
191
|
+
"implementations": {item.arm_id: item.to_dict() for item in self.implementations},
|
|
192
|
+
"contract_hash": self.contract_hash,
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def build_parity_implementation_contract(
|
|
197
|
+
manifest: ExperimentManifest,
|
|
198
|
+
binding_receipt: FrozenBindingReceipt,
|
|
199
|
+
operator_fingerprint: Mapping[str, Any],
|
|
200
|
+
) -> ParityImplementationContract:
|
|
201
|
+
"""Validate driver declarations and bind them to host-attested assets."""
|
|
202
|
+
|
|
203
|
+
if manifest.stage is not ExperimentStage.PARITY:
|
|
204
|
+
raise ValidationError("parity implementation contract requires a parity manifest")
|
|
205
|
+
if binding_receipt.manifest_hash != manifest.manifest_hash:
|
|
206
|
+
raise ValidationError("parity binding receipt belongs to another manifest")
|
|
207
|
+
if {arm.arm_id for arm in manifest.arms} != set(_IMPLEMENTATION_KINDS):
|
|
208
|
+
raise ValidationError("parity manifest requires exact upstream and candidate arms")
|
|
209
|
+
|
|
210
|
+
method_id = _metadata_text(manifest, "method_id")
|
|
211
|
+
candidate_release = _metadata_text(manifest, "candidate_release_id")
|
|
212
|
+
upstream_revision = _metadata_text(manifest, "upstream_commit")
|
|
213
|
+
upstream_checkout = binding_receipt.binding(
|
|
214
|
+
"AILAB_UPSTREAM_CHECKOUT",
|
|
215
|
+
kind="directory_env",
|
|
216
|
+
)
|
|
217
|
+
candidate_checkout = binding_receipt.binding(
|
|
218
|
+
"AILAB_CANDIDATE_CHECKOUT",
|
|
219
|
+
kind="directory_env",
|
|
220
|
+
)
|
|
221
|
+
git_head = upstream_checkout.identity.get("git_head")
|
|
222
|
+
git_tree = upstream_checkout.identity.get("git_tree")
|
|
223
|
+
if (
|
|
224
|
+
upstream_checkout.identity.get("identity_mode") != "clean_git_checkout"
|
|
225
|
+
or git_head != upstream_revision
|
|
226
|
+
or not isinstance(git_tree, str)
|
|
227
|
+
):
|
|
228
|
+
raise ValidationError("parity upstream identity requires the pinned clean checkout receipt")
|
|
229
|
+
candidate_head = candidate_checkout.identity.get("git_head")
|
|
230
|
+
candidate_tree = candidate_checkout.identity.get("git_tree")
|
|
231
|
+
if (
|
|
232
|
+
candidate_checkout.identity.get("identity_mode") != "clean_git_checkout"
|
|
233
|
+
or not isinstance(candidate_head, str)
|
|
234
|
+
or not isinstance(candidate_tree, str)
|
|
235
|
+
):
|
|
236
|
+
raise ValidationError("parity candidate identity requires a clean checkout receipt")
|
|
237
|
+
|
|
238
|
+
raw_declarations = operator_fingerprint.get("parity_implementations")
|
|
239
|
+
if not isinstance(raw_declarations, Mapping) or set(raw_declarations) != set(
|
|
240
|
+
_IMPLEMENTATION_KINDS
|
|
241
|
+
):
|
|
242
|
+
raise ValidationError(
|
|
243
|
+
"driver fingerprint requires exact parity_implementations declarations"
|
|
244
|
+
)
|
|
245
|
+
upstream = _implementation_from_declaration(
|
|
246
|
+
_UPSTREAM_ARM,
|
|
247
|
+
raw_declarations[_UPSTREAM_ARM],
|
|
248
|
+
checkout_binding_hash=upstream_checkout.binding_hash,
|
|
249
|
+
)
|
|
250
|
+
candidate = _implementation_from_declaration(
|
|
251
|
+
_CANDIDATE_ARM,
|
|
252
|
+
raw_declarations[_CANDIDATE_ARM],
|
|
253
|
+
checkout_binding_hash=candidate_checkout.binding_hash,
|
|
254
|
+
)
|
|
255
|
+
if upstream.source_revision != upstream_revision or upstream.source_tree != git_tree:
|
|
256
|
+
raise ValidationError("upstream implementation declaration differs from checkout HEAD/tree")
|
|
257
|
+
if upstream.release_id != f"upstream:{method_id}":
|
|
258
|
+
raise ValidationError("upstream implementation release_id is not method-specific")
|
|
259
|
+
if candidate.release_id != candidate_release:
|
|
260
|
+
raise ValidationError("candidate implementation declaration differs from manifest release")
|
|
261
|
+
if candidate.source_revision != candidate_head or candidate.source_tree != candidate_tree:
|
|
262
|
+
raise ValidationError(
|
|
263
|
+
"candidate implementation declaration differs from checkout HEAD/tree"
|
|
264
|
+
)
|
|
265
|
+
return ParityImplementationContract(
|
|
266
|
+
method_id=method_id,
|
|
267
|
+
manifest_hash=manifest.manifest_hash,
|
|
268
|
+
binding_receipt_hash=binding_receipt.receipt_hash,
|
|
269
|
+
implementations=(upstream, candidate),
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def validate_parity_cell_result(
|
|
274
|
+
cell: ExperimentCell,
|
|
275
|
+
result: ExperimentCellResult,
|
|
276
|
+
*,
|
|
277
|
+
binding_receipt: FrozenBindingReceipt,
|
|
278
|
+
contract: ParityImplementationContract,
|
|
279
|
+
operator_fingerprint: Mapping[str, Any] | None = None,
|
|
280
|
+
) -> None:
|
|
281
|
+
"""Verify per-arm runtime identities before the executor can persist a result."""
|
|
282
|
+
|
|
283
|
+
if result.status == "failed":
|
|
284
|
+
return
|
|
285
|
+
if set(cell.arm_ids) != set(_IMPLEMENTATION_KINDS):
|
|
286
|
+
raise ValidationError("parity cell does not contain the exact implementation arms")
|
|
287
|
+
payload = result.payload
|
|
288
|
+
if payload.get("frozen_binding_receipt_hash") != binding_receipt.receipt_hash:
|
|
289
|
+
raise ValidationError("parity result binding receipt hash is absent or different")
|
|
290
|
+
if payload.get("parity_contract_hash") != contract.contract_hash:
|
|
291
|
+
raise ValidationError("parity result contract hash is absent or different")
|
|
292
|
+
arms = payload.get("arms")
|
|
293
|
+
if isinstance(arms, (str, bytes)) or not isinstance(arms, Sequence):
|
|
294
|
+
raise ValidationError("parity result requires an arms array")
|
|
295
|
+
receipt_protocol = (
|
|
296
|
+
operator_fingerprint.get("execution_receipt_protocol")
|
|
297
|
+
if operator_fingerprint is not None
|
|
298
|
+
else None
|
|
299
|
+
)
|
|
300
|
+
if receipt_protocol is not None:
|
|
301
|
+
if receipt_protocol != HOST_EXECUTION_RECEIPT_PROTOCOL:
|
|
302
|
+
raise ValidationError("parity driver declared an unknown execution receipt protocol")
|
|
303
|
+
_validate_host_execution_receipts(
|
|
304
|
+
cell,
|
|
305
|
+
result,
|
|
306
|
+
arms=arms,
|
|
307
|
+
binding_receipt=binding_receipt,
|
|
308
|
+
contract=contract,
|
|
309
|
+
operator_fingerprint=operator_fingerprint,
|
|
310
|
+
)
|
|
311
|
+
return
|
|
312
|
+
observed: dict[str, Mapping[str, Any]] = {}
|
|
313
|
+
for item in arms:
|
|
314
|
+
if not isinstance(item, Mapping):
|
|
315
|
+
raise ValidationError("parity arm outcome must be an object")
|
|
316
|
+
arm_id = item.get("arm_id")
|
|
317
|
+
identity = item.get("implementation_identity")
|
|
318
|
+
if not isinstance(arm_id, str) or not isinstance(identity, Mapping):
|
|
319
|
+
raise ValidationError("parity arm outcome requires arm_id and implementation_identity")
|
|
320
|
+
if arm_id in observed:
|
|
321
|
+
raise ValidationError("parity result contains duplicate implementation arms")
|
|
322
|
+
observed[arm_id] = identity
|
|
323
|
+
if set(observed) != set(_IMPLEMENTATION_KINDS):
|
|
324
|
+
raise ValidationError("parity result implementation arms differ from its contract")
|
|
325
|
+
for arm_id, identity in observed.items():
|
|
326
|
+
expected = freeze_mapping(contract.implementation(arm_id).to_dict())
|
|
327
|
+
try:
|
|
328
|
+
actual = freeze_mapping(identity)
|
|
329
|
+
except (TypeError, ValueError) as exc:
|
|
330
|
+
raise ValidationError("parity runtime identity must be canonical JSON") from exc
|
|
331
|
+
if actual != expected:
|
|
332
|
+
raise ValidationError(f"parity arm {arm_id!r} reported another implementation identity")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def validate_host_process_cell_result(
|
|
336
|
+
cell: ExperimentCell,
|
|
337
|
+
result: ExperimentCellResult,
|
|
338
|
+
*,
|
|
339
|
+
binding_receipt: FrozenBindingReceipt,
|
|
340
|
+
operator_fingerprint: Mapping[str, Any],
|
|
341
|
+
) -> None:
|
|
342
|
+
"""Validate host-generated receipts for a non-parity process experiment."""
|
|
343
|
+
|
|
344
|
+
if result.status == "failed":
|
|
345
|
+
return
|
|
346
|
+
if result.payload.get("frozen_binding_receipt_hash") != binding_receipt.receipt_hash:
|
|
347
|
+
raise ValidationError("host-process result binding receipt hash is absent or different")
|
|
348
|
+
if operator_fingerprint.get("execution_receipt_protocol") != HOST_EXECUTION_RECEIPT_PROTOCOL:
|
|
349
|
+
raise ValidationError("host-process driver declared an unknown receipt protocol")
|
|
350
|
+
arms = result.payload.get("arms")
|
|
351
|
+
if isinstance(arms, (str, bytes)) or not isinstance(arms, Sequence):
|
|
352
|
+
raise ValidationError("host-process result requires an arms array")
|
|
353
|
+
_validate_host_execution_receipts(
|
|
354
|
+
cell,
|
|
355
|
+
result,
|
|
356
|
+
arms=arms,
|
|
357
|
+
binding_receipt=binding_receipt,
|
|
358
|
+
contract=None,
|
|
359
|
+
operator_fingerprint=operator_fingerprint,
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _validate_host_execution_receipts(
|
|
364
|
+
cell: ExperimentCell,
|
|
365
|
+
result: ExperimentCellResult,
|
|
366
|
+
*,
|
|
367
|
+
arms: Sequence[Any],
|
|
368
|
+
binding_receipt: FrozenBindingReceipt,
|
|
369
|
+
contract: ParityImplementationContract | None,
|
|
370
|
+
operator_fingerprint: Mapping[str, Any],
|
|
371
|
+
) -> None:
|
|
372
|
+
"""Validate identities produced by the host process launcher, not workers."""
|
|
373
|
+
|
|
374
|
+
if result.payload.get("host_execution_receipt_protocol") != (HOST_EXECUTION_RECEIPT_PROTOCOL):
|
|
375
|
+
raise ValidationError("host-process result execution receipt protocol is absent")
|
|
376
|
+
config_sha256 = operator_fingerprint.get("config_sha256")
|
|
377
|
+
require_sha256(config_sha256, label="host-process driver config_sha256")
|
|
378
|
+
expected_arms = set(cell.arm_ids)
|
|
379
|
+
bindings = operator_fingerprint.get("host_process_bindings")
|
|
380
|
+
if not isinstance(bindings, Mapping) or set(bindings) != expected_arms:
|
|
381
|
+
raise ValidationError("host-process driver bindings are invalid")
|
|
382
|
+
resource_hashes = {content_hash(receipt.as_dict()) for receipt in result.resource_receipts}
|
|
383
|
+
if len(result.resource_receipts) != len(expected_arms) or len(resource_hashes) != len(
|
|
384
|
+
expected_arms
|
|
385
|
+
):
|
|
386
|
+
raise ValidationError("host process requires one distinct resource receipt per arm")
|
|
387
|
+
observed_arms: set[str] = set()
|
|
388
|
+
referenced_resource_hashes: set[str] = set()
|
|
389
|
+
for item in arms:
|
|
390
|
+
if not isinstance(item, Mapping):
|
|
391
|
+
raise ValidationError("host-process arm outcome must be an object")
|
|
392
|
+
arm_id = item.get("arm_id")
|
|
393
|
+
if not isinstance(arm_id, str) or arm_id not in expected_arms:
|
|
394
|
+
raise ValidationError("host-process receipt has an invalid arm")
|
|
395
|
+
if arm_id in observed_arms:
|
|
396
|
+
raise ValidationError("host-process result contains duplicate receipt arms")
|
|
397
|
+
observed_arms.add(arm_id)
|
|
398
|
+
if _contains_key(item, "implementation_identity"):
|
|
399
|
+
raise ValidationError(
|
|
400
|
+
"worker-provided implementation_identity is forbidden "
|
|
401
|
+
"when host execution receipts are enabled"
|
|
402
|
+
)
|
|
403
|
+
receipt = item.get("host_execution_receipt")
|
|
404
|
+
if not isinstance(receipt, Mapping) or set(receipt) != _HOST_RECEIPT_FIELDS:
|
|
405
|
+
raise ValidationError("host execution receipt fields are invalid")
|
|
406
|
+
if receipt.get("schema_version") != 1 or receipt.get("protocol") != (
|
|
407
|
+
HOST_EXECUTION_RECEIPT_PROTOCOL
|
|
408
|
+
):
|
|
409
|
+
raise ValidationError("host execution receipt protocol is invalid")
|
|
410
|
+
if receipt.get("arm_id") != arm_id:
|
|
411
|
+
raise ValidationError("host execution receipt belongs to another arm")
|
|
412
|
+
if (
|
|
413
|
+
receipt.get("manifest_hash") != cell.manifest_hash
|
|
414
|
+
or receipt.get("cell_id") != cell.cell_id
|
|
415
|
+
or receipt.get("request_hash") != item.get("request_hash")
|
|
416
|
+
or receipt.get("config_sha256") != config_sha256
|
|
417
|
+
):
|
|
418
|
+
raise ValidationError("host execution receipt lineage is invalid")
|
|
419
|
+
receipt_without_hash = {
|
|
420
|
+
key: value for key, value in receipt.items() if key != "receipt_hash"
|
|
421
|
+
}
|
|
422
|
+
if receipt.get("receipt_hash") != content_hash(receipt_without_hash):
|
|
423
|
+
raise ValidationError("host execution receipt hash is invalid")
|
|
424
|
+
|
|
425
|
+
expected_binding = bindings[arm_id]
|
|
426
|
+
if not isinstance(expected_binding, Mapping) or set(expected_binding) != {
|
|
427
|
+
"checkout_environment",
|
|
428
|
+
"checkout",
|
|
429
|
+
"adapter_checkout_environment",
|
|
430
|
+
"adapter_checkout",
|
|
431
|
+
"command",
|
|
432
|
+
}:
|
|
433
|
+
raise ValidationError("host-process binding fields are invalid")
|
|
434
|
+
checkout_environment = expected_binding["checkout_environment"]
|
|
435
|
+
if (
|
|
436
|
+
not isinstance(checkout_environment, str)
|
|
437
|
+
or receipt.get("checkout_environment") != checkout_environment
|
|
438
|
+
):
|
|
439
|
+
raise ValidationError("host receipt checkout environment differs")
|
|
440
|
+
frozen_checkout = binding_receipt.binding(
|
|
441
|
+
checkout_environment,
|
|
442
|
+
kind="directory_env",
|
|
443
|
+
)
|
|
444
|
+
if receipt.get("checkout_binding_hash") != frozen_checkout.binding_hash:
|
|
445
|
+
raise ValidationError("host receipt checkout binding differs")
|
|
446
|
+
checkout = receipt.get("checkout")
|
|
447
|
+
expected_checkout = expected_binding["checkout"]
|
|
448
|
+
if (
|
|
449
|
+
not isinstance(checkout, Mapping)
|
|
450
|
+
or set(checkout) != _HOST_CHECKOUT_FIELDS
|
|
451
|
+
or not isinstance(expected_checkout, Mapping)
|
|
452
|
+
or set(expected_checkout) != {"resolved_path_sha256", "git_head", "git_tree"}
|
|
453
|
+
):
|
|
454
|
+
raise ValidationError("host receipt checkout identity fields are invalid")
|
|
455
|
+
if (
|
|
456
|
+
checkout.get("resolved_path_sha256") != expected_checkout.get("resolved_path_sha256")
|
|
457
|
+
or checkout.get("git_head") != expected_checkout.get("git_head")
|
|
458
|
+
or checkout.get("git_tree") != expected_checkout.get("git_tree")
|
|
459
|
+
or checkout.get("git_clean_before") is not True
|
|
460
|
+
or checkout.get("git_clean_after") is not True
|
|
461
|
+
):
|
|
462
|
+
raise ValidationError("host receipt checkout identity differs")
|
|
463
|
+
frozen_identity = frozen_checkout.identity
|
|
464
|
+
if (
|
|
465
|
+
frozen_identity.get("resolved_path_sha256") != checkout.get("resolved_path_sha256")
|
|
466
|
+
or frozen_identity.get("git_head") != checkout.get("git_head")
|
|
467
|
+
or frozen_identity.get("git_tree") != checkout.get("git_tree")
|
|
468
|
+
):
|
|
469
|
+
raise ValidationError("host receipt does not match frozen checkout identity")
|
|
470
|
+
|
|
471
|
+
adapter_checkout_environment = expected_binding["adapter_checkout_environment"]
|
|
472
|
+
if (
|
|
473
|
+
not isinstance(adapter_checkout_environment, str)
|
|
474
|
+
or receipt.get("adapter_checkout_environment") != adapter_checkout_environment
|
|
475
|
+
):
|
|
476
|
+
raise ValidationError("host receipt adapter checkout environment differs")
|
|
477
|
+
frozen_adapter_checkout = binding_receipt.binding(
|
|
478
|
+
adapter_checkout_environment,
|
|
479
|
+
kind="directory_env",
|
|
480
|
+
)
|
|
481
|
+
if receipt.get("adapter_checkout_binding_hash") != frozen_adapter_checkout.binding_hash:
|
|
482
|
+
raise ValidationError("host receipt adapter checkout binding differs")
|
|
483
|
+
adapter_checkout = receipt.get("adapter_checkout")
|
|
484
|
+
expected_adapter_checkout = expected_binding["adapter_checkout"]
|
|
485
|
+
if (
|
|
486
|
+
not isinstance(adapter_checkout, Mapping)
|
|
487
|
+
or set(adapter_checkout) != _HOST_CHECKOUT_FIELDS
|
|
488
|
+
or not isinstance(expected_adapter_checkout, Mapping)
|
|
489
|
+
or set(expected_adapter_checkout) != {"resolved_path_sha256", "git_head", "git_tree"}
|
|
490
|
+
):
|
|
491
|
+
raise ValidationError("host receipt adapter checkout identity fields are invalid")
|
|
492
|
+
if (
|
|
493
|
+
adapter_checkout.get("resolved_path_sha256")
|
|
494
|
+
!= expected_adapter_checkout.get("resolved_path_sha256")
|
|
495
|
+
or adapter_checkout.get("git_head") != expected_adapter_checkout.get("git_head")
|
|
496
|
+
or adapter_checkout.get("git_tree") != expected_adapter_checkout.get("git_tree")
|
|
497
|
+
or adapter_checkout.get("git_clean_before") is not True
|
|
498
|
+
or adapter_checkout.get("git_clean_after") is not True
|
|
499
|
+
):
|
|
500
|
+
raise ValidationError("host receipt adapter checkout identity differs")
|
|
501
|
+
frozen_adapter_identity = frozen_adapter_checkout.identity
|
|
502
|
+
if (
|
|
503
|
+
frozen_adapter_identity.get("resolved_path_sha256")
|
|
504
|
+
!= adapter_checkout.get("resolved_path_sha256")
|
|
505
|
+
or frozen_adapter_identity.get("git_head") != adapter_checkout.get("git_head")
|
|
506
|
+
or frozen_adapter_identity.get("git_tree") != adapter_checkout.get("git_tree")
|
|
507
|
+
):
|
|
508
|
+
raise ValidationError("host receipt does not match frozen adapter checkout identity")
|
|
509
|
+
implementation = contract.implementation(arm_id) if contract is not None else None
|
|
510
|
+
if implementation is not None and (
|
|
511
|
+
implementation.source_revision != checkout.get("git_head")
|
|
512
|
+
or implementation.source_tree != checkout.get("git_tree")
|
|
513
|
+
):
|
|
514
|
+
raise ValidationError("parity host receipt does not match implementation identity")
|
|
515
|
+
command = receipt.get("command")
|
|
516
|
+
if not isinstance(command, Mapping):
|
|
517
|
+
raise ValidationError("host receipt command identity is invalid")
|
|
518
|
+
try:
|
|
519
|
+
actual_command = freeze_mapping(command)
|
|
520
|
+
declared_command = freeze_mapping(expected_binding["command"])
|
|
521
|
+
except (TypeError, ValueError) as exc:
|
|
522
|
+
raise ValidationError("host receipt command must be canonical JSON") from exc
|
|
523
|
+
if actual_command != declared_command:
|
|
524
|
+
raise ValidationError("host receipt command differs from driver binding")
|
|
525
|
+
if implementation is not None and implementation.runtime_entrypoint != (
|
|
526
|
+
f"{adapter_checkout_environment}:{command.get('entrypoint_relative_path')}"
|
|
527
|
+
):
|
|
528
|
+
raise ValidationError(
|
|
529
|
+
"parity host receipt entrypoint differs from implementation contract"
|
|
530
|
+
)
|
|
531
|
+
process = receipt.get("process")
|
|
532
|
+
if not isinstance(process, Mapping) or set(process) != _HOST_PROCESS_FIELDS:
|
|
533
|
+
raise ValidationError("host receipt process fields are invalid")
|
|
534
|
+
for field_name in _HOST_PROCESS_FIELDS:
|
|
535
|
+
require_sha256(
|
|
536
|
+
process.get(field_name),
|
|
537
|
+
label=f"host receipt {field_name}",
|
|
538
|
+
)
|
|
539
|
+
resource_hash = process["resource_receipt_sha256"]
|
|
540
|
+
if resource_hash not in resource_hashes or resource_hash in referenced_resource_hashes:
|
|
541
|
+
raise ValidationError("host receipt references another resource receipt")
|
|
542
|
+
referenced_resource_hashes.add(resource_hash)
|
|
543
|
+
if observed_arms != expected_arms:
|
|
544
|
+
raise ValidationError("host receipt arms differ from the frozen cell")
|
|
545
|
+
if referenced_resource_hashes != resource_hashes:
|
|
546
|
+
raise ValidationError("host receipts do not cover all resource receipts")
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _contains_key(value: Any, key: str) -> bool:
|
|
550
|
+
if isinstance(value, Mapping):
|
|
551
|
+
return key in value or any(_contains_key(item, key) for item in value.values())
|
|
552
|
+
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
|
553
|
+
return any(_contains_key(item, key) for item in value)
|
|
554
|
+
return False
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _implementation_from_declaration(
|
|
558
|
+
arm_id: str,
|
|
559
|
+
value: Any,
|
|
560
|
+
*,
|
|
561
|
+
checkout_binding_hash: str | None,
|
|
562
|
+
) -> ParityImplementationIdentity:
|
|
563
|
+
expected_fields = {
|
|
564
|
+
"implementation_kind",
|
|
565
|
+
"implementation_id",
|
|
566
|
+
"runtime_entrypoint",
|
|
567
|
+
"source_revision",
|
|
568
|
+
"source_tree",
|
|
569
|
+
"release_id",
|
|
570
|
+
}
|
|
571
|
+
if not isinstance(value, Mapping) or set(value) != expected_fields:
|
|
572
|
+
raise ValidationError(f"parity implementation declaration {arm_id!r} has invalid fields")
|
|
573
|
+
expected_kind = _IMPLEMENTATION_KINDS[arm_id]
|
|
574
|
+
if value["implementation_kind"] != expected_kind:
|
|
575
|
+
raise ValidationError(f"parity implementation declaration {arm_id!r} has the wrong kind")
|
|
576
|
+
return ParityImplementationIdentity(
|
|
577
|
+
arm_id=arm_id,
|
|
578
|
+
implementation_kind=expected_kind,
|
|
579
|
+
implementation_id=_text(value["implementation_id"], "implementation_id"),
|
|
580
|
+
runtime_entrypoint=_text(value["runtime_entrypoint"], "runtime_entrypoint"),
|
|
581
|
+
source_revision=_text(value["source_revision"], "source_revision"),
|
|
582
|
+
source_tree=_text(value["source_tree"], "source_tree"),
|
|
583
|
+
release_id=_text(value["release_id"], "release_id"),
|
|
584
|
+
checkout_binding_hash=checkout_binding_hash,
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def _metadata_text(manifest: ExperimentManifest, name: str) -> str:
|
|
589
|
+
return _text(manifest.metadata.get(name), f"manifest metadata {name}")
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _text(value: Any, label: str) -> str:
|
|
593
|
+
if not isinstance(value, str) or not value.strip():
|
|
594
|
+
raise ValidationError(f"{label} must be a non-empty string")
|
|
595
|
+
return value
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
__all__ = [
|
|
599
|
+
"HOST_EXECUTION_RECEIPT_PROTOCOL",
|
|
600
|
+
"ParityImplementationContract",
|
|
601
|
+
"ParityImplementationIdentity",
|
|
602
|
+
"build_parity_implementation_contract",
|
|
603
|
+
"validate_host_process_cell_result",
|
|
604
|
+
"validate_parity_cell_result",
|
|
605
|
+
]
|