agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,727 @@
|
|
|
1
|
+
"""Machine-readable, fail-closed readiness checks for the v0.2 beta gate."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from agentinputlab.contracts.enums import ProfileKind, ReleaseTier
|
|
13
|
+
from agentinputlab.contracts.serialization import content_hash
|
|
14
|
+
|
|
15
|
+
from .loader import load_registry_file
|
|
16
|
+
from .result_manifest import validate_result_table_manifest
|
|
17
|
+
from .runtime import AlgorithmRegistry
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, slots=True)
|
|
21
|
+
class BetaGateCheck:
|
|
22
|
+
"""One independently auditable beta release criterion."""
|
|
23
|
+
|
|
24
|
+
check_id: str
|
|
25
|
+
passed: bool
|
|
26
|
+
detail: str
|
|
27
|
+
blockers: tuple[str, ...] = ()
|
|
28
|
+
|
|
29
|
+
def to_dict(self) -> dict[str, Any]:
|
|
30
|
+
return {
|
|
31
|
+
"check_id": self.check_id,
|
|
32
|
+
"status": "PASS" if self.passed else "BLOCKED",
|
|
33
|
+
"passed": self.passed,
|
|
34
|
+
"detail": self.detail,
|
|
35
|
+
"blockers": list(self.blockers),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True, slots=True)
|
|
40
|
+
class BetaReadinessReport:
|
|
41
|
+
"""Deterministic summary suitable for CLI output and release artifacts."""
|
|
42
|
+
|
|
43
|
+
gate_id: str
|
|
44
|
+
manifest_sha256: str
|
|
45
|
+
checks: tuple[BetaGateCheck, ...]
|
|
46
|
+
schema_version: int = 1
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def ready(self) -> bool:
|
|
50
|
+
return all(item.passed for item in self.checks)
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def blockers(self) -> tuple[str, ...]:
|
|
54
|
+
return tuple(
|
|
55
|
+
f"{item.check_id}: {blocker}" for item in self.checks for blocker in item.blockers
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def to_dict(self) -> dict[str, Any]:
|
|
59
|
+
return {
|
|
60
|
+
"schema_version": self.schema_version,
|
|
61
|
+
"gate_id": self.gate_id,
|
|
62
|
+
"status": "READY" if self.ready else "BLOCKED",
|
|
63
|
+
"ready": self.ready,
|
|
64
|
+
"manifest_sha256": self.manifest_sha256,
|
|
65
|
+
"checks": [item.to_dict() for item in self.checks],
|
|
66
|
+
"blockers": list(self.blockers),
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _blocked(check_id: str, detail: str, *blockers: str) -> BetaGateCheck:
|
|
71
|
+
return BetaGateCheck(
|
|
72
|
+
check_id=check_id,
|
|
73
|
+
passed=False,
|
|
74
|
+
detail=detail,
|
|
75
|
+
blockers=tuple(blockers) or ("criterion was not satisfied",),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _passed(check_id: str, detail: str) -> BetaGateCheck:
|
|
80
|
+
return BetaGateCheck(check_id=check_id, passed=True, detail=detail)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _repository_path(
|
|
84
|
+
root: Path,
|
|
85
|
+
value: object,
|
|
86
|
+
label: str,
|
|
87
|
+
*,
|
|
88
|
+
reject_symlinks: bool = False,
|
|
89
|
+
) -> tuple[Path | None, str | None]:
|
|
90
|
+
if not isinstance(value, str) or not value.strip():
|
|
91
|
+
return None, f"{label} must be a non-empty repository-relative path"
|
|
92
|
+
relative = Path(value)
|
|
93
|
+
if relative.is_absolute() or ".." in relative.parts:
|
|
94
|
+
return None, f"{label} must be a repository-relative path without parent traversal"
|
|
95
|
+
candidate = root / relative
|
|
96
|
+
if reject_symlinks:
|
|
97
|
+
current = root
|
|
98
|
+
for part in relative.parts:
|
|
99
|
+
current /= part
|
|
100
|
+
if current.is_symlink():
|
|
101
|
+
return None, f"{label} must not contain a symbolic link: {current}"
|
|
102
|
+
candidate = candidate.resolve()
|
|
103
|
+
try:
|
|
104
|
+
candidate.relative_to(root)
|
|
105
|
+
except ValueError:
|
|
106
|
+
return None, f"{label} escapes the repository root"
|
|
107
|
+
return candidate, None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _check_roster(
|
|
111
|
+
registry: AlgorithmRegistry,
|
|
112
|
+
manifest: Mapping[str, Any],
|
|
113
|
+
) -> BetaGateCheck:
|
|
114
|
+
expected = manifest.get("required_roster")
|
|
115
|
+
if not isinstance(expected, Mapping):
|
|
116
|
+
return _blocked("roster.20_plus_7", "Frozen algorithm roster", "required_roster missing")
|
|
117
|
+
core_expected = expected.get("core")
|
|
118
|
+
optional_expected = expected.get("optional")
|
|
119
|
+
methods = registry.document.algorithms
|
|
120
|
+
core_actual = sum(item.release_tier is ReleaseTier.CORE for item in methods)
|
|
121
|
+
optional_actual = sum(item.release_tier is ReleaseTier.OPTIONAL for item in methods)
|
|
122
|
+
blockers: list[str] = []
|
|
123
|
+
if core_expected != core_actual:
|
|
124
|
+
blockers.append(f"core roster expected {core_expected!r}, found {core_actual}")
|
|
125
|
+
if optional_expected != optional_actual:
|
|
126
|
+
blockers.append(f"optional roster expected {optional_expected!r}, found {optional_actual}")
|
|
127
|
+
if blockers:
|
|
128
|
+
return _blocked("roster.20_plus_7", "Frozen algorithm roster", *blockers)
|
|
129
|
+
return _passed(
|
|
130
|
+
"roster.20_plus_7",
|
|
131
|
+
f"registry contains {core_actual} core and {optional_actual} optional methods",
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _check_l2_paths(
|
|
136
|
+
root: Path,
|
|
137
|
+
registry: AlgorithmRegistry,
|
|
138
|
+
manifest: Mapping[str, Any],
|
|
139
|
+
) -> BetaGateCheck:
|
|
140
|
+
raw_paths = manifest.get("l2_paths")
|
|
141
|
+
minimum = manifest.get("minimum_l2_methods_per_family")
|
|
142
|
+
raw_required_families = manifest.get("required_l2_families")
|
|
143
|
+
if not isinstance(raw_paths, list) or not raw_paths:
|
|
144
|
+
return _blocked(
|
|
145
|
+
"l2.family_coverage",
|
|
146
|
+
"Resolvable documented L2 paths",
|
|
147
|
+
"l2_paths must be a non-empty array",
|
|
148
|
+
)
|
|
149
|
+
if not isinstance(minimum, int) or isinstance(minimum, bool) or minimum < 1:
|
|
150
|
+
return _blocked(
|
|
151
|
+
"l2.family_coverage",
|
|
152
|
+
"Resolvable documented L2 paths",
|
|
153
|
+
"minimum_l2_methods_per_family must be a positive integer",
|
|
154
|
+
)
|
|
155
|
+
if (
|
|
156
|
+
not isinstance(raw_required_families, list)
|
|
157
|
+
or not raw_required_families
|
|
158
|
+
or any(not isinstance(item, str) or not item for item in raw_required_families)
|
|
159
|
+
):
|
|
160
|
+
return _blocked(
|
|
161
|
+
"l2.family_coverage",
|
|
162
|
+
"Resolvable documented L2 paths",
|
|
163
|
+
"required_l2_families must be a non-empty array of family names",
|
|
164
|
+
)
|
|
165
|
+
required_families = list(dict.fromkeys(raw_required_families))
|
|
166
|
+
if len(required_families) != len(raw_required_families):
|
|
167
|
+
return _blocked(
|
|
168
|
+
"l2.family_coverage",
|
|
169
|
+
"Resolvable documented L2 paths",
|
|
170
|
+
"required_l2_families must not contain duplicates",
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
blockers: list[str] = []
|
|
174
|
+
declared_methods: list[str] = []
|
|
175
|
+
family_counts: Counter[str] = Counter()
|
|
176
|
+
registry_method_ids = {item.method_id for item in registry.document.algorithms}
|
|
177
|
+
for index, raw in enumerate(raw_paths):
|
|
178
|
+
label = f"l2_paths[{index}]"
|
|
179
|
+
if not isinstance(raw, Mapping):
|
|
180
|
+
blockers.append(f"{label} must be an object")
|
|
181
|
+
continue
|
|
182
|
+
method_id = raw.get("method_id")
|
|
183
|
+
release_id = raw.get("release_id")
|
|
184
|
+
if not isinstance(method_id, str) or not method_id:
|
|
185
|
+
blockers.append(f"{label}.method_id must be non-empty")
|
|
186
|
+
continue
|
|
187
|
+
declared_methods.append(method_id)
|
|
188
|
+
if raw.get("evidence_level") != "L2":
|
|
189
|
+
blockers.append(f"{label}.evidence_level must equal L2")
|
|
190
|
+
evidence_path, path_error = _repository_path(
|
|
191
|
+
root,
|
|
192
|
+
raw.get("evidence_path"),
|
|
193
|
+
f"{label}.evidence_path",
|
|
194
|
+
)
|
|
195
|
+
if path_error is not None:
|
|
196
|
+
blockers.append(path_error)
|
|
197
|
+
elif evidence_path is not None and not evidence_path.is_file():
|
|
198
|
+
blockers.append(f"{label}.evidence_path does not exist")
|
|
199
|
+
if not isinstance(release_id, str) or not release_id:
|
|
200
|
+
blockers.append(f"{label}.release_id must be non-empty")
|
|
201
|
+
continue
|
|
202
|
+
try:
|
|
203
|
+
method, release = registry.get_release(release_id)
|
|
204
|
+
except Exception as exc:
|
|
205
|
+
blockers.append(f"{label} references unknown release {release_id!r}: {exc}")
|
|
206
|
+
continue
|
|
207
|
+
if method.method_id != method_id:
|
|
208
|
+
blockers.append(
|
|
209
|
+
f"{label} release belongs to {method.method_id}, not declared {method_id}"
|
|
210
|
+
)
|
|
211
|
+
continue
|
|
212
|
+
if not release.entrypoint_available:
|
|
213
|
+
blockers.append(f"{label} release has no integration entrypoint")
|
|
214
|
+
continue
|
|
215
|
+
try:
|
|
216
|
+
resolved = registry.resolve_entrypoint(release_id)
|
|
217
|
+
except Exception as exc:
|
|
218
|
+
blockers.append(f"{label} entrypoint cannot resolve: {exc}")
|
|
219
|
+
continue
|
|
220
|
+
if not callable(resolved):
|
|
221
|
+
blockers.append(f"{label} entrypoint is not callable")
|
|
222
|
+
continue
|
|
223
|
+
family_counts[method.semantic_family.value] += 1
|
|
224
|
+
|
|
225
|
+
duplicates = sorted(
|
|
226
|
+
method_id for method_id, count in Counter(declared_methods).items() if count > 1
|
|
227
|
+
)
|
|
228
|
+
if duplicates:
|
|
229
|
+
blockers.append(f"duplicate L2 method declarations: {', '.join(duplicates)}")
|
|
230
|
+
missing = sorted(registry_method_ids - set(declared_methods))
|
|
231
|
+
extra = sorted(set(declared_methods) - registry_method_ids)
|
|
232
|
+
if missing:
|
|
233
|
+
blockers.append(f"L2 manifest omits methods: {', '.join(missing)}")
|
|
234
|
+
if extra:
|
|
235
|
+
blockers.append(f"L2 manifest declares unknown methods: {', '.join(extra)}")
|
|
236
|
+
|
|
237
|
+
registry_families = {item.semantic_family.value for item in registry.document.algorithms}
|
|
238
|
+
unknown_required_families = sorted(set(required_families) - registry_families)
|
|
239
|
+
if unknown_required_families:
|
|
240
|
+
blockers.append(
|
|
241
|
+
"required_l2_families contains unknown families: "
|
|
242
|
+
+ ", ".join(unknown_required_families)
|
|
243
|
+
)
|
|
244
|
+
for family in required_families:
|
|
245
|
+
count = family_counts[family]
|
|
246
|
+
if count < minimum:
|
|
247
|
+
blockers.append(
|
|
248
|
+
f"family {family} has {count} resolvable L2 methods; requires at least {minimum}"
|
|
249
|
+
)
|
|
250
|
+
if blockers:
|
|
251
|
+
return _blocked(
|
|
252
|
+
"l2.family_coverage",
|
|
253
|
+
"Resolvable documented L2 paths",
|
|
254
|
+
*blockers,
|
|
255
|
+
)
|
|
256
|
+
rendered = ", ".join(f"{family}={family_counts[family]}" for family in required_families)
|
|
257
|
+
return _passed(
|
|
258
|
+
"l2.family_coverage",
|
|
259
|
+
f"{len(declared_methods)} methods have resolvable L2 paths; {rendered}",
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _check_selectors(manifest: Mapping[str, Any]) -> BetaGateCheck:
|
|
264
|
+
minimum = manifest.get("minimum_selector_count")
|
|
265
|
+
if not isinstance(minimum, int) or isinstance(minimum, bool) or minimum < 1:
|
|
266
|
+
return _blocked(
|
|
267
|
+
"selector.catalog",
|
|
268
|
+
"Selector catalog size",
|
|
269
|
+
"minimum_selector_count must be a positive integer",
|
|
270
|
+
)
|
|
271
|
+
try:
|
|
272
|
+
from agentinputlab.selectors import list_selector_cards
|
|
273
|
+
|
|
274
|
+
cards = list_selector_cards()
|
|
275
|
+
except Exception as exc:
|
|
276
|
+
return _blocked(
|
|
277
|
+
"selector.catalog",
|
|
278
|
+
"Selector catalog size",
|
|
279
|
+
f"selector catalog could not be loaded: {exc}",
|
|
280
|
+
)
|
|
281
|
+
if len(cards) < minimum:
|
|
282
|
+
return _blocked(
|
|
283
|
+
"selector.catalog",
|
|
284
|
+
"Selector catalog size",
|
|
285
|
+
f"found {len(cards)} selectors; requires at least {minimum}",
|
|
286
|
+
)
|
|
287
|
+
return _passed(
|
|
288
|
+
"selector.catalog",
|
|
289
|
+
f"{len(cards)} selector cards are registered (minimum {minimum})",
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _check_algorithm_licenses(registry: AlgorithmRegistry) -> BetaGateCheck:
|
|
294
|
+
blockers: list[str] = []
|
|
295
|
+
for method in registry.document.algorithms:
|
|
296
|
+
upstream = method.upstream
|
|
297
|
+
if not upstream.license_id.strip():
|
|
298
|
+
blockers.append(f"{method.method_id} has no source license identity")
|
|
299
|
+
if upstream.license_id == "NOASSERTION" and upstream.vendoring_allowed:
|
|
300
|
+
blockers.append(f"{method.method_id} may not vendor a NOASSERTION source")
|
|
301
|
+
if blockers:
|
|
302
|
+
return _blocked(
|
|
303
|
+
"license.algorithm_boundaries",
|
|
304
|
+
"Algorithm source-license boundaries",
|
|
305
|
+
*blockers,
|
|
306
|
+
)
|
|
307
|
+
noassertion_count = sum(
|
|
308
|
+
item.upstream.license_id == "NOASSERTION" for item in registry.document.algorithms
|
|
309
|
+
)
|
|
310
|
+
return _passed(
|
|
311
|
+
"license.algorithm_boundaries",
|
|
312
|
+
f"all source licenses are explicit; {noassertion_count} NOASSERTION sources remain external",
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _validate_benchmark_card(
|
|
317
|
+
raw: Any,
|
|
318
|
+
*,
|
|
319
|
+
expected_cell_id: str,
|
|
320
|
+
registry_cell: Mapping[str, Any],
|
|
321
|
+
label: str,
|
|
322
|
+
) -> tuple[str, ...]:
|
|
323
|
+
if not isinstance(raw, Mapping):
|
|
324
|
+
return (f"{label} must contain an object",)
|
|
325
|
+
blockers: list[str] = []
|
|
326
|
+
if raw.get("schema_version") != 1:
|
|
327
|
+
blockers.append(f"{label}.schema_version must equal 1")
|
|
328
|
+
if raw.get("cell_id") != expected_cell_id:
|
|
329
|
+
blockers.append(f"{label}.cell_id must equal {expected_cell_id}")
|
|
330
|
+
for key in ("source_url", "source_revision", "license_id"):
|
|
331
|
+
value = raw.get(key)
|
|
332
|
+
if not isinstance(value, str) or not value.strip():
|
|
333
|
+
blockers.append(f"{label}.{key} must be a non-empty string")
|
|
334
|
+
if raw.get("source_revision") != registry_cell.get("source_revision"):
|
|
335
|
+
blockers.append(f"{label}.source_revision does not match benchmark registry")
|
|
336
|
+
vendoring_allowed = raw.get("vendoring_allowed")
|
|
337
|
+
if not isinstance(vendoring_allowed, bool):
|
|
338
|
+
blockers.append(f"{label}.vendoring_allowed must be a boolean")
|
|
339
|
+
if raw.get("license_id") == "NOASSERTION" and vendoring_allowed is not False:
|
|
340
|
+
blockers.append(f"{label} cannot vendor a NOASSERTION source")
|
|
341
|
+
|
|
342
|
+
scorer = raw.get("scorer")
|
|
343
|
+
if not isinstance(scorer, Mapping):
|
|
344
|
+
blockers.append(f"{label}.scorer must be an object")
|
|
345
|
+
else:
|
|
346
|
+
for key in ("identity", "revision"):
|
|
347
|
+
value = scorer.get(key)
|
|
348
|
+
if not isinstance(value, str) or not value.strip():
|
|
349
|
+
blockers.append(f"{label}.scorer.{key} must be a non-empty string")
|
|
350
|
+
|
|
351
|
+
gold = raw.get("gold_boundary")
|
|
352
|
+
if not isinstance(gold, Mapping):
|
|
353
|
+
blockers.append(f"{label}.gold_boundary must be an object")
|
|
354
|
+
else:
|
|
355
|
+
if gold.get("optimizer_visible") is not False:
|
|
356
|
+
blockers.append(f"{label}.gold_boundary.optimizer_visible must be false")
|
|
357
|
+
if gold.get("evaluator_only") is not True:
|
|
358
|
+
blockers.append(f"{label}.gold_boundary.evaluator_only must be true")
|
|
359
|
+
return tuple(blockers)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _check_benchmark_cards(
|
|
363
|
+
root: Path,
|
|
364
|
+
manifest: Mapping[str, Any],
|
|
365
|
+
) -> BetaGateCheck:
|
|
366
|
+
registry_path, path_error = _repository_path(
|
|
367
|
+
root,
|
|
368
|
+
manifest.get("benchmark_registry"),
|
|
369
|
+
"benchmark_registry",
|
|
370
|
+
)
|
|
371
|
+
if path_error is not None or registry_path is None:
|
|
372
|
+
return _blocked(
|
|
373
|
+
"benchmark.cards",
|
|
374
|
+
"Benchmark source/license/scorer/gold cards",
|
|
375
|
+
path_error or "benchmark_registry is invalid",
|
|
376
|
+
)
|
|
377
|
+
try:
|
|
378
|
+
registry_raw = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
379
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
380
|
+
return _blocked(
|
|
381
|
+
"benchmark.cards",
|
|
382
|
+
"Benchmark source/license/scorer/gold cards",
|
|
383
|
+
f"cannot load benchmark registry: {exc}",
|
|
384
|
+
)
|
|
385
|
+
raw_cells = registry_raw.get("cells") if isinstance(registry_raw, Mapping) else None
|
|
386
|
+
if not isinstance(raw_cells, list):
|
|
387
|
+
return _blocked(
|
|
388
|
+
"benchmark.cards",
|
|
389
|
+
"Benchmark source/license/scorer/gold cards",
|
|
390
|
+
"benchmark registry cells must be an array",
|
|
391
|
+
)
|
|
392
|
+
cells = {
|
|
393
|
+
item.get("cell_id"): item
|
|
394
|
+
for item in raw_cells
|
|
395
|
+
if isinstance(item, Mapping) and isinstance(item.get("cell_id"), str)
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
requirements = manifest.get("required_benchmark_cards")
|
|
399
|
+
if not isinstance(requirements, list) or not requirements:
|
|
400
|
+
return _blocked(
|
|
401
|
+
"benchmark.cards",
|
|
402
|
+
"Benchmark source/license/scorer/gold cards",
|
|
403
|
+
"required_benchmark_cards must be a non-empty array",
|
|
404
|
+
)
|
|
405
|
+
blockers: list[str] = []
|
|
406
|
+
declared: list[str] = []
|
|
407
|
+
for index, requirement in enumerate(requirements):
|
|
408
|
+
label = f"required_benchmark_cards[{index}]"
|
|
409
|
+
if not isinstance(requirement, Mapping):
|
|
410
|
+
blockers.append(f"{label} must be an object")
|
|
411
|
+
continue
|
|
412
|
+
cell_id = requirement.get("cell_id")
|
|
413
|
+
if not isinstance(cell_id, str) or not cell_id:
|
|
414
|
+
blockers.append(f"{label}.cell_id must be non-empty")
|
|
415
|
+
continue
|
|
416
|
+
declared.append(cell_id)
|
|
417
|
+
registry_cell = cells.get(cell_id)
|
|
418
|
+
if registry_cell is None:
|
|
419
|
+
blockers.append(f"{cell_id} is absent from the benchmark registry")
|
|
420
|
+
continue
|
|
421
|
+
card_path, card_error = _repository_path(
|
|
422
|
+
root,
|
|
423
|
+
requirement.get("path"),
|
|
424
|
+
f"{label}.path",
|
|
425
|
+
)
|
|
426
|
+
if card_error is not None or card_path is None:
|
|
427
|
+
blockers.append(card_error or f"{label}.path is invalid")
|
|
428
|
+
continue
|
|
429
|
+
if not card_path.is_file():
|
|
430
|
+
blockers.append(f"{cell_id} benchmark card is missing at {card_path.relative_to(root)}")
|
|
431
|
+
continue
|
|
432
|
+
try:
|
|
433
|
+
card = json.loads(card_path.read_text(encoding="utf-8"))
|
|
434
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
435
|
+
blockers.append(f"{cell_id} benchmark card cannot be loaded: {exc}")
|
|
436
|
+
continue
|
|
437
|
+
blockers.extend(
|
|
438
|
+
_validate_benchmark_card(
|
|
439
|
+
card,
|
|
440
|
+
expected_cell_id=cell_id,
|
|
441
|
+
registry_cell=registry_cell,
|
|
442
|
+
label=f"benchmark card {cell_id}",
|
|
443
|
+
)
|
|
444
|
+
)
|
|
445
|
+
duplicates = sorted(cell_id for cell_id, count in Counter(declared).items() if count > 1)
|
|
446
|
+
if duplicates:
|
|
447
|
+
blockers.append(f"duplicate benchmark card requirements: {', '.join(duplicates)}")
|
|
448
|
+
if blockers:
|
|
449
|
+
return _blocked(
|
|
450
|
+
"benchmark.cards",
|
|
451
|
+
"Benchmark source/license/scorer/gold cards",
|
|
452
|
+
*blockers,
|
|
453
|
+
)
|
|
454
|
+
return _passed(
|
|
455
|
+
"benchmark.cards",
|
|
456
|
+
f"{len(declared)} required benchmark cards have frozen source, license, scorer, and gold boundaries",
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _check_result_profile_policy(
|
|
461
|
+
root: Path,
|
|
462
|
+
manifest: Mapping[str, Any],
|
|
463
|
+
) -> BetaGateCheck:
|
|
464
|
+
policy = manifest.get("result_table_policy")
|
|
465
|
+
if not isinstance(policy, Mapping):
|
|
466
|
+
return _blocked(
|
|
467
|
+
"results.profile_boundary",
|
|
468
|
+
"Single-profile-kind result tables",
|
|
469
|
+
"result_table_policy missing",
|
|
470
|
+
)
|
|
471
|
+
blockers: list[str] = []
|
|
472
|
+
if policy.get("require_single_profile_kind") is not True:
|
|
473
|
+
blockers.append("result_table_policy.require_single_profile_kind must be true")
|
|
474
|
+
allowed = policy.get("allowed_profile_kinds")
|
|
475
|
+
expected = {item.value for item in ProfileKind}
|
|
476
|
+
if not isinstance(allowed, list) or set(allowed) != expected:
|
|
477
|
+
blockers.append(
|
|
478
|
+
"result_table_policy.allowed_profile_kinds must contain exactly "
|
|
479
|
+
+ ", ".join(sorted(expected))
|
|
480
|
+
)
|
|
481
|
+
if policy.get("validator") != "agentinputlab.registry:validate_result_table_manifest":
|
|
482
|
+
blockers.append("result_table_policy.validator must name the shipped validator")
|
|
483
|
+
minimum_manifests = policy.get("minimum_manifests")
|
|
484
|
+
if (
|
|
485
|
+
isinstance(minimum_manifests, bool)
|
|
486
|
+
or not isinstance(minimum_manifests, int)
|
|
487
|
+
or minimum_manifests < 1
|
|
488
|
+
):
|
|
489
|
+
blockers.append("result_table_policy.minimum_manifests must be a positive integer")
|
|
490
|
+
minimum_manifests = 1
|
|
491
|
+
manifest_glob = policy.get("manifest_glob")
|
|
492
|
+
if not isinstance(manifest_glob, str) or not manifest_glob.strip():
|
|
493
|
+
blockers.append("result_table_policy.manifest_glob must be non-empty")
|
|
494
|
+
result_paths: list[Path] = []
|
|
495
|
+
elif Path(manifest_glob).is_absolute() or ".." in Path(manifest_glob).parts:
|
|
496
|
+
blockers.append("result_table_policy.manifest_glob must remain repository-relative")
|
|
497
|
+
result_paths = []
|
|
498
|
+
else:
|
|
499
|
+
result_paths = sorted(root.glob(manifest_glob))
|
|
500
|
+
if len(result_paths) < minimum_manifests:
|
|
501
|
+
blockers.append(
|
|
502
|
+
"result-table manifest inventory is incomplete: "
|
|
503
|
+
f"found {len(result_paths)}, requires at least {minimum_manifests}"
|
|
504
|
+
)
|
|
505
|
+
seen_table_ids: dict[str, Path] = {}
|
|
506
|
+
seen_run_identities: dict[str, Path] = {}
|
|
507
|
+
for path in result_paths:
|
|
508
|
+
try:
|
|
509
|
+
relative_manifest_path = path.relative_to(root)
|
|
510
|
+
except ValueError:
|
|
511
|
+
blockers.append(f"result-table manifest path escapes repository root: {path}")
|
|
512
|
+
continue
|
|
513
|
+
checked_path, checked_path_error = _repository_path(
|
|
514
|
+
root,
|
|
515
|
+
relative_manifest_path.as_posix(),
|
|
516
|
+
f"result-table manifest {relative_manifest_path}",
|
|
517
|
+
reject_symlinks=True,
|
|
518
|
+
)
|
|
519
|
+
if checked_path_error is not None or checked_path is None:
|
|
520
|
+
blockers.append(
|
|
521
|
+
checked_path_error
|
|
522
|
+
or f"result-table manifest {relative_manifest_path} is invalid"
|
|
523
|
+
)
|
|
524
|
+
continue
|
|
525
|
+
if not checked_path.is_file():
|
|
526
|
+
blockers.append(
|
|
527
|
+
f"result-table manifest {relative_manifest_path} is not a regular file"
|
|
528
|
+
)
|
|
529
|
+
continue
|
|
530
|
+
try:
|
|
531
|
+
manifest_bytes = checked_path.read_bytes()
|
|
532
|
+
raw = json.loads(manifest_bytes)
|
|
533
|
+
table = validate_result_table_manifest(raw)
|
|
534
|
+
except Exception as exc:
|
|
535
|
+
blockers.append(f"{relative_manifest_path} violates profile boundary: {exc}")
|
|
536
|
+
continue
|
|
537
|
+
previous_table = seen_table_ids.get(table.table_id)
|
|
538
|
+
if previous_table is not None:
|
|
539
|
+
blockers.append(
|
|
540
|
+
f"{relative_manifest_path} duplicates table_id {table.table_id!r} "
|
|
541
|
+
f"already declared by {previous_table}"
|
|
542
|
+
)
|
|
543
|
+
else:
|
|
544
|
+
seen_table_ids[table.table_id] = relative_manifest_path
|
|
545
|
+
for study in table.studies:
|
|
546
|
+
previous_run = seen_run_identities.get(study.run_identity_hash)
|
|
547
|
+
if previous_run is not None:
|
|
548
|
+
blockers.append(
|
|
549
|
+
f"{relative_manifest_path} duplicates run_identity_hash "
|
|
550
|
+
f"{study.run_identity_hash} already referenced by {previous_run}"
|
|
551
|
+
)
|
|
552
|
+
else:
|
|
553
|
+
seen_run_identities[study.run_identity_hash] = relative_manifest_path
|
|
554
|
+
run_path, run_path_error = _repository_path(
|
|
555
|
+
root,
|
|
556
|
+
study.run_path,
|
|
557
|
+
f"{relative_manifest_path} study {study.study_id} run_path",
|
|
558
|
+
reject_symlinks=True,
|
|
559
|
+
)
|
|
560
|
+
if run_path_error is not None or run_path is None:
|
|
561
|
+
blockers.append(run_path_error or f"{study.study_id} run_path is invalid")
|
|
562
|
+
continue
|
|
563
|
+
try:
|
|
564
|
+
from agentinputlab.experiments import verify_experiment_run_directory
|
|
565
|
+
|
|
566
|
+
verified = verify_experiment_run_directory(run_path)
|
|
567
|
+
except Exception as exc:
|
|
568
|
+
blockers.append(
|
|
569
|
+
f"{relative_manifest_path} study {study.study_id} "
|
|
570
|
+
f"does not reference a verified run: {exc}"
|
|
571
|
+
)
|
|
572
|
+
continue
|
|
573
|
+
expected = {
|
|
574
|
+
"study_id": study.study_id,
|
|
575
|
+
"profile_kind": study.profile_kind.value,
|
|
576
|
+
"comparison_profile_hash": study.profile_hash,
|
|
577
|
+
"identity_hash": study.run_identity_hash,
|
|
578
|
+
"manifest_hash": study.manifest_hash,
|
|
579
|
+
"plan_hash": study.plan_hash,
|
|
580
|
+
"driver_hash": study.driver_hash,
|
|
581
|
+
}
|
|
582
|
+
mismatches = [key for key, value in expected.items() if verified.get(key) != value]
|
|
583
|
+
if mismatches:
|
|
584
|
+
blockers.append(
|
|
585
|
+
f"{relative_manifest_path} study {study.study_id} verified "
|
|
586
|
+
"run identity differs for: " + ", ".join(sorted(mismatches))
|
|
587
|
+
)
|
|
588
|
+
if not verified.get("complete") or not verified.get("all_succeeded"):
|
|
589
|
+
blockers.append(
|
|
590
|
+
f"{relative_manifest_path} study {study.study_id} "
|
|
591
|
+
"run is incomplete or contains failed cells"
|
|
592
|
+
)
|
|
593
|
+
if verified.get("is_shard"):
|
|
594
|
+
blockers.append(
|
|
595
|
+
f"{relative_manifest_path} study {study.study_id} "
|
|
596
|
+
"references a shard rather than a complete parent plan"
|
|
597
|
+
)
|
|
598
|
+
if blockers:
|
|
599
|
+
return _blocked(
|
|
600
|
+
"results.profile_boundary",
|
|
601
|
+
"Single-profile-kind result tables",
|
|
602
|
+
*blockers,
|
|
603
|
+
)
|
|
604
|
+
return _passed(
|
|
605
|
+
"results.profile_boundary",
|
|
606
|
+
f"profile-kind policy is enforced; {len(result_paths)} result-table manifests validated",
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def check_beta_readiness(
|
|
611
|
+
root: str | Path,
|
|
612
|
+
*,
|
|
613
|
+
manifest_path: str | Path | None = None,
|
|
614
|
+
registry: AlgorithmRegistry | None = None,
|
|
615
|
+
) -> BetaReadinessReport:
|
|
616
|
+
"""Evaluate integrity-oriented beta gates without claiming execution authenticity."""
|
|
617
|
+
|
|
618
|
+
supplied_root = Path(root).expanduser().absolute()
|
|
619
|
+
repository_root = supplied_root.resolve()
|
|
620
|
+
supplied_manifest = (
|
|
621
|
+
Path(manifest_path).expanduser()
|
|
622
|
+
if manifest_path is not None
|
|
623
|
+
else Path("manifests") / "releases" / "v0.2-beta.json"
|
|
624
|
+
)
|
|
625
|
+
if supplied_manifest.is_absolute():
|
|
626
|
+
try:
|
|
627
|
+
relative_manifest = supplied_manifest.relative_to(supplied_root)
|
|
628
|
+
except ValueError:
|
|
629
|
+
try:
|
|
630
|
+
relative_manifest = supplied_manifest.relative_to(repository_root)
|
|
631
|
+
except ValueError:
|
|
632
|
+
relative_manifest = None
|
|
633
|
+
else:
|
|
634
|
+
relative_manifest = supplied_manifest
|
|
635
|
+
if relative_manifest is None:
|
|
636
|
+
selected_manifest = supplied_manifest
|
|
637
|
+
selected_manifest_error = "beta release manifest escapes the repository root"
|
|
638
|
+
else:
|
|
639
|
+
selected_manifest, selected_manifest_error = _repository_path(
|
|
640
|
+
repository_root,
|
|
641
|
+
relative_manifest.as_posix(),
|
|
642
|
+
"beta release manifest",
|
|
643
|
+
reject_symlinks=True,
|
|
644
|
+
)
|
|
645
|
+
if selected_manifest_error is not None or selected_manifest is None:
|
|
646
|
+
return BetaReadinessReport(
|
|
647
|
+
gate_id="v0.2-beta",
|
|
648
|
+
manifest_sha256="",
|
|
649
|
+
checks=(
|
|
650
|
+
_blocked(
|
|
651
|
+
"manifest.schema",
|
|
652
|
+
"Beta release manifest",
|
|
653
|
+
selected_manifest_error or "beta release manifest path is invalid",
|
|
654
|
+
),
|
|
655
|
+
),
|
|
656
|
+
)
|
|
657
|
+
try:
|
|
658
|
+
manifest_bytes = selected_manifest.read_bytes()
|
|
659
|
+
raw = json.loads(manifest_bytes)
|
|
660
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
661
|
+
return BetaReadinessReport(
|
|
662
|
+
gate_id="v0.2-beta",
|
|
663
|
+
manifest_sha256="",
|
|
664
|
+
checks=(
|
|
665
|
+
_blocked(
|
|
666
|
+
"manifest.schema",
|
|
667
|
+
"Beta release manifest",
|
|
668
|
+
f"cannot load {selected_manifest}: {exc}",
|
|
669
|
+
),
|
|
670
|
+
),
|
|
671
|
+
)
|
|
672
|
+
if not isinstance(raw, Mapping):
|
|
673
|
+
return BetaReadinessReport(
|
|
674
|
+
gate_id="v0.2-beta",
|
|
675
|
+
manifest_sha256=content_hash(raw),
|
|
676
|
+
checks=(
|
|
677
|
+
_blocked(
|
|
678
|
+
"manifest.schema",
|
|
679
|
+
"Beta release manifest",
|
|
680
|
+
"manifest root must be an object",
|
|
681
|
+
),
|
|
682
|
+
),
|
|
683
|
+
)
|
|
684
|
+
gate_id = raw.get("gate_id")
|
|
685
|
+
schema_version = raw.get("schema_version")
|
|
686
|
+
manifest_blockers: list[str] = []
|
|
687
|
+
if schema_version != 1:
|
|
688
|
+
manifest_blockers.append("schema_version must equal 1")
|
|
689
|
+
if gate_id != "v0.2-beta":
|
|
690
|
+
manifest_blockers.append("gate_id must equal v0.2-beta")
|
|
691
|
+
if manifest_blockers:
|
|
692
|
+
return BetaReadinessReport(
|
|
693
|
+
gate_id=gate_id if isinstance(gate_id, str) else "v0.2-beta",
|
|
694
|
+
manifest_sha256=content_hash(raw),
|
|
695
|
+
checks=(
|
|
696
|
+
_blocked(
|
|
697
|
+
"manifest.schema",
|
|
698
|
+
"Beta release manifest",
|
|
699
|
+
*manifest_blockers,
|
|
700
|
+
),
|
|
701
|
+
),
|
|
702
|
+
)
|
|
703
|
+
|
|
704
|
+
active_registry = registry
|
|
705
|
+
if active_registry is None:
|
|
706
|
+
source_registry = repository_root / "manifests" / "algorithms" / "registry.yaml"
|
|
707
|
+
active_registry = AlgorithmRegistry(load_registry_file(source_registry))
|
|
708
|
+
checks = (
|
|
709
|
+
_check_roster(active_registry, raw),
|
|
710
|
+
_check_l2_paths(repository_root, active_registry, raw),
|
|
711
|
+
_check_selectors(raw),
|
|
712
|
+
_check_algorithm_licenses(active_registry),
|
|
713
|
+
_check_benchmark_cards(repository_root, raw),
|
|
714
|
+
_check_result_profile_policy(repository_root, raw),
|
|
715
|
+
)
|
|
716
|
+
return BetaReadinessReport(
|
|
717
|
+
gate_id=gate_id,
|
|
718
|
+
manifest_sha256=content_hash(raw),
|
|
719
|
+
checks=checks,
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
__all__ = [
|
|
724
|
+
"BetaGateCheck",
|
|
725
|
+
"BetaReadinessReport",
|
|
726
|
+
"check_beta_readiness",
|
|
727
|
+
]
|