agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Optional integrations outside the default runtime-core installation."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Post-v1 tool/schema and executable action-space intervention adapters."""
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Typed bridges to pinned upstream runtimes in isolated environments."""
|
|
2
|
+
|
|
3
|
+
from .ace import (
|
|
4
|
+
ACE_UPSTREAM_COMMIT,
|
|
5
|
+
ACEDiffOperation,
|
|
6
|
+
ACEPlaybook,
|
|
7
|
+
ACEWorkerRequest,
|
|
8
|
+
ACEWorkerResponse,
|
|
9
|
+
PlaybookBullet,
|
|
10
|
+
apply_playbook_diff,
|
|
11
|
+
)
|
|
12
|
+
from .base import JsonLineSubprocessWorker, WorkerClient
|
|
13
|
+
from .decomp_live import DecompLiveRuntime
|
|
14
|
+
from .decomp_planner import DecompPlannerLiveRuntime
|
|
15
|
+
from .ircot import (
|
|
16
|
+
IRCOT_UPSTREAM_COMMIT,
|
|
17
|
+
IRCoTControlledWorker,
|
|
18
|
+
IRCoTControls,
|
|
19
|
+
IRCoTWorkerRequest,
|
|
20
|
+
IRCoTWorkerResponse,
|
|
21
|
+
)
|
|
22
|
+
from .llmcompiler_live import LLMCompilerLiveWorker
|
|
23
|
+
from .method import (
|
|
24
|
+
MethodWorkerRequest,
|
|
25
|
+
MethodWorkerResponse,
|
|
26
|
+
call_worker,
|
|
27
|
+
dependency_fingerprint,
|
|
28
|
+
resolve_worker,
|
|
29
|
+
)
|
|
30
|
+
from .process import ProcessResourceObservation
|
|
31
|
+
from .supervisor import (
|
|
32
|
+
AcceleratorSampler,
|
|
33
|
+
DefaultProcessRunner,
|
|
34
|
+
DockerWorkerSupervisor,
|
|
35
|
+
LocalSubprocessSupervisor,
|
|
36
|
+
ProcessRunner,
|
|
37
|
+
SupervisedProcessResult,
|
|
38
|
+
SupervisedWorkerError,
|
|
39
|
+
WorkerSupervisor,
|
|
40
|
+
worker_supervisor_from_environment,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"ACE_UPSTREAM_COMMIT",
|
|
45
|
+
"IRCOT_UPSTREAM_COMMIT",
|
|
46
|
+
"ACEDiffOperation",
|
|
47
|
+
"ACEPlaybook",
|
|
48
|
+
"ACEWorkerRequest",
|
|
49
|
+
"ACEWorkerResponse",
|
|
50
|
+
"AcceleratorSampler",
|
|
51
|
+
"DecompLiveRuntime",
|
|
52
|
+
"DecompPlannerLiveRuntime",
|
|
53
|
+
"DefaultProcessRunner",
|
|
54
|
+
"DockerWorkerSupervisor",
|
|
55
|
+
"IRCoTControlledWorker",
|
|
56
|
+
"IRCoTControls",
|
|
57
|
+
"IRCoTWorkerRequest",
|
|
58
|
+
"IRCoTWorkerResponse",
|
|
59
|
+
"JsonLineSubprocessWorker",
|
|
60
|
+
"LLMCompilerLiveWorker",
|
|
61
|
+
"LocalSubprocessSupervisor",
|
|
62
|
+
"MethodWorkerRequest",
|
|
63
|
+
"MethodWorkerResponse",
|
|
64
|
+
"PlaybookBullet",
|
|
65
|
+
"ProcessResourceObservation",
|
|
66
|
+
"ProcessRunner",
|
|
67
|
+
"SupervisedProcessResult",
|
|
68
|
+
"SupervisedWorkerError",
|
|
69
|
+
"WorkerClient",
|
|
70
|
+
"WorkerSupervisor",
|
|
71
|
+
"apply_playbook_diff",
|
|
72
|
+
"call_worker",
|
|
73
|
+
"dependency_fingerprint",
|
|
74
|
+
"resolve_worker",
|
|
75
|
+
"worker_supervisor_from_environment",
|
|
76
|
+
]
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Schema and validators for the pinned ACE offline worker."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from math import isfinite
|
|
8
|
+
from types import MappingProxyType
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from agentinputlab.contracts.errors import ExternalDependencyError, ValidationError
|
|
12
|
+
from agentinputlab.contracts.serialization import content_hash, to_jsonable
|
|
13
|
+
|
|
14
|
+
ACE_UPSTREAM_COMMIT = "bcb7cea0504afad6f55fec4845dd4864c9f9eee7"
|
|
15
|
+
_OPERATIONS = {"add", "edit", "delete", "merge"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, slots=True)
|
|
19
|
+
class PlaybookBullet:
|
|
20
|
+
bullet_id: str
|
|
21
|
+
content: str
|
|
22
|
+
helpful: int = 0
|
|
23
|
+
harmful: int = 0
|
|
24
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
25
|
+
|
|
26
|
+
def __post_init__(self) -> None:
|
|
27
|
+
if not self.bullet_id.strip() or not self.content.strip():
|
|
28
|
+
raise ValidationError("ACE playbook bullets require ID and content")
|
|
29
|
+
if self.helpful < 0 or self.harmful < 0:
|
|
30
|
+
raise ValidationError("ACE helpful/harmful counters must be non-negative")
|
|
31
|
+
object.__setattr__(self, "metadata", _frozen_mapping(self.metadata))
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def from_mapping(cls, value: Mapping[str, Any]) -> PlaybookBullet:
|
|
35
|
+
metadata = value.get("metadata", {})
|
|
36
|
+
if not isinstance(metadata, Mapping):
|
|
37
|
+
raise ValidationError("ACE bullet metadata must be a mapping")
|
|
38
|
+
try:
|
|
39
|
+
return cls(
|
|
40
|
+
bullet_id=str(value["bullet_id"]),
|
|
41
|
+
content=str(value["content"]),
|
|
42
|
+
helpful=int(value.get("helpful", 0)),
|
|
43
|
+
harmful=int(value.get("harmful", 0)),
|
|
44
|
+
metadata=dict(metadata),
|
|
45
|
+
)
|
|
46
|
+
except KeyError as exc:
|
|
47
|
+
raise ValidationError("ACE bullet lacks a required field") from exc
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True, slots=True)
|
|
51
|
+
class ACEPlaybook:
|
|
52
|
+
bullets: tuple[PlaybookBullet, ...] = ()
|
|
53
|
+
|
|
54
|
+
def __post_init__(self) -> None:
|
|
55
|
+
object.__setattr__(self, "bullets", tuple(self.bullets))
|
|
56
|
+
ids = [bullet.bullet_id for bullet in self.bullets]
|
|
57
|
+
if len(ids) != len(set(ids)):
|
|
58
|
+
raise ValidationError("ACE playbook bullet IDs must be unique")
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def playbook_hash(self) -> str:
|
|
62
|
+
return content_hash(self)
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def from_sequence(cls, value: Sequence[Mapping[str, Any]]) -> ACEPlaybook:
|
|
66
|
+
return cls(tuple(PlaybookBullet.from_mapping(item) for item in value))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True, slots=True)
|
|
70
|
+
class ACEDiffOperation:
|
|
71
|
+
operation: str
|
|
72
|
+
bullet_id: str
|
|
73
|
+
content: str = ""
|
|
74
|
+
source_ids: tuple[str, ...] = ()
|
|
75
|
+
helpful: int | None = None
|
|
76
|
+
harmful: int | None = None
|
|
77
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
78
|
+
|
|
79
|
+
def __post_init__(self) -> None:
|
|
80
|
+
if self.operation not in _OPERATIONS:
|
|
81
|
+
raise ValidationError(f"unsupported ACE curator operation: {self.operation}")
|
|
82
|
+
if not self.bullet_id.strip():
|
|
83
|
+
raise ValidationError("ACE curator operation requires bullet_id")
|
|
84
|
+
object.__setattr__(self, "source_ids", tuple(self.source_ids))
|
|
85
|
+
if len(self.source_ids) != len(set(self.source_ids)):
|
|
86
|
+
raise ValidationError("ACE merge source_ids must be unique")
|
|
87
|
+
for counter in (self.helpful, self.harmful):
|
|
88
|
+
if counter is not None and counter < 0:
|
|
89
|
+
raise ValidationError("ACE helpful/harmful counters must be non-negative")
|
|
90
|
+
object.__setattr__(self, "metadata", _frozen_mapping(self.metadata))
|
|
91
|
+
|
|
92
|
+
@classmethod
|
|
93
|
+
def from_mapping(cls, value: Mapping[str, Any]) -> ACEDiffOperation:
|
|
94
|
+
sources = value.get("source_ids", ())
|
|
95
|
+
if not isinstance(sources, Sequence) or isinstance(sources, (str, bytes)):
|
|
96
|
+
raise ValidationError("ACE source_ids must be a sequence")
|
|
97
|
+
metadata = value.get("metadata", {})
|
|
98
|
+
if not isinstance(metadata, Mapping):
|
|
99
|
+
raise ValidationError("ACE diff metadata must be a mapping")
|
|
100
|
+
try:
|
|
101
|
+
return cls(
|
|
102
|
+
operation=str(value["operation"]),
|
|
103
|
+
bullet_id=str(value["bullet_id"]),
|
|
104
|
+
content=str(value.get("content", "")),
|
|
105
|
+
source_ids=tuple(str(item) for item in sources),
|
|
106
|
+
helpful=_optional_counter(value, "helpful"),
|
|
107
|
+
harmful=_optional_counter(value, "harmful"),
|
|
108
|
+
metadata=dict(metadata),
|
|
109
|
+
)
|
|
110
|
+
except KeyError as exc:
|
|
111
|
+
raise ValidationError("ACE diff lacks a required field") from exc
|
|
112
|
+
except (TypeError, ValueError) as exc:
|
|
113
|
+
raise ValidationError("ACE diff counters are malformed") from exc
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def apply_playbook_diff(
|
|
117
|
+
playbook: ACEPlaybook,
|
|
118
|
+
operations: Sequence[ACEDiffOperation],
|
|
119
|
+
) -> ACEPlaybook:
|
|
120
|
+
"""Replay curator operations exactly and fail on ambiguous diffs."""
|
|
121
|
+
|
|
122
|
+
bullets = {bullet.bullet_id: bullet for bullet in playbook.bullets}
|
|
123
|
+
order = [bullet.bullet_id for bullet in playbook.bullets]
|
|
124
|
+
for operation in operations:
|
|
125
|
+
if operation.operation == "add":
|
|
126
|
+
if operation.bullet_id in bullets:
|
|
127
|
+
raise ValidationError("ACE add targets an existing bullet")
|
|
128
|
+
bullet = _bullet_from_operation(operation)
|
|
129
|
+
bullets[bullet.bullet_id] = bullet
|
|
130
|
+
order.append(bullet.bullet_id)
|
|
131
|
+
elif operation.operation == "edit":
|
|
132
|
+
if operation.bullet_id not in bullets:
|
|
133
|
+
raise ValidationError("ACE edit targets a missing bullet")
|
|
134
|
+
old = bullets[operation.bullet_id]
|
|
135
|
+
if not operation.content.strip():
|
|
136
|
+
raise ValidationError("ACE edit requires non-empty content")
|
|
137
|
+
bullets[operation.bullet_id] = PlaybookBullet(
|
|
138
|
+
bullet_id=operation.bullet_id,
|
|
139
|
+
content=operation.content,
|
|
140
|
+
helpful=(old.helpful if operation.helpful is None else operation.helpful),
|
|
141
|
+
harmful=(old.harmful if operation.harmful is None else operation.harmful),
|
|
142
|
+
metadata={**dict(old.metadata), **dict(operation.metadata)},
|
|
143
|
+
)
|
|
144
|
+
elif operation.operation == "delete":
|
|
145
|
+
if operation.bullet_id not in bullets:
|
|
146
|
+
raise ValidationError("ACE delete targets a missing bullet")
|
|
147
|
+
del bullets[operation.bullet_id]
|
|
148
|
+
order.remove(operation.bullet_id)
|
|
149
|
+
elif operation.operation == "merge":
|
|
150
|
+
if len(operation.source_ids) < 2:
|
|
151
|
+
raise ValidationError("ACE merge requires at least two source bullets")
|
|
152
|
+
if not set(operation.source_ids) <= set(bullets):
|
|
153
|
+
raise ValidationError("ACE merge references a missing source bullet")
|
|
154
|
+
if operation.bullet_id in bullets and operation.bullet_id not in operation.source_ids:
|
|
155
|
+
raise ValidationError("ACE merge target conflicts with an unrelated bullet")
|
|
156
|
+
if not operation.content.strip():
|
|
157
|
+
raise ValidationError("ACE merge requires non-empty content")
|
|
158
|
+
insertion = min(order.index(source) for source in operation.source_ids)
|
|
159
|
+
helpful = (
|
|
160
|
+
sum(bullets[source].helpful for source in operation.source_ids)
|
|
161
|
+
if operation.helpful is None
|
|
162
|
+
else operation.helpful
|
|
163
|
+
)
|
|
164
|
+
harmful = (
|
|
165
|
+
sum(bullets[source].harmful for source in operation.source_ids)
|
|
166
|
+
if operation.harmful is None
|
|
167
|
+
else operation.harmful
|
|
168
|
+
)
|
|
169
|
+
for source in operation.source_ids:
|
|
170
|
+
del bullets[source]
|
|
171
|
+
order.remove(source)
|
|
172
|
+
bullet = PlaybookBullet(
|
|
173
|
+
bullet_id=operation.bullet_id,
|
|
174
|
+
content=operation.content,
|
|
175
|
+
helpful=helpful,
|
|
176
|
+
harmful=harmful,
|
|
177
|
+
metadata=dict(operation.metadata),
|
|
178
|
+
)
|
|
179
|
+
bullets[bullet.bullet_id] = bullet
|
|
180
|
+
order.insert(insertion, bullet.bullet_id)
|
|
181
|
+
return ACEPlaybook(tuple(bullets[bullet_id] for bullet_id in order))
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _bullet_from_operation(operation: ACEDiffOperation) -> PlaybookBullet:
|
|
185
|
+
return PlaybookBullet(
|
|
186
|
+
bullet_id=operation.bullet_id,
|
|
187
|
+
content=operation.content,
|
|
188
|
+
helpful=0 if operation.helpful is None else operation.helpful,
|
|
189
|
+
harmful=0 if operation.harmful is None else operation.harmful,
|
|
190
|
+
metadata=dict(operation.metadata),
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@dataclass(frozen=True, slots=True)
|
|
195
|
+
class ACEWorkerIteration:
|
|
196
|
+
index: int
|
|
197
|
+
operations: tuple[ACEDiffOperation, ...]
|
|
198
|
+
candidate_playbook_hash: str
|
|
199
|
+
validation_score: float
|
|
200
|
+
|
|
201
|
+
@classmethod
|
|
202
|
+
def from_mapping(cls, value: Mapping[str, Any]) -> ACEWorkerIteration:
|
|
203
|
+
raw_operations = value.get("operations", ())
|
|
204
|
+
if not isinstance(raw_operations, Sequence) or isinstance(raw_operations, (str, bytes)):
|
|
205
|
+
raise ValidationError("ACE iteration operations must be a sequence")
|
|
206
|
+
try:
|
|
207
|
+
return cls(
|
|
208
|
+
index=int(value["index"]),
|
|
209
|
+
operations=tuple(
|
|
210
|
+
ACEDiffOperation.from_mapping(_mapping(item, "ACE operation"))
|
|
211
|
+
for item in raw_operations
|
|
212
|
+
),
|
|
213
|
+
candidate_playbook_hash=str(value["candidate_playbook_hash"]),
|
|
214
|
+
validation_score=_finite_float(
|
|
215
|
+
value["validation_score"],
|
|
216
|
+
"ACE validation_score",
|
|
217
|
+
),
|
|
218
|
+
)
|
|
219
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
220
|
+
raise ValidationError("ACE iteration fields are malformed") from exc
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@dataclass(frozen=True, slots=True)
|
|
224
|
+
class ACEWorkerRequest:
|
|
225
|
+
request_id: str
|
|
226
|
+
train_examples: tuple[Mapping[str, Any], ...]
|
|
227
|
+
validation_examples: tuple[Mapping[str, Any], ...]
|
|
228
|
+
initial_playbook: ACEPlaybook
|
|
229
|
+
controls: Mapping[str, Any]
|
|
230
|
+
model_fingerprints: Mapping[str, Mapping[str, Any]]
|
|
231
|
+
split_audit_hash: str
|
|
232
|
+
upstream_commit: str = ACE_UPSTREAM_COMMIT
|
|
233
|
+
protocol_version: str = "agentinputlab.ace.offline.v1"
|
|
234
|
+
|
|
235
|
+
def __post_init__(self) -> None:
|
|
236
|
+
if not self.request_id or not self.split_audit_hash:
|
|
237
|
+
raise ValidationError("ACE worker request requires request and split-audit IDs")
|
|
238
|
+
if not self.train_examples or not self.validation_examples:
|
|
239
|
+
raise ValidationError("ACE offline worker requires train and validation examples")
|
|
240
|
+
if set(self.model_fingerprints) != {"target", "reflector", "curator"}:
|
|
241
|
+
raise ValidationError("ACE model_fingerprints must bind target, reflector, and curator")
|
|
242
|
+
if self.upstream_commit != ACE_UPSTREAM_COMMIT:
|
|
243
|
+
raise ValidationError("ACE request does not target the pinned upstream commit")
|
|
244
|
+
object.__setattr__(self, "controls", _frozen_mapping(self.controls))
|
|
245
|
+
object.__setattr__(
|
|
246
|
+
self,
|
|
247
|
+
"model_fingerprints",
|
|
248
|
+
MappingProxyType(
|
|
249
|
+
{
|
|
250
|
+
str(role): _frozen_mapping(fingerprint)
|
|
251
|
+
for role, fingerprint in self.model_fingerprints.items()
|
|
252
|
+
}
|
|
253
|
+
),
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
def as_dict(self) -> dict[str, Any]:
|
|
257
|
+
return to_jsonable(self)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
@dataclass(frozen=True, slots=True)
|
|
261
|
+
class ACEWorkerResponse:
|
|
262
|
+
request_id: str
|
|
263
|
+
final_playbook: ACEPlaybook
|
|
264
|
+
iterations: tuple[ACEWorkerIteration, ...]
|
|
265
|
+
selected_iteration: int
|
|
266
|
+
usage: Mapping[str, int | float]
|
|
267
|
+
split_audit_hash: str
|
|
268
|
+
train_ids: tuple[str, ...]
|
|
269
|
+
validation_ids: tuple[str, ...]
|
|
270
|
+
upstream_commit: str
|
|
271
|
+
protocol_version: str
|
|
272
|
+
|
|
273
|
+
@classmethod
|
|
274
|
+
def from_mapping(
|
|
275
|
+
cls,
|
|
276
|
+
value: Mapping[str, Any],
|
|
277
|
+
*,
|
|
278
|
+
request: ACEWorkerRequest,
|
|
279
|
+
) -> ACEWorkerResponse:
|
|
280
|
+
if str(value.get("status", "")) != "ok":
|
|
281
|
+
code = str(value.get("failure_code", "WORKER_FAILURE"))
|
|
282
|
+
raise ExternalDependencyError(f"ACE worker failed with {code}")
|
|
283
|
+
if str(value.get("request_id", "")) != request.request_id:
|
|
284
|
+
raise ValidationError("ACE response request_id mismatch")
|
|
285
|
+
if str(value.get("upstream_commit", "")) != request.upstream_commit:
|
|
286
|
+
raise ValidationError("ACE worker upstream revision mismatch")
|
|
287
|
+
if str(value.get("protocol_version", "")) != request.protocol_version:
|
|
288
|
+
raise ValidationError("ACE worker protocol version mismatch")
|
|
289
|
+
if str(value.get("split_audit_hash", "")) != request.split_audit_hash:
|
|
290
|
+
raise ValidationError("ACE worker split-audit hash mismatch")
|
|
291
|
+
train_ids = _response_ids(value, "train_ids")
|
|
292
|
+
validation_ids = _response_ids(value, "validation_ids")
|
|
293
|
+
expected_train_ids = _request_example_ids(request.train_examples, "train")
|
|
294
|
+
expected_validation_ids = _request_example_ids(
|
|
295
|
+
request.validation_examples,
|
|
296
|
+
"validation",
|
|
297
|
+
)
|
|
298
|
+
if train_ids != expected_train_ids:
|
|
299
|
+
raise ValidationError("ACE worker train IDs differ from the audited split")
|
|
300
|
+
if validation_ids != expected_validation_ids:
|
|
301
|
+
raise ValidationError("ACE worker validation IDs differ from the audited split")
|
|
302
|
+
raw_iterations = value.get("iterations", ())
|
|
303
|
+
if not isinstance(raw_iterations, Sequence) or isinstance(raw_iterations, (str, bytes)):
|
|
304
|
+
raise ValidationError("ACE response iterations must be a sequence")
|
|
305
|
+
iterations = tuple(
|
|
306
|
+
ACEWorkerIteration.from_mapping(_mapping(item, "ACE iteration"))
|
|
307
|
+
for item in raw_iterations
|
|
308
|
+
)
|
|
309
|
+
if not iterations:
|
|
310
|
+
raise ValidationError("ACE worker returned no optimization iterations")
|
|
311
|
+
if len(iterations) > int(request.controls.get("max_rounds", len(iterations))):
|
|
312
|
+
raise ValidationError("ACE worker exceeded max_rounds")
|
|
313
|
+
snapshots: dict[int, ACEPlaybook] = {}
|
|
314
|
+
current = request.initial_playbook
|
|
315
|
+
for expected, iteration in enumerate(iterations):
|
|
316
|
+
if iteration.index != expected:
|
|
317
|
+
raise ValidationError("ACE iteration indices must be contiguous and zero-based")
|
|
318
|
+
current = apply_playbook_diff(current, iteration.operations)
|
|
319
|
+
if current.playbook_hash != iteration.candidate_playbook_hash:
|
|
320
|
+
raise ValidationError("ACE candidate playbook hash does not match its diff")
|
|
321
|
+
snapshots[iteration.index] = current
|
|
322
|
+
try:
|
|
323
|
+
selected = int(value["selected_iteration"])
|
|
324
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
325
|
+
raise ValidationError("ACE selected_iteration is malformed") from exc
|
|
326
|
+
if selected not in snapshots:
|
|
327
|
+
raise ValidationError("ACE selected_iteration does not exist")
|
|
328
|
+
best_score = max(iteration.validation_score for iteration in iterations)
|
|
329
|
+
selected_score = iterations[selected].validation_score
|
|
330
|
+
if selected_score != best_score:
|
|
331
|
+
raise ValidationError("ACE selected iteration is not validation-best")
|
|
332
|
+
raw_playbook = value.get("final_playbook")
|
|
333
|
+
if not isinstance(raw_playbook, Sequence) or isinstance(raw_playbook, (str, bytes)):
|
|
334
|
+
raise ValidationError("ACE final_playbook must be a sequence")
|
|
335
|
+
final_playbook = ACEPlaybook.from_sequence(
|
|
336
|
+
tuple(_mapping(item, "ACE final bullet") for item in raw_playbook)
|
|
337
|
+
)
|
|
338
|
+
if final_playbook.playbook_hash != snapshots[selected].playbook_hash:
|
|
339
|
+
raise ValidationError("ACE final playbook differs from the selected snapshot")
|
|
340
|
+
usage = value.get("usage", {})
|
|
341
|
+
if not isinstance(usage, Mapping):
|
|
342
|
+
raise ValidationError("ACE usage must be a mapping")
|
|
343
|
+
numeric_usage: dict[str, int | float] = {}
|
|
344
|
+
for key, amount in usage.items():
|
|
345
|
+
if (
|
|
346
|
+
not isinstance(amount, (int, float))
|
|
347
|
+
or isinstance(amount, bool)
|
|
348
|
+
or not isfinite(float(amount))
|
|
349
|
+
or amount < 0
|
|
350
|
+
):
|
|
351
|
+
raise ValidationError("ACE usage values must be non-negative numbers")
|
|
352
|
+
numeric_usage[str(key)] = amount
|
|
353
|
+
_validate_usage_limits(numeric_usage, request.controls)
|
|
354
|
+
return cls(
|
|
355
|
+
request_id=request.request_id,
|
|
356
|
+
final_playbook=final_playbook,
|
|
357
|
+
iterations=iterations,
|
|
358
|
+
selected_iteration=selected,
|
|
359
|
+
usage=numeric_usage,
|
|
360
|
+
split_audit_hash=request.split_audit_hash,
|
|
361
|
+
train_ids=train_ids,
|
|
362
|
+
validation_ids=validation_ids,
|
|
363
|
+
upstream_commit=request.upstream_commit,
|
|
364
|
+
protocol_version=request.protocol_version,
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _validate_usage_limits(
|
|
369
|
+
usage: Mapping[str, int | float],
|
|
370
|
+
controls: Mapping[str, Any],
|
|
371
|
+
) -> None:
|
|
372
|
+
pairs = (
|
|
373
|
+
("model_calls", "max_model_calls"),
|
|
374
|
+
("tokens", "max_tokens"),
|
|
375
|
+
("cost_usd", "max_cost_usd"),
|
|
376
|
+
)
|
|
377
|
+
for usage_key, control_key in pairs:
|
|
378
|
+
if (
|
|
379
|
+
usage_key in usage
|
|
380
|
+
and control_key in controls
|
|
381
|
+
and float(usage[usage_key]) > float(controls[control_key])
|
|
382
|
+
):
|
|
383
|
+
raise ValidationError(f"ACE worker exceeded {control_key}")
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _mapping(value: Any, label: str) -> Mapping[str, Any]:
|
|
387
|
+
if not isinstance(value, Mapping):
|
|
388
|
+
raise ValidationError(f"{label} must be a mapping")
|
|
389
|
+
return value
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _frozen_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
393
|
+
return MappingProxyType(dict(value))
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _optional_counter(value: Mapping[str, Any], key: str) -> int | None:
|
|
397
|
+
raw = value.get(key)
|
|
398
|
+
return None if raw is None else int(raw)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _finite_float(value: Any, label: str) -> float:
|
|
402
|
+
try:
|
|
403
|
+
result = float(value)
|
|
404
|
+
except (TypeError, ValueError) as exc:
|
|
405
|
+
raise ValidationError(f"{label} must be numeric") from exc
|
|
406
|
+
if not isfinite(result):
|
|
407
|
+
raise ValidationError(f"{label} must be finite")
|
|
408
|
+
return result
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _response_ids(value: Mapping[str, Any], key: str) -> tuple[str, ...]:
|
|
412
|
+
raw = value.get(key)
|
|
413
|
+
if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)):
|
|
414
|
+
raise ValidationError(f"ACE response {key} must be a sequence")
|
|
415
|
+
ids = tuple(str(item) for item in raw)
|
|
416
|
+
if any(not item for item in ids) or len(ids) != len(set(ids)):
|
|
417
|
+
raise ValidationError(f"ACE response {key} contains empty or duplicate IDs")
|
|
418
|
+
return ids
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _request_example_ids(
|
|
422
|
+
examples: Sequence[Mapping[str, Any]],
|
|
423
|
+
split: str,
|
|
424
|
+
) -> tuple[str, ...]:
|
|
425
|
+
try:
|
|
426
|
+
ids = tuple(str(item["example_id"]) for item in examples)
|
|
427
|
+
except KeyError as exc:
|
|
428
|
+
raise ValidationError(f"ACE {split} request examples lack example_id") from exc
|
|
429
|
+
if any(not item for item in ids) or len(ids) != len(set(ids)):
|
|
430
|
+
raise ValidationError(f"ACE {split} request IDs are empty or duplicated")
|
|
431
|
+
return ids
|