agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Frozen ACE playbook replay for the three common-support paper arenas.
|
|
2
|
+
|
|
3
|
+
ACE learns its playbook offline through the unchanged pinned upstream training
|
|
4
|
+
loop. This controlled adapter performs no further optimization at evaluation
|
|
5
|
+
time: it appends the frozen, hash-addressed playbook as one visible context
|
|
6
|
+
block while preserving the captured request, tools, messages, and evidence.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from agentinputlab.contracts import (
|
|
14
|
+
ContextBlock,
|
|
15
|
+
ExecutionTrace,
|
|
16
|
+
FrozenAgentState,
|
|
17
|
+
InputPatch,
|
|
18
|
+
InterventionContext,
|
|
19
|
+
PatchOp,
|
|
20
|
+
PatchOperation,
|
|
21
|
+
SemanticFamily,
|
|
22
|
+
TraceEvent,
|
|
23
|
+
TracePhase,
|
|
24
|
+
ValidationError,
|
|
25
|
+
)
|
|
26
|
+
from agentinputlab.contracts.serialization import content_hash
|
|
27
|
+
|
|
28
|
+
from .ace import ACEContextArtifact
|
|
29
|
+
|
|
30
|
+
ACE_COMMON_SUPPORT_RELEASE = "recipe.ace.eval_only"
|
|
31
|
+
ACE_COMMON_SUPPORT_PLACEMENT_RULE = "append_frozen_playbook_context.v1"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(slots=True)
|
|
35
|
+
class ACEPlaybookCommonSupport:
|
|
36
|
+
"""Replay one leakage-audited ACE artifact through the unified patch seam."""
|
|
37
|
+
|
|
38
|
+
artifact: ACEContextArtifact
|
|
39
|
+
method_id: str = "ace"
|
|
40
|
+
release_id: str = ACE_COMMON_SUPPORT_RELEASE
|
|
41
|
+
family: SemanticFamily = SemanticFamily.CURATE
|
|
42
|
+
|
|
43
|
+
def __post_init__(self) -> None:
|
|
44
|
+
if not isinstance(self.artifact, ACEContextArtifact):
|
|
45
|
+
raise ValidationError("ACE common support requires an ACEContextArtifact")
|
|
46
|
+
|
|
47
|
+
def propose(self, state: FrozenAgentState, context: InterventionContext) -> InputPatch:
|
|
48
|
+
del context
|
|
49
|
+
playbook_text = _render_playbook(self.artifact)
|
|
50
|
+
applied_characters = len(playbook_text) if self.artifact.playbook.bullets else 0
|
|
51
|
+
playbook_hash = self.artifact.playbook.playbook_hash
|
|
52
|
+
model_fingerprints_hash = content_hash(self.artifact.model_fingerprints)
|
|
53
|
+
worker_fingerprint_hash = content_hash(self.artifact.worker_fingerprint)
|
|
54
|
+
block = ContextBlock(
|
|
55
|
+
text=playbook_text,
|
|
56
|
+
block_id=f"ace-playbook:{self.artifact.artifact_id[:20]}",
|
|
57
|
+
source="ACE frozen offline playbook",
|
|
58
|
+
metadata={
|
|
59
|
+
"artifact_id": self.artifact.artifact_id,
|
|
60
|
+
"playbook_hash": playbook_hash,
|
|
61
|
+
"split_audit_hash": self.artifact.split_audit_hash,
|
|
62
|
+
"upstream_commit": self.artifact.upstream_commit,
|
|
63
|
+
"placement_rule": ACE_COMMON_SUPPORT_PLACEMENT_RULE,
|
|
64
|
+
},
|
|
65
|
+
)
|
|
66
|
+
trace = ExecutionTrace(
|
|
67
|
+
method_id=self.method_id,
|
|
68
|
+
release_id=self.release_id,
|
|
69
|
+
events=(
|
|
70
|
+
TraceEvent(
|
|
71
|
+
TracePhase.PREPARE,
|
|
72
|
+
"ace_frozen_artifact_verified",
|
|
73
|
+
{
|
|
74
|
+
"artifact_id": self.artifact.artifact_id,
|
|
75
|
+
"playbook_hash": playbook_hash,
|
|
76
|
+
"split_audit_hash": self.artifact.split_audit_hash,
|
|
77
|
+
"upstream_commit": self.artifact.upstream_commit,
|
|
78
|
+
"selected_iteration": self.artifact.selected_iteration,
|
|
79
|
+
"model_fingerprints_hash": model_fingerprints_hash,
|
|
80
|
+
"worker_fingerprint_hash": worker_fingerprint_hash,
|
|
81
|
+
},
|
|
82
|
+
),
|
|
83
|
+
TraceEvent(
|
|
84
|
+
TracePhase.FINALIZE,
|
|
85
|
+
"ace_playbook_appended_to_visible_context"
|
|
86
|
+
if self.artifact.playbook.bullets
|
|
87
|
+
else "ace_native_empty_playbook_no_op",
|
|
88
|
+
{
|
|
89
|
+
"placement_rule": ACE_COMMON_SUPPORT_PLACEMENT_RULE,
|
|
90
|
+
"source_context_blocks": len(state.context),
|
|
91
|
+
"output_context_blocks": len(state.context)
|
|
92
|
+
+ bool(self.artifact.playbook.bullets),
|
|
93
|
+
"playbook_bullets": len(self.artifact.playbook.bullets),
|
|
94
|
+
"playbook_characters": applied_characters,
|
|
95
|
+
},
|
|
96
|
+
),
|
|
97
|
+
),
|
|
98
|
+
counters={
|
|
99
|
+
"model_calls": 0,
|
|
100
|
+
"retrieval_calls": 0,
|
|
101
|
+
"tool_calls": 0,
|
|
102
|
+
"input_tokens": 0,
|
|
103
|
+
"output_tokens": 0,
|
|
104
|
+
"source_context_blocks": len(state.context),
|
|
105
|
+
"output_context_blocks": len(state.context) + bool(self.artifact.playbook.bullets),
|
|
106
|
+
"playbook_bullets": len(self.artifact.playbook.bullets),
|
|
107
|
+
"playbook_characters": applied_characters,
|
|
108
|
+
"offline_training_model_calls": int(self.artifact.usage.get("model_calls", 0)),
|
|
109
|
+
"offline_training_tokens": int(self.artifact.usage.get("tokens", 0)),
|
|
110
|
+
},
|
|
111
|
+
warnings=(
|
|
112
|
+
"offline_training_cost_is_reported_on_the_artifact_not_charged_per_eval_unit",
|
|
113
|
+
"controlled_common_support_playbook_mapping_not_paper_result_parity",
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
return InputPatch(
|
|
117
|
+
method_id=self.method_id,
|
|
118
|
+
release_id=self.release_id,
|
|
119
|
+
source_state_hash=state.state_hash,
|
|
120
|
+
operations=(
|
|
121
|
+
PatchOperation(
|
|
122
|
+
PatchOp.ADD_CONTEXT,
|
|
123
|
+
block,
|
|
124
|
+
target="context",
|
|
125
|
+
rationale="Expose the frozen ACE playbook without changing captured evidence.",
|
|
126
|
+
),
|
|
127
|
+
)
|
|
128
|
+
if self.artifact.playbook.bullets
|
|
129
|
+
else (
|
|
130
|
+
PatchOperation(PatchOp.NO_OP, rationale="Native ACE selected an empty playbook."),
|
|
131
|
+
),
|
|
132
|
+
trace=trace,
|
|
133
|
+
implementation_identity=(
|
|
134
|
+
"pinned_upstream_ace_offline+" + ACE_COMMON_SUPPORT_PLACEMENT_RULE
|
|
135
|
+
),
|
|
136
|
+
profile_kind="controlled_adapter",
|
|
137
|
+
metadata={
|
|
138
|
+
"artifact_id": self.artifact.artifact_id,
|
|
139
|
+
"playbook_hash": playbook_hash,
|
|
140
|
+
"split_audit_hash": self.artifact.split_audit_hash,
|
|
141
|
+
"upstream_commit": self.artifact.upstream_commit,
|
|
142
|
+
"selected_iteration": self.artifact.selected_iteration,
|
|
143
|
+
"model_fingerprints_hash": model_fingerprints_hash,
|
|
144
|
+
"worker_fingerprint_hash": worker_fingerprint_hash,
|
|
145
|
+
"offline_training_usage": dict(self.artifact.usage),
|
|
146
|
+
"common_support_placement_rule": ACE_COMMON_SUPPORT_PLACEMENT_RULE,
|
|
147
|
+
"source_state_hash": state.state_hash,
|
|
148
|
+
"paper_result_parity_claimed": False,
|
|
149
|
+
"native_empty_playbook_no_op": not self.artifact.playbook.bullets,
|
|
150
|
+
},
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _render_playbook(artifact: ACEContextArtifact) -> str:
|
|
155
|
+
lines = [
|
|
156
|
+
"ACE LEARNED PLAYBOOK (frozen offline artifact)",
|
|
157
|
+
"Use these learned strategies when relevant; the user request and visible evidence "
|
|
158
|
+
"remain authoritative.",
|
|
159
|
+
]
|
|
160
|
+
for bullet in artifact.playbook.bullets:
|
|
161
|
+
lines.append(f"- [{bullet.bullet_id}] {bullet.content}")
|
|
162
|
+
return "\n".join(lines)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
__all__ = [
|
|
166
|
+
"ACE_COMMON_SUPPORT_PLACEMENT_RULE",
|
|
167
|
+
"ACE_COMMON_SUPPORT_RELEASE",
|
|
168
|
+
"ACEPlaybookCommonSupport",
|
|
169
|
+
]
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
"""Common contracts for ACE's explicitly noninteractive native tool-plan feedback.
|
|
2
|
+
|
|
3
|
+
Task-specific implementations invoke registered native tools and scorers. This
|
|
4
|
+
module neither executes generated Python nor supplies a replacement scorer.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import os
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import threading
|
|
17
|
+
from collections.abc import Mapping
|
|
18
|
+
from pathlib import Path, PurePosixPath
|
|
19
|
+
|
|
20
|
+
from agentinputlab.api import _state
|
|
21
|
+
from agentinputlab.artifacts import load_pinned_json_mapping
|
|
22
|
+
from agentinputlab.contracts import ExternalDependencyError, ValidationError
|
|
23
|
+
from agentinputlab.contracts.serialization import canonical_json, content_hash, to_jsonable
|
|
24
|
+
|
|
25
|
+
from .ace import ACEExample, validate_split_isolation
|
|
26
|
+
|
|
27
|
+
_NATIVE_LOCK = threading.RLock()
|
|
28
|
+
PLAN_INSTRUCTION = (
|
|
29
|
+
"Solve the supplied task as one noninteractive tool plan. Preserve all visible "
|
|
30
|
+
"task rules and function schemas. Put exactly one JSON object in your final_answer string: "
|
|
31
|
+
'{"actions":[{"name":"visible_function_name","arguments":{}}],"response":"message to user"}. '
|
|
32
|
+
"Use only visible function names with their required arguments. Actions execute in order "
|
|
33
|
+
"without another model call or access to their returned values while planning. "
|
|
34
|
+
"Do not output Python, code fences, unknown functions, or extra fields.\n"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _sha(path):
|
|
39
|
+
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _save(path, value):
|
|
43
|
+
path = Path(path)
|
|
44
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
with path.open("x", encoding="utf-8") as stream:
|
|
46
|
+
json.dump(to_jsonable(value), stream, sort_keys=True, indent=2, allow_nan=False)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _unique_object(pairs):
|
|
50
|
+
result = {}
|
|
51
|
+
for key, value in pairs:
|
|
52
|
+
if key in result:
|
|
53
|
+
raise ValueError("duplicate tool-plan JSON key")
|
|
54
|
+
result[key] = value
|
|
55
|
+
return result
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parse_native_tool_plan(text, *, tool_names, tool_cap):
|
|
59
|
+
"""Decode a complete plan with advertised names, never fences or code repair."""
|
|
60
|
+
if not isinstance(text, str) or type(tool_cap) is not int or tool_cap < 0:
|
|
61
|
+
raise ValueError("tool plan requires a string and nonnegative integer tool cap")
|
|
62
|
+
value = json.loads(
|
|
63
|
+
text,
|
|
64
|
+
object_pairs_hook=_unique_object,
|
|
65
|
+
parse_constant=lambda _: (_ for _ in ()).throw(ValueError("nonfinite JSON")),
|
|
66
|
+
)
|
|
67
|
+
canonical_json(value)
|
|
68
|
+
if not isinstance(value, dict) or set(value) != {"actions", "response"}:
|
|
69
|
+
raise ValueError("tool plan requires actions and response")
|
|
70
|
+
if (
|
|
71
|
+
not isinstance(value["actions"], list)
|
|
72
|
+
or len(value["actions"]) > tool_cap
|
|
73
|
+
or not isinstance(value["response"], str)
|
|
74
|
+
or not value["response"].strip()
|
|
75
|
+
):
|
|
76
|
+
raise ValueError("tool-plan actions/response violate the declared contract")
|
|
77
|
+
for action in value["actions"]:
|
|
78
|
+
if (
|
|
79
|
+
not isinstance(action, dict)
|
|
80
|
+
or set(action) != {"name", "arguments"}
|
|
81
|
+
or not isinstance(action["name"], str)
|
|
82
|
+
or action["name"] not in tool_names
|
|
83
|
+
or not isinstance(action["arguments"], dict)
|
|
84
|
+
):
|
|
85
|
+
raise ValueError("tool plan must use advertised tools and argument objects")
|
|
86
|
+
return value
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def verify_native_checkout(checkout, *, revision):
|
|
90
|
+
root = Path(checkout).expanduser().resolve()
|
|
91
|
+
if not root.is_dir():
|
|
92
|
+
raise ExternalDependencyError("native benchmark checkout is unavailable")
|
|
93
|
+
try:
|
|
94
|
+
head = subprocess.check_output(
|
|
95
|
+
["git", "-C", str(root), "rev-parse", "HEAD"],
|
|
96
|
+
text=True,
|
|
97
|
+
stderr=subprocess.DEVNULL,
|
|
98
|
+
).strip()
|
|
99
|
+
dirty = subprocess.check_output(
|
|
100
|
+
["git", "-C", str(root), "status", "--porcelain"],
|
|
101
|
+
text=True,
|
|
102
|
+
stderr=subprocess.DEVNULL,
|
|
103
|
+
).strip()
|
|
104
|
+
except subprocess.CalledProcessError as exc:
|
|
105
|
+
raise ExternalDependencyError("native benchmark must be a verified Git checkout") from exc
|
|
106
|
+
if head != revision or dirty:
|
|
107
|
+
raise ExternalDependencyError("native benchmark checkout revision or clean state differs")
|
|
108
|
+
return root
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@contextlib.contextmanager
|
|
112
|
+
def native_process_scope(root, *, package):
|
|
113
|
+
"""Native runtimes use process-global imports/CWD; serialize and restore them."""
|
|
114
|
+
root = Path(root).resolve()
|
|
115
|
+
with _NATIVE_LOCK:
|
|
116
|
+
for name, module in tuple(sys.modules.items()):
|
|
117
|
+
if name == package or name.startswith(package + "."):
|
|
118
|
+
path = getattr(module, "__file__", None)
|
|
119
|
+
if path and not Path(path).resolve().is_relative_to(root):
|
|
120
|
+
raise ExternalDependencyError(
|
|
121
|
+
"a different native benchmark checkout is imported"
|
|
122
|
+
)
|
|
123
|
+
previous_cwd, previous_path = Path.cwd(), list(sys.path)
|
|
124
|
+
try:
|
|
125
|
+
os.chdir(root)
|
|
126
|
+
sys.path[:0] = [str(root / "src"), str(root)]
|
|
127
|
+
yield
|
|
128
|
+
finally:
|
|
129
|
+
sys.path[:] = previous_path
|
|
130
|
+
os.chdir(previous_cwd)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def source_file_pins(root, paths):
|
|
134
|
+
root = Path(root).resolve()
|
|
135
|
+
result = {}
|
|
136
|
+
for path in paths:
|
|
137
|
+
path = Path(path).resolve()
|
|
138
|
+
if not path.is_file() or not path.is_relative_to(root):
|
|
139
|
+
raise ValidationError("native feedback source file must be inside its checkout")
|
|
140
|
+
result[path.relative_to(root).as_posix()] = _sha(path)
|
|
141
|
+
return result
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def verify_source_files(root, files):
|
|
145
|
+
if not isinstance(files, Mapping) or not files:
|
|
146
|
+
raise ValidationError("native feedback needs pinned source files")
|
|
147
|
+
for name, digest in files.items():
|
|
148
|
+
path = PurePosixPath(name)
|
|
149
|
+
if path.is_absolute() or ".." in path.parts or str(path) != name:
|
|
150
|
+
raise ValidationError("native feedback source path must be relative")
|
|
151
|
+
actual = (Path(root) / name).resolve()
|
|
152
|
+
if not actual.is_relative_to(root) or not actual.is_file() or _sha(actual) != digest:
|
|
153
|
+
raise ValidationError("native feedback source checksum changed")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class NativePlanFeedback:
|
|
157
|
+
"""Shared fit/stream validation and audited feedback, not a native agent."""
|
|
158
|
+
|
|
159
|
+
def _initialize(self, *, source_input, checkout, output_root, mode):
|
|
160
|
+
if mode not in ("offline", "online"):
|
|
161
|
+
raise ValidationError("native ACE feedback mode must be offline or online")
|
|
162
|
+
if not isinstance(source_input, Mapping) or set(source_input) != {"path", "sha256"}:
|
|
163
|
+
raise ValidationError("native ACE feedback source needs a path and trusted SHA-256")
|
|
164
|
+
self.source_input = dict(source_input)
|
|
165
|
+
self.source = load_pinned_json_mapping(
|
|
166
|
+
source_input["path"], expected_sha256=source_input["sha256"], max_bytes=16 * 1024 * 1024
|
|
167
|
+
)
|
|
168
|
+
self.mode, self.checkout = mode, Path(checkout).expanduser().resolve()
|
|
169
|
+
rows = self.source.get("fitting_rows")
|
|
170
|
+
required = {"case_id", "role", "state", "state_hash", "protected_checkpoint"}
|
|
171
|
+
if (
|
|
172
|
+
not isinstance(rows, list)
|
|
173
|
+
or not rows
|
|
174
|
+
or any(
|
|
175
|
+
not isinstance(r, dict)
|
|
176
|
+
or not required <= r.keys()
|
|
177
|
+
or not isinstance(r["case_id"], str)
|
|
178
|
+
or not isinstance(r["role"], str)
|
|
179
|
+
for r in rows
|
|
180
|
+
)
|
|
181
|
+
):
|
|
182
|
+
raise ValidationError("native ACE feedback needs nonempty frozen fitting groups")
|
|
183
|
+
self.rows = {r["case_id"]: r for r in rows}
|
|
184
|
+
if len(self.rows) != len(rows) or any(
|
|
185
|
+
not isinstance(k, str) or not k.strip() for k in self.rows
|
|
186
|
+
):
|
|
187
|
+
raise ValidationError("native ACE fitting IDs must be unique strings")
|
|
188
|
+
self.states = {key: _state(row["state"]) for key, row in self.rows.items()}
|
|
189
|
+
if (
|
|
190
|
+
set(row["role"] for row in rows) != {"train", "validation"}
|
|
191
|
+
or any(self.states[k].state_hash != r["state_hash"] for k, r in self.rows.items())
|
|
192
|
+
or len({s.state_hash for s in self.states.values()}) != len(rows)
|
|
193
|
+
or set(self.rows) & set(self.source.get("heldout_ids", ()))
|
|
194
|
+
):
|
|
195
|
+
raise ValidationError("native ACE fitting groups, states or heldout isolation differ")
|
|
196
|
+
self.root = Path(output_root).expanduser().absolute()
|
|
197
|
+
if self.root.resolve().is_relative_to(self.checkout):
|
|
198
|
+
raise ValidationError("native ACE feedback output must be outside the clean checkout")
|
|
199
|
+
if self.root.exists() or self.root.is_symlink():
|
|
200
|
+
raise ValidationError("native ACE feedback output must be create-only")
|
|
201
|
+
self.events, self.cache = [], {}
|
|
202
|
+
|
|
203
|
+
def _create_output(self):
|
|
204
|
+
validate_split_isolation(self.fit_examples("train"), self.fit_examples("validation"))
|
|
205
|
+
self.root.mkdir(parents=True, exist_ok=False, mode=0o700)
|
|
206
|
+
|
|
207
|
+
def _verify_source(self):
|
|
208
|
+
original = load_pinned_json_mapping(
|
|
209
|
+
self.source_input["path"],
|
|
210
|
+
expected_sha256=self.source_input["sha256"],
|
|
211
|
+
max_bytes=16 * 1024 * 1024,
|
|
212
|
+
)
|
|
213
|
+
if (
|
|
214
|
+
original != self.source
|
|
215
|
+
or self.rows != {r["case_id"]: r for r in original["fitting_rows"]}
|
|
216
|
+
or set(self.states) != set(self.rows)
|
|
217
|
+
or any(self.states[k].state_hash != r["state_hash"] for k, r in self.rows.items())
|
|
218
|
+
):
|
|
219
|
+
raise ValidationError("native feedback in-memory source or states changed")
|
|
220
|
+
verify_native_checkout(self.checkout, revision=self.source["native_source_revision"])
|
|
221
|
+
verify_source_files(self.checkout, self.source["source_files"])
|
|
222
|
+
|
|
223
|
+
def fit_examples(self, split):
|
|
224
|
+
if split not in ("train", "validation"):
|
|
225
|
+
raise ValidationError("native ACE fitting exposes no heldout examples")
|
|
226
|
+
return tuple(
|
|
227
|
+
self._example(key, split) for key, row in self.rows.items() if row["role"] == split
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def validate_examples(self, train, validation):
|
|
231
|
+
self.fingerprint()
|
|
232
|
+
for supplied, split in ((train, "train"), (validation, "validation")):
|
|
233
|
+
if tuple(to_jsonable(e) for e in supplied) != tuple(
|
|
234
|
+
e.as_dict() for e in self.fit_examples(split)
|
|
235
|
+
):
|
|
236
|
+
raise ValidationError("native ACE examples differ from the frozen fitting groups")
|
|
237
|
+
|
|
238
|
+
def stream_examples(self):
|
|
239
|
+
if self.mode != "online":
|
|
240
|
+
raise ValidationError("native ACE stream feedback needs explicit online mode")
|
|
241
|
+
return tuple(
|
|
242
|
+
ACEExample.from_value(
|
|
243
|
+
dict(
|
|
244
|
+
e.as_dict(),
|
|
245
|
+
metadata=dict(
|
|
246
|
+
e.as_dict()["metadata"],
|
|
247
|
+
original_operator_role=e.split,
|
|
248
|
+
source_official_split=self.source["source_official_split"],
|
|
249
|
+
online_protocol_label_NOT_official_test_split=True,
|
|
250
|
+
),
|
|
251
|
+
),
|
|
252
|
+
split="test",
|
|
253
|
+
)
|
|
254
|
+
for split in ("train", "validation")
|
|
255
|
+
for e in self.fit_examples(split)
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
def validate_online_examples(self, supplied):
|
|
259
|
+
self.fingerprint()
|
|
260
|
+
if tuple(ACEExample.from_value(e, split="test").as_dict() for e in supplied) != tuple(
|
|
261
|
+
e.as_dict() for e in self.stream_examples()
|
|
262
|
+
):
|
|
263
|
+
raise ValidationError("native ACE online order, source or feedback targets changed")
|
|
264
|
+
|
|
265
|
+
def score_answer(self, predicted, target):
|
|
266
|
+
if not isinstance(target, str) or target not in self.rows:
|
|
267
|
+
raise ValidationError("native ACE feedback target is outside the frozen fitting groups")
|
|
268
|
+
fingerprint = self.fingerprint()
|
|
269
|
+
state = self.states[target]
|
|
270
|
+
try:
|
|
271
|
+
plan = parse_native_tool_plan(
|
|
272
|
+
predicted, tool_names=self.tool_names[target], tool_cap=state.budget.max_tool_calls
|
|
273
|
+
)
|
|
274
|
+
except (ValueError, TypeError):
|
|
275
|
+
self.events.append(
|
|
276
|
+
dict(
|
|
277
|
+
target=target,
|
|
278
|
+
prediction_sha256=content_hash(predicted),
|
|
279
|
+
status="invalid_tool_plan_training_feedback",
|
|
280
|
+
score=0.0,
|
|
281
|
+
source_state_hash=state.state_hash,
|
|
282
|
+
native_task_score_NOT_claimed=True,
|
|
283
|
+
native_metrics=None,
|
|
284
|
+
new_environment_calls=0,
|
|
285
|
+
)
|
|
286
|
+
)
|
|
287
|
+
return 0.0
|
|
288
|
+
key = content_hash([target, state.state_hash, plan, fingerprint])
|
|
289
|
+
with _NATIVE_LOCK:
|
|
290
|
+
reused = key in self.cache
|
|
291
|
+
if not reused:
|
|
292
|
+
folder = self.root / "native-feedback" / key
|
|
293
|
+
folder.mkdir(parents=True, exist_ok=False)
|
|
294
|
+
self.cache[key] = self._native_score(target, plan, folder)
|
|
295
|
+
result = self.cache[key]
|
|
296
|
+
score = result["native_metrics"][self.metric]
|
|
297
|
+
if type(score) not in (int, float) or not math.isfinite(score) or not 0 <= score <= 1:
|
|
298
|
+
raise ValidationError("native ACE task feedback score is invalid")
|
|
299
|
+
self.events.append(
|
|
300
|
+
dict(
|
|
301
|
+
target=target,
|
|
302
|
+
prediction_sha256=content_hash(predicted),
|
|
303
|
+
score=score,
|
|
304
|
+
result=result,
|
|
305
|
+
source_state_hash=state.state_hash,
|
|
306
|
+
exact_feedback_reuse_NOT_new_model_draw=reused,
|
|
307
|
+
)
|
|
308
|
+
)
|
|
309
|
+
return float(score)
|
|
310
|
+
|
|
311
|
+
def answer_is_correct(self, predicted, ground_truth):
|
|
312
|
+
return self.score_answer(predicted, ground_truth) > 0
|
|
313
|
+
|
|
314
|
+
def evaluate_accuracy(self, predictions, ground_truths):
|
|
315
|
+
if len(predictions) != len(ground_truths):
|
|
316
|
+
raise ValidationError("native ACE prediction/target lengths differ")
|
|
317
|
+
return (
|
|
318
|
+
sum(self.score_answer(p, t) for p, t in zip(predictions, ground_truths, strict=True))
|
|
319
|
+
/ len(predictions)
|
|
320
|
+
if predictions
|
|
321
|
+
else 0.0
|
|
322
|
+
)
|