agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
"""ACE offline recipe delegating optimization to the pinned upstream worker."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from dataclasses import asdict, dataclass, field, fields
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from agentinputlab.artifacts import load_pinned_json_mapping
|
|
12
|
+
from agentinputlab.contracts import (
|
|
13
|
+
ExternalDependencyError,
|
|
14
|
+
InterventionContext,
|
|
15
|
+
ValidationError,
|
|
16
|
+
)
|
|
17
|
+
from agentinputlab.contracts.serialization import (
|
|
18
|
+
content_hash,
|
|
19
|
+
deep_freeze_jsonable,
|
|
20
|
+
freeze_mapping,
|
|
21
|
+
to_jsonable,
|
|
22
|
+
)
|
|
23
|
+
from agentinputlab.integrations.external_workers.ace import (
|
|
24
|
+
ACE_UPSTREAM_COMMIT,
|
|
25
|
+
ACEPlaybook,
|
|
26
|
+
ACEWorkerRequest,
|
|
27
|
+
ACEWorkerResponse,
|
|
28
|
+
PlaybookBullet,
|
|
29
|
+
)
|
|
30
|
+
from agentinputlab.integrations.external_workers.base import WorkerClient
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True, slots=True)
|
|
34
|
+
class ACEExample:
|
|
35
|
+
example_id: str
|
|
36
|
+
input: Any
|
|
37
|
+
expected_output: Any = None
|
|
38
|
+
split: str = ""
|
|
39
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
40
|
+
|
|
41
|
+
def __post_init__(self) -> None:
|
|
42
|
+
if not self.example_id.strip():
|
|
43
|
+
raise ValidationError("ACE examples require a stable example_id")
|
|
44
|
+
if self.split not in {"train", "validation", "test"}:
|
|
45
|
+
raise ValidationError("ACE example split must be train/validation/test")
|
|
46
|
+
try:
|
|
47
|
+
frozen_input = deep_freeze_jsonable(self.input)
|
|
48
|
+
frozen_expected_output = deep_freeze_jsonable(self.expected_output)
|
|
49
|
+
frozen_metadata = freeze_mapping(self.metadata)
|
|
50
|
+
content_hash(frozen_input)
|
|
51
|
+
content_hash(frozen_expected_output)
|
|
52
|
+
except (TypeError, ValueError) as exc:
|
|
53
|
+
raise ValidationError("ACE example values must be canonically serializable") from exc
|
|
54
|
+
object.__setattr__(self, "input", frozen_input)
|
|
55
|
+
object.__setattr__(self, "expected_output", frozen_expected_output)
|
|
56
|
+
object.__setattr__(self, "metadata", frozen_metadata)
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def input_hash(self) -> str:
|
|
60
|
+
return content_hash(self.input)
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def from_value(cls, value: Any, *, split: str) -> ACEExample:
|
|
64
|
+
if isinstance(value, cls):
|
|
65
|
+
if value.split != split:
|
|
66
|
+
raise ValidationError(
|
|
67
|
+
f"ACE example {value.example_id!r} is labeled {value.split}, expected {split}"
|
|
68
|
+
)
|
|
69
|
+
return value
|
|
70
|
+
if not isinstance(value, Mapping):
|
|
71
|
+
raise ValidationError("ACE examples must be ACEExample or mappings")
|
|
72
|
+
metadata = value.get("metadata", {})
|
|
73
|
+
if not isinstance(metadata, Mapping):
|
|
74
|
+
raise ValidationError("ACE example metadata must be a mapping")
|
|
75
|
+
try:
|
|
76
|
+
return cls(
|
|
77
|
+
example_id=str(value["example_id"]),
|
|
78
|
+
input=value["input"],
|
|
79
|
+
expected_output=value.get("expected_output"),
|
|
80
|
+
split=split,
|
|
81
|
+
metadata=dict(metadata),
|
|
82
|
+
)
|
|
83
|
+
except KeyError as exc:
|
|
84
|
+
raise ValidationError("ACE example lacks example_id or input") from exc
|
|
85
|
+
|
|
86
|
+
def as_dict(self) -> dict[str, Any]:
|
|
87
|
+
return to_jsonable(self)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True, slots=True)
|
|
91
|
+
class SplitAudit:
|
|
92
|
+
train_ids: tuple[str, ...]
|
|
93
|
+
validation_ids: tuple[str, ...]
|
|
94
|
+
test_ids: tuple[str, ...]
|
|
95
|
+
train_input_hashes: tuple[str, ...]
|
|
96
|
+
validation_input_hashes: tuple[str, ...]
|
|
97
|
+
test_input_hashes: tuple[str, ...]
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def audit_hash(self) -> str:
|
|
101
|
+
return content_hash(self)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def validate_split_isolation(
|
|
105
|
+
train: Sequence[ACEExample],
|
|
106
|
+
validation: Sequence[ACEExample],
|
|
107
|
+
test: Sequence[ACEExample] = (),
|
|
108
|
+
) -> SplitAudit:
|
|
109
|
+
"""Fail on ID or exact-input overlap between any ACE data split."""
|
|
110
|
+
|
|
111
|
+
groups = {
|
|
112
|
+
"train": tuple(train),
|
|
113
|
+
"validation": tuple(validation),
|
|
114
|
+
"test": tuple(test),
|
|
115
|
+
}
|
|
116
|
+
for split, examples in groups.items():
|
|
117
|
+
if any(example.split != split for example in examples):
|
|
118
|
+
raise ValidationError(f"ACE {split} collection contains a mislabeled example")
|
|
119
|
+
ids = [example.example_id for example in examples]
|
|
120
|
+
hashes = [example.input_hash for example in examples]
|
|
121
|
+
if len(ids) != len(set(ids)):
|
|
122
|
+
raise ValidationError(f"ACE {split} split contains duplicate IDs")
|
|
123
|
+
if len(hashes) != len(set(hashes)):
|
|
124
|
+
raise ValidationError(f"ACE {split} split contains duplicate inputs")
|
|
125
|
+
names = tuple(groups)
|
|
126
|
+
for left_index, left in enumerate(names):
|
|
127
|
+
for right in names[left_index + 1 :]:
|
|
128
|
+
left_ids = {item.example_id for item in groups[left]}
|
|
129
|
+
right_ids = {item.example_id for item in groups[right]}
|
|
130
|
+
if left_ids & right_ids:
|
|
131
|
+
raise ValidationError(f"ACE {left}/{right} example IDs overlap")
|
|
132
|
+
left_hashes = {item.input_hash for item in groups[left]}
|
|
133
|
+
right_hashes = {item.input_hash for item in groups[right]}
|
|
134
|
+
if left_hashes & right_hashes:
|
|
135
|
+
raise ValidationError(f"ACE {left}/{right} inputs overlap")
|
|
136
|
+
return SplitAudit(
|
|
137
|
+
train_ids=tuple(item.example_id for item in groups["train"]),
|
|
138
|
+
validation_ids=tuple(item.example_id for item in groups["validation"]),
|
|
139
|
+
test_ids=tuple(item.example_id for item in groups["test"]),
|
|
140
|
+
train_input_hashes=tuple(item.input_hash for item in groups["train"]),
|
|
141
|
+
validation_input_hashes=tuple(item.input_hash for item in groups["validation"]),
|
|
142
|
+
test_input_hashes=tuple(item.input_hash for item in groups["test"]),
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass(frozen=True, slots=True)
|
|
147
|
+
class ACEControls:
|
|
148
|
+
max_rounds: int = 4
|
|
149
|
+
max_model_calls: int = 100
|
|
150
|
+
max_tokens: int = 200_000
|
|
151
|
+
max_cost_usd: float = 100.0
|
|
152
|
+
seed: int = 0
|
|
153
|
+
|
|
154
|
+
def __post_init__(self) -> None:
|
|
155
|
+
for name in ("max_rounds", "max_model_calls", "max_tokens"):
|
|
156
|
+
if type(getattr(self, name)) is not int or getattr(self, name) < 1:
|
|
157
|
+
raise ValidationError(f"ACE {name} must be a positive integer")
|
|
158
|
+
if type(self.seed) is not int or self.seed < 0:
|
|
159
|
+
raise ValidationError("ACE seed must be a non-negative integer")
|
|
160
|
+
if (
|
|
161
|
+
isinstance(self.max_cost_usd, bool)
|
|
162
|
+
or not isinstance(self.max_cost_usd, (int, float))
|
|
163
|
+
or (isinstance(self.max_cost_usd, float) and not math.isfinite(self.max_cost_usd))
|
|
164
|
+
or self.max_cost_usd < 0
|
|
165
|
+
):
|
|
166
|
+
raise ValidationError("ACE max_cost_usd must be finite and non-negative")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@dataclass(frozen=True, slots=True)
|
|
170
|
+
class ACEContextArtifact:
|
|
171
|
+
artifact_id: str
|
|
172
|
+
release_id: str
|
|
173
|
+
playbook: ACEPlaybook
|
|
174
|
+
selected_iteration: int
|
|
175
|
+
split_audit_hash: str
|
|
176
|
+
upstream_commit: str
|
|
177
|
+
worker_fingerprint: Mapping[str, Any]
|
|
178
|
+
model_fingerprints: Mapping[str, Mapping[str, Any]]
|
|
179
|
+
controls: Mapping[str, Any]
|
|
180
|
+
usage: Mapping[str, int | float]
|
|
181
|
+
iteration_hashes: tuple[str, ...]
|
|
182
|
+
|
|
183
|
+
def __post_init__(self) -> None:
|
|
184
|
+
if self.release_id != "recipe.ace.offline":
|
|
185
|
+
raise ValidationError("ACE artifact release identity mismatch")
|
|
186
|
+
if self.upstream_commit != ACE_UPSTREAM_COMMIT:
|
|
187
|
+
raise ValidationError("ACE artifact upstream revision mismatch")
|
|
188
|
+
if not isinstance(self.playbook, ACEPlaybook):
|
|
189
|
+
raise ValidationError("ACE artifact playbook must be an ACEPlaybook")
|
|
190
|
+
if type(self.selected_iteration) is not int or self.selected_iteration < 0:
|
|
191
|
+
raise ValidationError("ACE selected_iteration must be a non-negative integer")
|
|
192
|
+
if any(
|
|
193
|
+
not isinstance(getattr(self, name), Mapping)
|
|
194
|
+
for name in (
|
|
195
|
+
"worker_fingerprint",
|
|
196
|
+
"model_fingerprints",
|
|
197
|
+
"controls",
|
|
198
|
+
"usage",
|
|
199
|
+
)
|
|
200
|
+
) or any(not isinstance(value, Mapping) for value in self.model_fingerprints.values()):
|
|
201
|
+
raise ValidationError("ACE artifact fingerprints, controls, and usage must be mappings")
|
|
202
|
+
if not isinstance(self.iteration_hashes, (list, tuple)) or any(
|
|
203
|
+
not isinstance(value, str) for value in self.iteration_hashes
|
|
204
|
+
):
|
|
205
|
+
raise ValidationError("ACE iteration_hashes must be an array of strings")
|
|
206
|
+
object.__setattr__(
|
|
207
|
+
self,
|
|
208
|
+
"playbook",
|
|
209
|
+
ACEPlaybook(
|
|
210
|
+
tuple(
|
|
211
|
+
PlaybookBullet(
|
|
212
|
+
b.bullet_id,
|
|
213
|
+
b.content,
|
|
214
|
+
b.helpful,
|
|
215
|
+
b.harmful,
|
|
216
|
+
metadata=freeze_mapping(b.metadata),
|
|
217
|
+
)
|
|
218
|
+
for b in self.playbook.bullets
|
|
219
|
+
)
|
|
220
|
+
),
|
|
221
|
+
)
|
|
222
|
+
for name in ("worker_fingerprint", "model_fingerprints", "controls", "usage"):
|
|
223
|
+
object.__setattr__(self, name, freeze_mapping(getattr(self, name)))
|
|
224
|
+
object.__setattr__(self, "iteration_hashes", tuple(self.iteration_hashes))
|
|
225
|
+
if self.artifact_id != content_hash(self.identity_payload()):
|
|
226
|
+
raise ValidationError("ACE artifact content identity mismatch")
|
|
227
|
+
|
|
228
|
+
def identity_payload(self) -> dict[str, Any]:
|
|
229
|
+
"""Original fit identity; a pinned file checksum separately protects usage."""
|
|
230
|
+
return {
|
|
231
|
+
"release_id": self.release_id,
|
|
232
|
+
"playbook_hash": self.playbook.playbook_hash,
|
|
233
|
+
"selected_iteration": self.selected_iteration,
|
|
234
|
+
"split_audit_hash": self.split_audit_hash,
|
|
235
|
+
"upstream_commit": self.upstream_commit,
|
|
236
|
+
"worker_fingerprint": self.worker_fingerprint,
|
|
237
|
+
"model_fingerprints": self.model_fingerprints,
|
|
238
|
+
"controls": self.controls,
|
|
239
|
+
"iteration_hashes": self.iteration_hashes,
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
@classmethod
|
|
243
|
+
def from_mapping(cls, value: Mapping[str, Any]) -> ACEContextArtifact:
|
|
244
|
+
if not isinstance(value, Mapping) or set(value) != {f.name for f in fields(cls)}:
|
|
245
|
+
raise ValidationError("ACE artifact fields are invalid")
|
|
246
|
+
book = value["playbook"]
|
|
247
|
+
if (
|
|
248
|
+
not isinstance(book, Mapping)
|
|
249
|
+
or set(book) != {"bullets"}
|
|
250
|
+
or not isinstance(book["bullets"], (list, tuple))
|
|
251
|
+
):
|
|
252
|
+
raise ValidationError("ACE artifact playbook schema is invalid")
|
|
253
|
+
bullets = []
|
|
254
|
+
for item in book["bullets"]:
|
|
255
|
+
if (
|
|
256
|
+
not isinstance(item, Mapping)
|
|
257
|
+
or set(item) != {f.name for f in fields(PlaybookBullet)}
|
|
258
|
+
or any(
|
|
259
|
+
not isinstance(item.get(k), str) or not item[k].strip()
|
|
260
|
+
for k in ("bullet_id", "content")
|
|
261
|
+
)
|
|
262
|
+
or any(type(item.get(k)) is not int or item[k] < 0 for k in ("helpful", "harmful"))
|
|
263
|
+
or not isinstance(item.get("metadata"), Mapping)
|
|
264
|
+
):
|
|
265
|
+
raise ValidationError("ACE artifact bullet schema is invalid")
|
|
266
|
+
bullets.append(PlaybookBullet(**dict(item)))
|
|
267
|
+
malformed = False
|
|
268
|
+
try:
|
|
269
|
+
return cls(**{**dict(value), "playbook": ACEPlaybook(tuple(bullets))})
|
|
270
|
+
except ValidationError:
|
|
271
|
+
raise
|
|
272
|
+
except (AttributeError, TypeError, ValueError, RecursionError):
|
|
273
|
+
malformed = True
|
|
274
|
+
if malformed:
|
|
275
|
+
raise ValidationError("ACE artifact values are malformed")
|
|
276
|
+
|
|
277
|
+
@classmethod
|
|
278
|
+
def from_json(
|
|
279
|
+
cls, path: str | Path, *, expected_sha256: str, max_bytes: int = 4 * 1024 * 1024
|
|
280
|
+
) -> ACEContextArtifact:
|
|
281
|
+
return cls.from_mapping(
|
|
282
|
+
load_pinned_json_mapping(path, expected_sha256=expected_sha256, max_bytes=max_bytes)
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def as_dict(self) -> dict[str, Any]:
|
|
286
|
+
return to_jsonable(self)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@dataclass(slots=True)
|
|
290
|
+
class ACEOfflineRecipe:
|
|
291
|
+
"""Run train/validation-only ACE and return an eval-only playbook artifact."""
|
|
292
|
+
|
|
293
|
+
client: WorkerClient
|
|
294
|
+
initial_playbook: ACEPlaybook = field(default_factory=ACEPlaybook)
|
|
295
|
+
controls: ACEControls = field(default_factory=ACEControls)
|
|
296
|
+
method_id: str = "ace"
|
|
297
|
+
release_id: str = "recipe.ace.offline"
|
|
298
|
+
|
|
299
|
+
def __post_init__(self) -> None:
|
|
300
|
+
if not isinstance(self.client, WorkerClient):
|
|
301
|
+
raise ExternalDependencyError("ACE offline recipe requires a WorkerClient")
|
|
302
|
+
|
|
303
|
+
def fit(
|
|
304
|
+
self,
|
|
305
|
+
train_examples: tuple[Any, ...],
|
|
306
|
+
*,
|
|
307
|
+
validation_examples: tuple[Any, ...] = (),
|
|
308
|
+
context: InterventionContext,
|
|
309
|
+
) -> ACEContextArtifact:
|
|
310
|
+
train = tuple(ACEExample.from_value(item, split="train") for item in train_examples)
|
|
311
|
+
validation = tuple(
|
|
312
|
+
ACEExample.from_value(item, split="validation") for item in validation_examples
|
|
313
|
+
)
|
|
314
|
+
if not train or not validation:
|
|
315
|
+
raise ValidationError("ACE offline recipe requires non-empty train and validation")
|
|
316
|
+
if self.controls.seed != context.seed:
|
|
317
|
+
raise ValidationError(
|
|
318
|
+
"ACE controls.seed must equal InterventionContext.seed; "
|
|
319
|
+
"the recipe does not silently override either seed"
|
|
320
|
+
)
|
|
321
|
+
audit = validate_split_isolation(train, validation)
|
|
322
|
+
fingerprints = _model_fingerprints(context)
|
|
323
|
+
controls = asdict(self.controls)
|
|
324
|
+
serialized_train = tuple(item.as_dict() for item in train)
|
|
325
|
+
serialized_validation = tuple(item.as_dict() for item in validation)
|
|
326
|
+
request_base = {
|
|
327
|
+
"train_ids": audit.train_ids,
|
|
328
|
+
"validation_ids": audit.validation_ids,
|
|
329
|
+
"train_examples": serialized_train,
|
|
330
|
+
"validation_examples": serialized_validation,
|
|
331
|
+
"initial_playbook_hash": self.initial_playbook.playbook_hash,
|
|
332
|
+
"controls": controls,
|
|
333
|
+
"model_fingerprints": fingerprints,
|
|
334
|
+
"split_audit_hash": audit.audit_hash,
|
|
335
|
+
"upstream_commit": ACE_UPSTREAM_COMMIT,
|
|
336
|
+
}
|
|
337
|
+
request = ACEWorkerRequest(
|
|
338
|
+
request_id=content_hash(request_base),
|
|
339
|
+
train_examples=serialized_train,
|
|
340
|
+
validation_examples=serialized_validation,
|
|
341
|
+
initial_playbook=self.initial_playbook,
|
|
342
|
+
controls=controls,
|
|
343
|
+
model_fingerprints=fingerprints,
|
|
344
|
+
split_audit_hash=audit.audit_hash,
|
|
345
|
+
)
|
|
346
|
+
raw = self.client.call(request.as_dict())
|
|
347
|
+
response = ACEWorkerResponse.from_mapping(raw, request=request)
|
|
348
|
+
iteration_hashes = tuple(
|
|
349
|
+
content_hash(
|
|
350
|
+
{
|
|
351
|
+
"index": iteration.index,
|
|
352
|
+
"operations": iteration.operations,
|
|
353
|
+
"candidate_playbook_hash": iteration.candidate_playbook_hash,
|
|
354
|
+
"validation_score": iteration.validation_score,
|
|
355
|
+
}
|
|
356
|
+
)
|
|
357
|
+
for iteration in response.iterations
|
|
358
|
+
)
|
|
359
|
+
artifact_base = {
|
|
360
|
+
"release_id": self.release_id,
|
|
361
|
+
"playbook_hash": response.final_playbook.playbook_hash,
|
|
362
|
+
"selected_iteration": response.selected_iteration,
|
|
363
|
+
"split_audit_hash": audit.audit_hash,
|
|
364
|
+
"upstream_commit": response.upstream_commit,
|
|
365
|
+
"worker_fingerprint": self.client.fingerprint(),
|
|
366
|
+
"model_fingerprints": fingerprints,
|
|
367
|
+
"controls": controls,
|
|
368
|
+
"iteration_hashes": iteration_hashes,
|
|
369
|
+
}
|
|
370
|
+
return ACEContextArtifact(
|
|
371
|
+
artifact_id=content_hash(artifact_base),
|
|
372
|
+
release_id=self.release_id,
|
|
373
|
+
playbook=response.final_playbook,
|
|
374
|
+
selected_iteration=response.selected_iteration,
|
|
375
|
+
split_audit_hash=audit.audit_hash,
|
|
376
|
+
upstream_commit=response.upstream_commit,
|
|
377
|
+
worker_fingerprint=dict(self.client.fingerprint()),
|
|
378
|
+
model_fingerprints=fingerprints,
|
|
379
|
+
controls=controls,
|
|
380
|
+
usage=response.usage,
|
|
381
|
+
iteration_hashes=iteration_hashes,
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _model_fingerprints(
|
|
386
|
+
context: InterventionContext,
|
|
387
|
+
) -> Mapping[str, Mapping[str, Any]]:
|
|
388
|
+
configured = context.run_config.get("ace_model_fingerprints")
|
|
389
|
+
if configured is not None:
|
|
390
|
+
if not isinstance(configured, Mapping):
|
|
391
|
+
raise ValidationError("ace_model_fingerprints must be a mapping")
|
|
392
|
+
fingerprints: dict[str, Mapping[str, Any]] = {}
|
|
393
|
+
for role in ("target", "reflector", "curator"):
|
|
394
|
+
value = configured.get(role)
|
|
395
|
+
if not isinstance(value, Mapping):
|
|
396
|
+
raise ValidationError(f"ACE is missing the {role} model fingerprint")
|
|
397
|
+
fingerprints[role] = dict(value)
|
|
398
|
+
return fingerprints
|
|
399
|
+
if context.backend is None:
|
|
400
|
+
raise ExternalDependencyError(
|
|
401
|
+
"ACE requires explicit target/reflector/curator fingerprints or a backend"
|
|
402
|
+
)
|
|
403
|
+
try:
|
|
404
|
+
shared = dict(context.backend.fingerprint())
|
|
405
|
+
except AttributeError as exc:
|
|
406
|
+
raise ExternalDependencyError("ACE backend does not expose a fingerprint") from exc
|
|
407
|
+
return {role: dict(shared) for role in ("target", "reflector", "curator")}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def playbook_from_strings(values: Sequence[str]) -> ACEPlaybook:
|
|
411
|
+
"""Convenience constructor with stable, position-derived bullet IDs."""
|
|
412
|
+
|
|
413
|
+
return ACEPlaybook(
|
|
414
|
+
tuple(
|
|
415
|
+
PlaybookBullet(bullet_id=f"bullet-{index:04d}", content=value)
|
|
416
|
+
for index, value in enumerate(values)
|
|
417
|
+
)
|
|
418
|
+
)
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"""BFCL Python-simple AST feedback for ACE's native text generator.
|
|
2
|
+
|
|
3
|
+
The explicit task adapter asks ACE for function-call JSON in ``final_answer``;
|
|
4
|
+
it does not replace ACE's generator with a function-calling agent. Evaluation
|
|
5
|
+
uses the caller's source-verified BFCL adapter, never text exact match. Training
|
|
6
|
+
targets are opaque IDs, not the benchmark's hidden reference arguments.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
from collections.abc import Mapping, Sequence
|
|
14
|
+
|
|
15
|
+
from agentinputlab.benchmarks.base import BenchmarkExample
|
|
16
|
+
from agentinputlab.contracts import ValidationError
|
|
17
|
+
from agentinputlab.contracts.serialization import canonical_json, content_hash, to_jsonable
|
|
18
|
+
|
|
19
|
+
from .ace import ACEExample
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _unique_object(pairs):
|
|
23
|
+
value = {}
|
|
24
|
+
for key, item in pairs:
|
|
25
|
+
if key in value:
|
|
26
|
+
raise ValueError("duplicate function-call JSON key")
|
|
27
|
+
value[key] = item
|
|
28
|
+
return value
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def parse_bfcl_ace_answer(text: str) -> list[dict]:
|
|
32
|
+
"""Decode exactly one FC-JSON sequence; no fences, repair or code execution."""
|
|
33
|
+
if not isinstance(text, str):
|
|
34
|
+
raise ValueError("ACE BFCL final_answer must be a JSON string")
|
|
35
|
+
value = json.loads(
|
|
36
|
+
text,
|
|
37
|
+
object_pairs_hook=_unique_object,
|
|
38
|
+
parse_constant=lambda x: (_ for _ in ()).throw(ValueError("nonfinite JSON")),
|
|
39
|
+
)
|
|
40
|
+
canonical_json(value) # Reject exponent overflow as well as explicit NaN/Infinity.
|
|
41
|
+
if not isinstance(value, list) or not value:
|
|
42
|
+
raise ValueError("ACE BFCL answer must be a nonempty function-call sequence")
|
|
43
|
+
for call in value:
|
|
44
|
+
if not isinstance(call, dict) or len(call) != 1:
|
|
45
|
+
raise ValueError("each BFCL call must contain one function name")
|
|
46
|
+
name, arguments = next(iter(call.items()))
|
|
47
|
+
if not isinstance(name, str) or not name.strip() or not isinstance(arguments, dict):
|
|
48
|
+
raise ValueError("BFCL function name and argument object required")
|
|
49
|
+
return value
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ACEBFCLFeedback:
|
|
53
|
+
"""Bind disjoint caller-declared groups to the native Python AST checker.
|
|
54
|
+
|
|
55
|
+
This is a task-specific text-to-FC adapter, not a claim of native BFCL model
|
|
56
|
+
handler prompt parity or a standard leaderboard training split.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
processor_id = "agentinputlab.ace.BFCL_AST_feedback.v1"
|
|
60
|
+
|
|
61
|
+
def __init__(
|
|
62
|
+
self,
|
|
63
|
+
*,
|
|
64
|
+
adapter,
|
|
65
|
+
train: Sequence[BenchmarkExample],
|
|
66
|
+
validation: Sequence[BenchmarkExample],
|
|
67
|
+
partition_label: str = "caller_declared_compatibility_partitions",
|
|
68
|
+
):
|
|
69
|
+
self.train, self.validation = tuple(train), tuple(validation)
|
|
70
|
+
self._initialize(
|
|
71
|
+
adapter=adapter,
|
|
72
|
+
groups=(("train", self.train), ("validation", self.validation)),
|
|
73
|
+
partition_label=partition_label,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def _initialize(self, *, adapter, groups, partition_label):
|
|
77
|
+
if getattr(adapter, "benchmark_id", None) != "bfcl.v3.simple_python":
|
|
78
|
+
raise ValidationError("ACE BFCL requires the Python-simple adapter")
|
|
79
|
+
if (
|
|
80
|
+
not callable(getattr(adapter, "verify_integrity", None))
|
|
81
|
+
or not callable(getattr(adapter, "score", None))
|
|
82
|
+
or not callable(getattr(adapter, "function_tools", None))
|
|
83
|
+
):
|
|
84
|
+
raise ValidationError("ACE BFCL requires integrity-checked native scoring")
|
|
85
|
+
if not isinstance(partition_label, str) or not partition_label.strip():
|
|
86
|
+
raise ValidationError("ACE BFCL partition label must be explicit")
|
|
87
|
+
self.adapter = adapter
|
|
88
|
+
self._groups = tuple(groups)
|
|
89
|
+
cases = tuple(e for _, group in self._groups for e in group)
|
|
90
|
+
if any(not group for _, group in self._groups) or any(
|
|
91
|
+
not isinstance(e, BenchmarkExample) for e in cases
|
|
92
|
+
):
|
|
93
|
+
raise ValidationError("ACE BFCL requires nonempty typed fit groups")
|
|
94
|
+
if len({e.example_id for e in cases}) != len(cases):
|
|
95
|
+
raise ValidationError("ACE BFCL fit IDs must be disjoint")
|
|
96
|
+
for e in cases:
|
|
97
|
+
if not isinstance(e.visible_metadata.get("function"), Mapping):
|
|
98
|
+
raise ValidationError("ACE BFCL example lacks its visible function catalog")
|
|
99
|
+
if len({content_hash(self._visible(e)) for e in cases}) != len(cases):
|
|
100
|
+
raise ValidationError("ACE BFCL fit inputs must be disjoint")
|
|
101
|
+
self.partition_label = partition_label
|
|
102
|
+
self._examples = {e.example_id: e for e in cases}
|
|
103
|
+
self._binding = self._native_binding()
|
|
104
|
+
self.events = []
|
|
105
|
+
|
|
106
|
+
def _visible(self, example):
|
|
107
|
+
state = example.visible_state()
|
|
108
|
+
return dict(
|
|
109
|
+
query=state.query,
|
|
110
|
+
instruction=state.instruction,
|
|
111
|
+
messages=state.messages,
|
|
112
|
+
context=state.context,
|
|
113
|
+
tool_names=state.tool_names,
|
|
114
|
+
function=example.visible_metadata.get("function"),
|
|
115
|
+
native_function_tools=self.adapter.function_tools(example),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def _native_binding(self):
|
|
119
|
+
self.adapter.verify_integrity()
|
|
120
|
+
return to_jsonable(
|
|
121
|
+
dict(
|
|
122
|
+
prepared_files=dict(self.adapter.prepared.file_sha256),
|
|
123
|
+
evaluation_provenance=self.adapter.evaluation_provenance,
|
|
124
|
+
)
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
def fit_examples(self, split):
|
|
128
|
+
if split not in {"train", "validation"}:
|
|
129
|
+
raise ValidationError("ACE BFCL exposes only train and validation groups")
|
|
130
|
+
cases = self.train if split == "train" else self.validation
|
|
131
|
+
return self._examples_for(cases, split)
|
|
132
|
+
|
|
133
|
+
def _examples_for(self, cases, split):
|
|
134
|
+
return tuple(
|
|
135
|
+
ACEExample(
|
|
136
|
+
example_id=e.example_id,
|
|
137
|
+
input=dict(
|
|
138
|
+
question=(
|
|
139
|
+
"Solve the supplied function-calling task. Preserve its task "
|
|
140
|
+
"instruction, user query and function schema. Put exactly a JSON "
|
|
141
|
+
"array of function calls in your final_answer string, using "
|
|
142
|
+
'[{"function_name": {"argument_name": "value"}}]. '
|
|
143
|
+
"Use the names in native_function_tools and their argument types, not this "
|
|
144
|
+
"illustrative name. Do not include code fences or commentary.\n"
|
|
145
|
+
+ canonical_json(self._visible(e))
|
|
146
|
+
)
|
|
147
|
+
),
|
|
148
|
+
expected_output=e.example_id,
|
|
149
|
+
split=split,
|
|
150
|
+
metadata=dict(
|
|
151
|
+
task_processor=self.processor_id,
|
|
152
|
+
source_state_hash=e.visible_state().state_hash,
|
|
153
|
+
partition_label=self.partition_label,
|
|
154
|
+
evaluator_target_is_opaque_id=True,
|
|
155
|
+
),
|
|
156
|
+
)
|
|
157
|
+
for e in cases
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
def validate_examples(self, train, validation):
|
|
161
|
+
for supplied, split in ((train, "train"), (validation, "validation")):
|
|
162
|
+
if tuple(to_jsonable(e) for e in supplied) != tuple(
|
|
163
|
+
e.as_dict() for e in self.fit_examples(split)
|
|
164
|
+
):
|
|
165
|
+
raise ValidationError("ACE BFCL examples differ from frozen fit groups")
|
|
166
|
+
|
|
167
|
+
def fingerprint(self):
|
|
168
|
+
return dict(
|
|
169
|
+
processor_id=self.processor_id,
|
|
170
|
+
native_binding=self._binding,
|
|
171
|
+
metric="ast_accuracy",
|
|
172
|
+
partition_label=self.partition_label,
|
|
173
|
+
generator_output_adapter="native_ACE_text_final_answer_to_BFCL_FC_JSON",
|
|
174
|
+
native_BFCL_handler_prompt_parity_claimed=False,
|
|
175
|
+
training_success_criterion="native_ast_accuracy_strictly_greater_than_zero",
|
|
176
|
+
validation_selection="mean_native_ast_accuracy",
|
|
177
|
+
inputs={
|
|
178
|
+
split: [
|
|
179
|
+
dict(
|
|
180
|
+
case_id=e.example_id,
|
|
181
|
+
visible_hash=content_hash(self._visible(e)),
|
|
182
|
+
evaluator_gold_sha256=content_hash(e.evaluator_gold),
|
|
183
|
+
)
|
|
184
|
+
for e in cases
|
|
185
|
+
]
|
|
186
|
+
for split, cases in self._groups
|
|
187
|
+
},
|
|
188
|
+
automatic_retry=False,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
def score_answer(self, predicted, target):
|
|
192
|
+
if target not in self._examples:
|
|
193
|
+
raise ValidationError("ACE BFCL target is outside frozen fit groups")
|
|
194
|
+
if self._native_binding() != self._binding:
|
|
195
|
+
raise ValidationError("ACE BFCL source/scorer fingerprint changed")
|
|
196
|
+
try:
|
|
197
|
+
calls = parse_bfcl_ace_answer(predicted)
|
|
198
|
+
except (ValueError, TypeError):
|
|
199
|
+
metrics, status = {"ast_accuracy": 0.0}, "invalid_FC_JSON_training_feedback"
|
|
200
|
+
else:
|
|
201
|
+
# The pinned OpenAI FC decoder calls json.loads on each argument
|
|
202
|
+
# string. Preserve argument types while crossing that exact seam.
|
|
203
|
+
native_calls = [
|
|
204
|
+
{name: canonical_json(arguments)}
|
|
205
|
+
for call in calls
|
|
206
|
+
for name, arguments in call.items()
|
|
207
|
+
]
|
|
208
|
+
metrics = dict(self.adapter.score(self._examples[target], native_calls))
|
|
209
|
+
status = "native_AST_scored"
|
|
210
|
+
value = metrics.get("ast_accuracy")
|
|
211
|
+
if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1:
|
|
212
|
+
raise ValidationError("ACE BFCL native ast_accuracy is absent or invalid")
|
|
213
|
+
self.events.append(
|
|
214
|
+
dict(
|
|
215
|
+
target=target,
|
|
216
|
+
prediction_sha256=content_hash(predicted),
|
|
217
|
+
metrics=metrics,
|
|
218
|
+
status=status,
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
return float(value)
|
|
222
|
+
|
|
223
|
+
def answer_is_correct(self, predicted, ground_truth):
|
|
224
|
+
return self.score_answer(predicted, ground_truth) > 0
|
|
225
|
+
|
|
226
|
+
def evaluate_accuracy(self, predictions, ground_truths):
|
|
227
|
+
if len(predictions) != len(ground_truths):
|
|
228
|
+
raise ValidationError("ACE BFCL predictions and targets differ in length")
|
|
229
|
+
return (
|
|
230
|
+
sum(self.score_answer(p, t) for p, t in zip(predictions, ground_truths, strict=True))
|
|
231
|
+
/ len(predictions)
|
|
232
|
+
if predictions
|
|
233
|
+
else 0.0
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class ACEOnlineBFCLFeedback(ACEBFCLFeedback):
|
|
238
|
+
"""AST feedback for an explicitly transductive native ACE online stream.
|
|
239
|
+
|
|
240
|
+
Stream order is frozen before generation. This object is not an offline
|
|
241
|
+
training split and its adapted artifact is not independent held-out evidence.
|
|
242
|
+
"""
|
|
243
|
+
|
|
244
|
+
processor_id = "agentinputlab.ace.BFCL_AST_online_feedback.v1"
|
|
245
|
+
|
|
246
|
+
def __init__(self, *, adapter, stream: Sequence[BenchmarkExample]):
|
|
247
|
+
self.stream = tuple(stream)
|
|
248
|
+
self._initialize(
|
|
249
|
+
adapter=adapter,
|
|
250
|
+
groups=(("stream", self.stream),),
|
|
251
|
+
partition_label="explicit_native_BFCL_feedback_transductive_online_stream",
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
def stream_examples(self):
|
|
255
|
+
return self._examples_for(self.stream, "test")
|
|
256
|
+
|
|
257
|
+
def validate_online_examples(self, examples):
|
|
258
|
+
supplied = tuple(ACEExample.from_value(row, split="test").as_dict() for row in examples)
|
|
259
|
+
if supplied != tuple(row.as_dict() for row in self.stream_examples()):
|
|
260
|
+
raise ValidationError("ACE online BFCL inputs/order differ from frozen stream")
|
|
261
|
+
|
|
262
|
+
def fingerprint(self):
|
|
263
|
+
return dict(
|
|
264
|
+
super().fingerprint(),
|
|
265
|
+
transductive_feedback_not_offline_or_heldout=True,
|
|
266
|
+
stream_order_preserved=True,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
def fit_examples(self, split):
|
|
270
|
+
raise ValidationError("online BFCL feedback cannot be used as offline fit groups")
|
|
271
|
+
|
|
272
|
+
def validate_examples(self, train, validation):
|
|
273
|
+
raise ValidationError("online BFCL feedback cannot validate offline fit groups")
|