agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,1672 @@
|
|
|
1
|
+
"""Production provider for tau3 Banking Knowledge at the pinned revision."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import subprocess
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
from urllib.parse import urlparse
|
|
14
|
+
|
|
15
|
+
from ..protocol import (
|
|
16
|
+
OfficialArenaError,
|
|
17
|
+
ProviderArmResult,
|
|
18
|
+
content_hash,
|
|
19
|
+
intervention_usage,
|
|
20
|
+
materialized_state,
|
|
21
|
+
require_materialized_budget,
|
|
22
|
+
resolve_shared_budget_config,
|
|
23
|
+
validate_dependency_hashes,
|
|
24
|
+
visible_shared_budget,
|
|
25
|
+
)
|
|
26
|
+
from ._live_history import NativeHistorySession, validate_history_transform
|
|
27
|
+
|
|
28
|
+
_SHARED_BUDGET_STOP_MESSAGE = "I could not complete the task within the shared run budget."
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _is_replayable_initial_user_message(message: Any) -> bool:
|
|
32
|
+
"""Return whether a captured user turn is safe to replay as a checkpoint.
|
|
33
|
+
|
|
34
|
+
tau3 user messages may contain a user-side tool call together with visible
|
|
35
|
+
text. Stopping capture on that message leaves a dangling call without the
|
|
36
|
+
following ToolMessage, and environment replay then fails before the agent
|
|
37
|
+
can run. Let the orchestrator finish that tool exchange and checkpoint the
|
|
38
|
+
next ordinary visible user turn instead.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
content = getattr(message, "content", None)
|
|
42
|
+
if not content:
|
|
43
|
+
return False
|
|
44
|
+
is_tool_call = getattr(message, "is_tool_call", None)
|
|
45
|
+
return not (callable(is_tool_call) and is_tool_call())
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _remaining_agent_budget(
|
|
49
|
+
budget: Mapping[str, Any],
|
|
50
|
+
optimizer_usage: Mapping[str, int | float],
|
|
51
|
+
) -> dict[str, Any]:
|
|
52
|
+
"""Subtract optimizer consumption from every cumulative agent-side cap."""
|
|
53
|
+
|
|
54
|
+
remaining = dict(budget)
|
|
55
|
+
fields = {
|
|
56
|
+
"max_model_calls": "model_calls",
|
|
57
|
+
"max_retrieval_calls": "retrieval_calls",
|
|
58
|
+
"max_tool_calls": "tool_calls",
|
|
59
|
+
"max_output_tokens": "output_tokens",
|
|
60
|
+
"max_context_tokens": "input_tokens",
|
|
61
|
+
}
|
|
62
|
+
for budget_name, usage_name in fields.items():
|
|
63
|
+
cap = budget.get(budget_name)
|
|
64
|
+
if cap is None:
|
|
65
|
+
remaining[budget_name] = None
|
|
66
|
+
continue
|
|
67
|
+
remaining[budget_name] = max(
|
|
68
|
+
0,
|
|
69
|
+
int(cap) - int(optimizer_usage.get(usage_name, 0)),
|
|
70
|
+
)
|
|
71
|
+
return remaining
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _apply_tool_call_budget(
|
|
75
|
+
response: Any,
|
|
76
|
+
*,
|
|
77
|
+
remaining_tool_calls: int,
|
|
78
|
+
) -> tuple[Any, int, int]:
|
|
79
|
+
"""Truncate generated tool calls before tau3 can execute beyond the cap.
|
|
80
|
+
|
|
81
|
+
The returned counters are the number of calls retained for execution and
|
|
82
|
+
the number discarded by the hard budget boundary. Generated text-only
|
|
83
|
+
answers remain legal when the tool budget is exhausted.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
if isinstance(remaining_tool_calls, bool) or remaining_tool_calls < 0:
|
|
87
|
+
raise OfficialArenaError("tau3 remaining tool-call budget is invalid")
|
|
88
|
+
calls = getattr(response, "tool_calls", None)
|
|
89
|
+
if calls is None:
|
|
90
|
+
return response, 0, 0
|
|
91
|
+
if not isinstance(calls, (list, tuple)):
|
|
92
|
+
raise OfficialArenaError("tau3 generated tool_calls are invalid")
|
|
93
|
+
generated = len(calls)
|
|
94
|
+
kept = min(generated, remaining_tool_calls)
|
|
95
|
+
discarded = generated - kept
|
|
96
|
+
if discarded == 0:
|
|
97
|
+
return response, kept, 0
|
|
98
|
+
copier = getattr(response, "model_copy", None)
|
|
99
|
+
if not callable(copier):
|
|
100
|
+
raise OfficialArenaError("tau3 assistant response cannot be budget-truncated")
|
|
101
|
+
if kept:
|
|
102
|
+
response = copier(update={"tool_calls": list(calls[:kept])}, deep=True)
|
|
103
|
+
else:
|
|
104
|
+
response = copier(
|
|
105
|
+
update={
|
|
106
|
+
"tool_calls": None,
|
|
107
|
+
"content": _SHARED_BUDGET_STOP_MESSAGE,
|
|
108
|
+
},
|
|
109
|
+
deep=True,
|
|
110
|
+
)
|
|
111
|
+
return response, kept, discarded
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _budget_exhausted_result(
|
|
115
|
+
*,
|
|
116
|
+
replay_record: Mapping[str, Any],
|
|
117
|
+
role: str,
|
|
118
|
+
seed: int,
|
|
119
|
+
work_dir: Path,
|
|
120
|
+
optimizer_usage: Mapping[str, int | float],
|
|
121
|
+
remaining_budget: Mapping[str, int | float],
|
|
122
|
+
) -> ProviderArmResult:
|
|
123
|
+
"""Materialize a scored, auditable zero-action arm after optimizer exhaustion.
|
|
124
|
+
|
|
125
|
+
A shared resource cap is an experimental outcome, not an infrastructure
|
|
126
|
+
failure. In particular, the canonical eight-document HyDE intervention
|
|
127
|
+
consumes all eight x1 model calls before the downstream tau3 agent can
|
|
128
|
+
act. Preserve the paired baseline, report a hard stop, and let the common
|
|
129
|
+
observation normalizer mark the treatment contract invalid with valid
|
|
130
|
+
success zero.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
trace = {
|
|
134
|
+
"provider": "tau3",
|
|
135
|
+
"source_revision": Tau3Provider.source_revision,
|
|
136
|
+
"task_id": replay_record["protected"]["task_id"],
|
|
137
|
+
"role": role,
|
|
138
|
+
"seed": seed,
|
|
139
|
+
"termination_reason": "shared_budget_exhausted_before_agent",
|
|
140
|
+
"messages": [],
|
|
141
|
+
"reward_info": None,
|
|
142
|
+
"generation_controls": {
|
|
143
|
+
"hard_stops": 1,
|
|
144
|
+
"optimizer_usage": dict(optimizer_usage),
|
|
145
|
+
"remaining_budget": dict(remaining_budget),
|
|
146
|
+
},
|
|
147
|
+
"score_provenance": "paper_shared_budget_zero_action",
|
|
148
|
+
}
|
|
149
|
+
work_dir.joinpath("budget-stop.json").write_text(
|
|
150
|
+
json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
151
|
+
encoding="utf-8",
|
|
152
|
+
)
|
|
153
|
+
return ProviderArmResult(
|
|
154
|
+
metrics={
|
|
155
|
+
"official_reward": 0.0,
|
|
156
|
+
"cost": 0.0,
|
|
157
|
+
"adapter_hard_stops": 1.0,
|
|
158
|
+
},
|
|
159
|
+
trace=trace,
|
|
160
|
+
usage={
|
|
161
|
+
"model_calls": 0,
|
|
162
|
+
"tool_calls": 0,
|
|
163
|
+
"input_tokens": 0,
|
|
164
|
+
"output_tokens": 0,
|
|
165
|
+
"cost_usd": 0.0,
|
|
166
|
+
"unpriced_model_calls": 0,
|
|
167
|
+
},
|
|
168
|
+
runtime_instance_id=content_hash(
|
|
169
|
+
{
|
|
170
|
+
"provider": "tau3",
|
|
171
|
+
"checkpoint_id": replay_record["checkpoint_id"],
|
|
172
|
+
"role": role,
|
|
173
|
+
"seed": seed,
|
|
174
|
+
"termination_reason": trace["termination_reason"],
|
|
175
|
+
"work_dir": str(work_dir),
|
|
176
|
+
}
|
|
177
|
+
),
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _model_output_invalid_result(
|
|
182
|
+
*,
|
|
183
|
+
replay_record: Mapping[str, Any],
|
|
184
|
+
role: str,
|
|
185
|
+
seed: int,
|
|
186
|
+
work_dir: Path,
|
|
187
|
+
error: Exception,
|
|
188
|
+
messages: list[object],
|
|
189
|
+
termination_reason: str = "generated_tool_arguments_invalid",
|
|
190
|
+
failed_generation_scope: str = "evaluated_agent",
|
|
191
|
+
measured_usage: Mapping[str, int | float] | None = None,
|
|
192
|
+
) -> ProviderArmResult:
|
|
193
|
+
"""Score a recognized malformed generation as an arm outcome, not infra loss.
|
|
194
|
+
|
|
195
|
+
``tau2`` raises before producing ``reward_info`` when either the evaluated
|
|
196
|
+
agent emits malformed tool arguments or the benchmark user simulator emits
|
|
197
|
+
an empty message. Both are real trajectory terminal states. The latter
|
|
198
|
+
must retain the evaluated agent's already-consumed usage while excluding
|
|
199
|
+
the benchmark user's failed generation from the paper-wide agent budget.
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
serialized_messages = []
|
|
203
|
+
for message in messages:
|
|
204
|
+
dump = getattr(message, "model_dump", None)
|
|
205
|
+
if callable(dump):
|
|
206
|
+
serialized_messages.append(dump(mode="json"))
|
|
207
|
+
trace = {
|
|
208
|
+
"provider": "tau3",
|
|
209
|
+
"source_revision": Tau3Provider.source_revision,
|
|
210
|
+
"task_id": replay_record["protected"]["task_id"],
|
|
211
|
+
"role": role,
|
|
212
|
+
"seed": seed,
|
|
213
|
+
"termination_reason": termination_reason,
|
|
214
|
+
"messages": serialized_messages,
|
|
215
|
+
"reward_info": None,
|
|
216
|
+
"generation_controls": {
|
|
217
|
+
"hard_stops": 1,
|
|
218
|
+
"failure_type": type(error).__name__,
|
|
219
|
+
"failed_generation_scope": failed_generation_scope,
|
|
220
|
+
"failed_generation_usage_unknown": True,
|
|
221
|
+
},
|
|
222
|
+
"score_provenance": "paper_model_output_invalid_zero_action",
|
|
223
|
+
}
|
|
224
|
+
work_dir.joinpath("model-output-invalid.json").write_text(
|
|
225
|
+
json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
226
|
+
encoding="utf-8",
|
|
227
|
+
)
|
|
228
|
+
usage = (
|
|
229
|
+
{
|
|
230
|
+
"model_calls": 1,
|
|
231
|
+
"tool_calls": 0,
|
|
232
|
+
"input_tokens": 0,
|
|
233
|
+
"output_tokens": 0,
|
|
234
|
+
"cost_usd": 0.0,
|
|
235
|
+
"unpriced_model_calls": 1,
|
|
236
|
+
}
|
|
237
|
+
if measured_usage is None
|
|
238
|
+
else dict(measured_usage)
|
|
239
|
+
)
|
|
240
|
+
return ProviderArmResult(
|
|
241
|
+
metrics={
|
|
242
|
+
"official_reward": 0.0,
|
|
243
|
+
"cost": 0.0,
|
|
244
|
+
"adapter_hard_stops": 1.0,
|
|
245
|
+
},
|
|
246
|
+
trace=trace,
|
|
247
|
+
usage=usage,
|
|
248
|
+
runtime_instance_id=content_hash(
|
|
249
|
+
{
|
|
250
|
+
"provider": "tau3",
|
|
251
|
+
"checkpoint_id": replay_record["checkpoint_id"],
|
|
252
|
+
"role": role,
|
|
253
|
+
"seed": seed,
|
|
254
|
+
"termination_reason": trace["termination_reason"],
|
|
255
|
+
"work_dir": str(work_dir),
|
|
256
|
+
}
|
|
257
|
+
),
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _is_generated_tool_argument_validation(error: Exception) -> bool:
|
|
262
|
+
errors = getattr(error, "errors", None)
|
|
263
|
+
if not callable(errors):
|
|
264
|
+
return False
|
|
265
|
+
return any(
|
|
266
|
+
item.get("type") == "dict_type" and tuple(item.get("loc", ()))[-1:] == ("arguments",)
|
|
267
|
+
for item in errors()
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _is_generated_user_message_validation(error: Exception) -> bool:
|
|
272
|
+
"""Recognize only tau2's explicit empty generated-user-message failure."""
|
|
273
|
+
|
|
274
|
+
return isinstance(error, ValueError) and str(error).startswith(
|
|
275
|
+
"UserMessage must have either content or tool_calls. Got UserMessage"
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _downstream_agent_usage(
|
|
280
|
+
messages: list[object],
|
|
281
|
+
*,
|
|
282
|
+
initial_history_length: int,
|
|
283
|
+
assistant_message_type: type,
|
|
284
|
+
measured_model_calls: int | None = None,
|
|
285
|
+
measured_input_tokens: int | None = None,
|
|
286
|
+
measured_output_tokens: int | None = None,
|
|
287
|
+
) -> dict[str, int]:
|
|
288
|
+
"""Measure only calls made by the evaluated agent after the replay checkpoint.
|
|
289
|
+
|
|
290
|
+
The tau3 user simulator is part of the benchmark environment. Its generated
|
|
291
|
+
``UserMessage`` objects may also carry provider usage, but charging those
|
|
292
|
+
tokens to the evaluated arm can falsely exceed the paper-wide agent budget.
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
input_tokens = 0
|
|
296
|
+
output_tokens = 0
|
|
297
|
+
model_calls = 0
|
|
298
|
+
tool_calls = 0
|
|
299
|
+
for message in messages[initial_history_length:]:
|
|
300
|
+
if not isinstance(message, assistant_message_type):
|
|
301
|
+
continue
|
|
302
|
+
model_calls += 1
|
|
303
|
+
usage = getattr(message, "usage", None)
|
|
304
|
+
if isinstance(usage, Mapping):
|
|
305
|
+
input_tokens += int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0)
|
|
306
|
+
output_tokens += int(usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0)
|
|
307
|
+
calls = getattr(message, "tool_calls", None)
|
|
308
|
+
if calls:
|
|
309
|
+
tool_calls += len(calls)
|
|
310
|
+
measured = {
|
|
311
|
+
"model_calls": model_calls if measured_model_calls is None else measured_model_calls,
|
|
312
|
+
"tool_calls": tool_calls,
|
|
313
|
+
"input_tokens": input_tokens if measured_input_tokens is None else measured_input_tokens,
|
|
314
|
+
"output_tokens": (
|
|
315
|
+
output_tokens if measured_output_tokens is None else measured_output_tokens
|
|
316
|
+
),
|
|
317
|
+
}
|
|
318
|
+
if any(
|
|
319
|
+
isinstance(value, bool) or not isinstance(value, int) or value < 0
|
|
320
|
+
for value in measured.values()
|
|
321
|
+
):
|
|
322
|
+
raise OfficialArenaError("tau3 evaluated-agent usage counters are invalid")
|
|
323
|
+
return measured
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _orchestrator_step_cap(
|
|
327
|
+
*,
|
|
328
|
+
remaining_budget: Mapping[str, int | float],
|
|
329
|
+
materialized: Mapping[str, Any],
|
|
330
|
+
) -> int:
|
|
331
|
+
"""Translate the evaluated-agent call budget into half-duplex steps.
|
|
332
|
+
|
|
333
|
+
tau3 counts each USER/AGENT/ENV transition as an orchestrator step, whereas
|
|
334
|
+
the paper budget counts only generations made by the evaluated agent. A
|
|
335
|
+
normal agent turn therefore needs two orchestrator steps. A materialized
|
|
336
|
+
clarification contributes one deterministic, zero-generation agent turn
|
|
337
|
+
before the remaining model calls.
|
|
338
|
+
"""
|
|
339
|
+
|
|
340
|
+
model_calls = int(remaining_budget["max_model_calls"])
|
|
341
|
+
metadata = materialized.get("metadata", {})
|
|
342
|
+
clarification = metadata.get("clarification_request") if isinstance(metadata, Mapping) else None
|
|
343
|
+
deterministic_agent_turns = int(isinstance(clarification, str) and bool(clarification.strip()))
|
|
344
|
+
return 2 * (model_calls + deterministic_agent_turns)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _local_tau_model_ids(config: Mapping[str, Any]) -> frozenset[str]:
|
|
348
|
+
"""Return model ids whose tau3 endpoint is explicitly loopback-local."""
|
|
349
|
+
|
|
350
|
+
local: set[str] = set()
|
|
351
|
+
for role in ("agent", "user"):
|
|
352
|
+
args = config.get(f"llm_args_{role}")
|
|
353
|
+
model = config.get(f"llm_{role}")
|
|
354
|
+
if not isinstance(args, Mapping) or not isinstance(model, str):
|
|
355
|
+
continue
|
|
356
|
+
raw_url = args.get("api_base") or args.get("base_url")
|
|
357
|
+
if not isinstance(raw_url, str):
|
|
358
|
+
continue
|
|
359
|
+
hostname = urlparse(raw_url).hostname
|
|
360
|
+
if hostname not in {"127.0.0.1", "localhost", "::1"}:
|
|
361
|
+
continue
|
|
362
|
+
local.add(model.rsplit("/", 1)[-1])
|
|
363
|
+
return frozenset(local)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _install_local_tau_cost_accounting(config: Mapping[str, Any]) -> frozenset[str]:
|
|
367
|
+
"""Keep local unpriced models out of LiteLLM's remote price lookup path."""
|
|
368
|
+
|
|
369
|
+
local_models = _local_tau_model_ids(config)
|
|
370
|
+
if not local_models:
|
|
371
|
+
return local_models
|
|
372
|
+
from tau2.utils import llm_utils
|
|
373
|
+
|
|
374
|
+
current = llm_utils.get_response_cost
|
|
375
|
+
registered = getattr(current, "_agentinputlab_local_models", None)
|
|
376
|
+
if isinstance(registered, set):
|
|
377
|
+
registered.update(local_models)
|
|
378
|
+
return local_models
|
|
379
|
+
original = current
|
|
380
|
+
registered = set(local_models)
|
|
381
|
+
|
|
382
|
+
def local_aware_cost(response: Any) -> float:
|
|
383
|
+
model = str(getattr(response, "model", "")).rsplit("/", 1)[-1]
|
|
384
|
+
if model in registered:
|
|
385
|
+
return 0.0
|
|
386
|
+
return float(original(response))
|
|
387
|
+
|
|
388
|
+
local_aware_cost._agentinputlab_local_models = registered # type: ignore[attr-defined]
|
|
389
|
+
llm_utils.get_response_cost = local_aware_cost
|
|
390
|
+
return local_models
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _local_tau_nl_evaluator_binding(config: Mapping[str, Any]) -> Mapping[str, Any] | None:
|
|
394
|
+
"""Bind τ³'s NL-assertion judge to the declared local model endpoint.
|
|
395
|
+
|
|
396
|
+
The pinned τ³ scorer evaluates the final Banking Knowledge task with an
|
|
397
|
+
LLM-backed natural-language assertion. Upstream defaults that judge to a
|
|
398
|
+
closed GPT endpoint. A fully local paper profile must not silently leave
|
|
399
|
+
the declared runtime and contact that endpoint. For explicit loopback
|
|
400
|
+
profiles, use the same frozen model/load set and deterministic decoding
|
|
401
|
+
controls as the two simulated participants. External/API profiles retain
|
|
402
|
+
the upstream evaluator binding.
|
|
403
|
+
"""
|
|
404
|
+
|
|
405
|
+
local_models = _local_tau_model_ids(config)
|
|
406
|
+
model = config.get("llm_agent")
|
|
407
|
+
raw_args = config.get("llm_args_agent")
|
|
408
|
+
if not isinstance(model, str) or not isinstance(raw_args, Mapping):
|
|
409
|
+
return None
|
|
410
|
+
if model.rsplit("/", 1)[-1] not in local_models:
|
|
411
|
+
return None
|
|
412
|
+
args = dict(raw_args)
|
|
413
|
+
args["temperature"] = 0.0
|
|
414
|
+
args["response_format"] = {"type": "json_object"}
|
|
415
|
+
return {"model": model, "arguments": args, "provenance": "declared_local_load_set"}
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _install_local_tau_nl_evaluator(config: Mapping[str, Any]) -> Mapping[str, Any] | None:
|
|
419
|
+
"""Install the explicit local NL-assertion evaluator in the pinned scorer."""
|
|
420
|
+
|
|
421
|
+
binding = _local_tau_nl_evaluator_binding(config)
|
|
422
|
+
if binding is None:
|
|
423
|
+
return None
|
|
424
|
+
from tau2.evaluator import evaluator_nl_assertions
|
|
425
|
+
|
|
426
|
+
evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS = binding["model"]
|
|
427
|
+
evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS_ARGS = dict(binding["arguments"])
|
|
428
|
+
return binding
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
class Tau3Provider:
|
|
432
|
+
benchmark_id = "tau3.banking_knowledge"
|
|
433
|
+
source_revision = "cf71a8070269883e38a365ffa85f78f46844c1f4"
|
|
434
|
+
scorer_id = "tau3.official_v1.0.1"
|
|
435
|
+
scorer_revision = "32526f9c17ba3725cc8327d43783a2b5d2c48015dc72dbc6f7af4d25a95328a6"
|
|
436
|
+
|
|
437
|
+
_CONFIG_FIELDS = frozenset(
|
|
438
|
+
{
|
|
439
|
+
"task_set_name",
|
|
440
|
+
"task_split_name",
|
|
441
|
+
"limit",
|
|
442
|
+
"llm_agent",
|
|
443
|
+
"llm_user",
|
|
444
|
+
"llm_args_agent",
|
|
445
|
+
"llm_args_user",
|
|
446
|
+
"retrieval_config",
|
|
447
|
+
"retrieval_config_kwargs",
|
|
448
|
+
"max_steps",
|
|
449
|
+
"max_errors",
|
|
450
|
+
"timeout",
|
|
451
|
+
"locked_dependency_hashes",
|
|
452
|
+
}
|
|
453
|
+
)
|
|
454
|
+
_OPTIONAL_CONFIG_FIELDS = frozenset(
|
|
455
|
+
{
|
|
456
|
+
"retriever_override",
|
|
457
|
+
"budget_override",
|
|
458
|
+
"capture_max_attempts",
|
|
459
|
+
"capture_text_only_fallback",
|
|
460
|
+
"capture_task_ids",
|
|
461
|
+
}
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
def __init__(self, *, history_transform: Any = None) -> None:
|
|
465
|
+
validate_history_transform(history_transform)
|
|
466
|
+
self.history_transform = history_transform
|
|
467
|
+
|
|
468
|
+
@staticmethod
|
|
469
|
+
def _history_result(result, agent, work_dir):
|
|
470
|
+
session = getattr(agent, "_history_session", None)
|
|
471
|
+
if session is None:
|
|
472
|
+
return result
|
|
473
|
+
receipt = work_dir / "live-history.json"
|
|
474
|
+
session.write_receipt(receipt, target_usage=agent._history_target_usage())
|
|
475
|
+
metrics = dict(result.metrics)
|
|
476
|
+
if "cost" in metrics:
|
|
477
|
+
metrics["cost"] += float(session.usage.get("cost_usd", 0))
|
|
478
|
+
return ProviderArmResult(
|
|
479
|
+
metrics=metrics,
|
|
480
|
+
trace={**result.trace, "live_history_transform": session.hook.fingerprint(),
|
|
481
|
+
"live_history_receipt": {"path": receipt.name, "sha256": _file_sha256(receipt)},
|
|
482
|
+
"history_usage": dict(session.usage)},
|
|
483
|
+
usage=session.combined_usage(result.usage),
|
|
484
|
+
runtime_instance_id=content_hash({"native_runtime": result.runtime_instance_id,
|
|
485
|
+
"live_history_receipt_sha256": _file_sha256(receipt)}),
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
def doctor(self, config: Mapping[str, Any]) -> tuple[str, ...]:
|
|
489
|
+
try:
|
|
490
|
+
self._validate_config(config)
|
|
491
|
+
head = subprocess.check_output(
|
|
492
|
+
["git", "rev-parse", "HEAD"],
|
|
493
|
+
text=True,
|
|
494
|
+
stderr=subprocess.DEVNULL,
|
|
495
|
+
).strip()
|
|
496
|
+
if head != self.source_revision:
|
|
497
|
+
raise OfficialArenaError("tau3 checkout revision differs")
|
|
498
|
+
from tau2.agent.llm_agent import LLMAgent
|
|
499
|
+
from tau2.runner.build import build_orchestrator
|
|
500
|
+
from tau2.runner.helpers import get_tasks
|
|
501
|
+
from tau2.runner.simulation import run_simulation
|
|
502
|
+
|
|
503
|
+
del LLMAgent, build_orchestrator, get_tasks, run_simulation
|
|
504
|
+
except Exception as exc:
|
|
505
|
+
return (f"{type(exc).__name__}: {exc}",)
|
|
506
|
+
return ()
|
|
507
|
+
|
|
508
|
+
def capture(
|
|
509
|
+
self,
|
|
510
|
+
config: Mapping[str, Any],
|
|
511
|
+
*,
|
|
512
|
+
output_root: Path,
|
|
513
|
+
) -> tuple[Mapping[str, Any], ...]:
|
|
514
|
+
del output_root
|
|
515
|
+
parsed = self._validate_config(config)
|
|
516
|
+
_install_local_tau_cost_accounting(parsed)
|
|
517
|
+
_install_local_tau_nl_evaluator(parsed)
|
|
518
|
+
_install_agentinputlab_retriever(parsed)
|
|
519
|
+
_install_tau_tool_argument_normalizer()
|
|
520
|
+
from tau2.data_model.message import AssistantMessage, UserMessage
|
|
521
|
+
from tau2.runner.build import build_orchestrator
|
|
522
|
+
from tau2.runner.helpers import get_tasks
|
|
523
|
+
|
|
524
|
+
run_config = self._run_config(parsed)
|
|
525
|
+
tasks = get_tasks(
|
|
526
|
+
parsed["task_set_name"],
|
|
527
|
+
task_split_name=parsed["task_split_name"],
|
|
528
|
+
num_tasks=parsed["limit"],
|
|
529
|
+
task_ids=parsed.get("capture_task_ids"),
|
|
530
|
+
)
|
|
531
|
+
rows: list[Mapping[str, Any]] = []
|
|
532
|
+
capture_attempts, text_only_fallback = _capture_attempt_policy(
|
|
533
|
+
parsed.get("capture_max_attempts", 1),
|
|
534
|
+
parsed.get("capture_text_only_fallback", False),
|
|
535
|
+
)
|
|
536
|
+
capture_policy = {
|
|
537
|
+
"policy_id": "tau3.single_capture.v1" if capture_attempts == 1 else "tau3.explicit_bounded_capture.v1",
|
|
538
|
+
"max_attempts": capture_attempts,
|
|
539
|
+
"text_only_fallback_allowed": text_only_fallback,
|
|
540
|
+
"usage_scope": "environment_preparation_not_evaluated_agent_budget",
|
|
541
|
+
"attempt_count_is_not_model_call_or_token_usage": True,
|
|
542
|
+
}
|
|
543
|
+
for index, task in enumerate(tasks):
|
|
544
|
+
orchestrator, message, capture_attempt, capture_seed = _capture_initial_visible_user(
|
|
545
|
+
build_orchestrator=build_orchestrator,
|
|
546
|
+
run_config=run_config,
|
|
547
|
+
task=task,
|
|
548
|
+
base_seed=index,
|
|
549
|
+
assistant_message_type=AssistantMessage,
|
|
550
|
+
user_message_type=UserMessage,
|
|
551
|
+
max_attempts=capture_attempts,
|
|
552
|
+
allow_text_only_fallback=text_only_fallback,
|
|
553
|
+
)
|
|
554
|
+
try:
|
|
555
|
+
initial_history = []
|
|
556
|
+
for item in orchestrator.get_messages():
|
|
557
|
+
normalized = item.model_copy(update={"timestamp": None, "turn_idx": None})
|
|
558
|
+
initial_history.append(normalized.model_dump(mode="json"))
|
|
559
|
+
tools = orchestrator.environment.get_tools()
|
|
560
|
+
tool_schemas = [tool.openai_schema for tool in tools]
|
|
561
|
+
tool_names = tuple(
|
|
562
|
+
getattr(tool, "name", None) or getattr(tool, "__name__", None) for tool in tools
|
|
563
|
+
)
|
|
564
|
+
if any(not isinstance(name, str) or not name for name in tool_names):
|
|
565
|
+
raise OfficialArenaError("tau3 tool inventory contains unnamed tools")
|
|
566
|
+
policy = orchestrator.environment.get_policy()
|
|
567
|
+
if (
|
|
568
|
+
not isinstance(tool_schemas, list)
|
|
569
|
+
or not tool_schemas
|
|
570
|
+
or any(not isinstance(item, Mapping) for item in tool_schemas)
|
|
571
|
+
):
|
|
572
|
+
raise OfficialArenaError("tau3 tool schemas are unavailable")
|
|
573
|
+
visible_state = {
|
|
574
|
+
"query": message.content,
|
|
575
|
+
"task_id": task.id,
|
|
576
|
+
"instruction": policy,
|
|
577
|
+
"context": [
|
|
578
|
+
{
|
|
579
|
+
"text": policy,
|
|
580
|
+
"block_id": "official_policy",
|
|
581
|
+
"source": "tau3.environment.policy",
|
|
582
|
+
"score": None,
|
|
583
|
+
"metadata": {"domain": "banking_knowledge"},
|
|
584
|
+
},
|
|
585
|
+
{
|
|
586
|
+
"text": json.dumps(
|
|
587
|
+
tool_schemas,
|
|
588
|
+
default=str,
|
|
589
|
+
ensure_ascii=False,
|
|
590
|
+
sort_keys=True,
|
|
591
|
+
),
|
|
592
|
+
"block_id": "official_tool_catalog",
|
|
593
|
+
"source": "tau3.environment.tools",
|
|
594
|
+
"score": None,
|
|
595
|
+
"metadata": {
|
|
596
|
+
"domain": "banking_knowledge",
|
|
597
|
+
"tool_defs_hash": content_hash(tool_schemas),
|
|
598
|
+
},
|
|
599
|
+
},
|
|
600
|
+
],
|
|
601
|
+
"messages": [
|
|
602
|
+
{
|
|
603
|
+
"role": "user",
|
|
604
|
+
"content": message.content,
|
|
605
|
+
"name": None,
|
|
606
|
+
}
|
|
607
|
+
],
|
|
608
|
+
"tool_names": list(tool_names),
|
|
609
|
+
"metadata": {
|
|
610
|
+
"benchmark_id": self.benchmark_id,
|
|
611
|
+
"checkpoint_phase": "after_initial_user_message",
|
|
612
|
+
"clarification_roundtrip_available": True,
|
|
613
|
+
"state_restore_available": True,
|
|
614
|
+
"retrieval_config": parsed["retrieval_config"],
|
|
615
|
+
"capture_attempt": capture_attempt,
|
|
616
|
+
"capture_seed": capture_seed,
|
|
617
|
+
"capture_policy": capture_policy,
|
|
618
|
+
},
|
|
619
|
+
"budget": visible_shared_budget(
|
|
620
|
+
parsed,
|
|
621
|
+
{
|
|
622
|
+
"max_model_calls": parsed["max_steps"],
|
|
623
|
+
"max_retrieval_calls": parsed["max_steps"],
|
|
624
|
+
"max_tool_calls": parsed["max_steps"],
|
|
625
|
+
"max_output_tokens": 65536,
|
|
626
|
+
"max_context_tokens": None,
|
|
627
|
+
"max_cost_usd": None,
|
|
628
|
+
},
|
|
629
|
+
),
|
|
630
|
+
}
|
|
631
|
+
rows.append(
|
|
632
|
+
{
|
|
633
|
+
"checkpoint_id": str(task.id),
|
|
634
|
+
"group_key": str(task.id),
|
|
635
|
+
"visible_state": visible_state,
|
|
636
|
+
"eligible_families": [
|
|
637
|
+
"reformulate",
|
|
638
|
+
"decompose",
|
|
639
|
+
"acquire",
|
|
640
|
+
"clarify",
|
|
641
|
+
"curate",
|
|
642
|
+
],
|
|
643
|
+
"protected": {
|
|
644
|
+
"task_id": str(task.id),
|
|
645
|
+
"initial_user_message": message.model_dump(mode="json"),
|
|
646
|
+
"initial_message_history": initial_history,
|
|
647
|
+
"capture_attempt": capture_attempt,
|
|
648
|
+
"capture_seed": capture_seed,
|
|
649
|
+
"capture_policy": capture_policy,
|
|
650
|
+
"locked_dependency_hashes": parsed["locked_dependency_hashes"],
|
|
651
|
+
},
|
|
652
|
+
}
|
|
653
|
+
)
|
|
654
|
+
finally:
|
|
655
|
+
orchestrator._cleanup()
|
|
656
|
+
if not rows:
|
|
657
|
+
raise OfficialArenaError("tau3 capture found no tasks")
|
|
658
|
+
return tuple(rows)
|
|
659
|
+
|
|
660
|
+
def run_arm(
|
|
661
|
+
self,
|
|
662
|
+
config: Mapping[str, Any],
|
|
663
|
+
*,
|
|
664
|
+
replay_record: Mapping[str, Any],
|
|
665
|
+
arm_payload: Mapping[str, Any],
|
|
666
|
+
seed: int,
|
|
667
|
+
role: str,
|
|
668
|
+
work_dir: Path,
|
|
669
|
+
) -> ProviderArmResult:
|
|
670
|
+
if self.history_transform is not None and (work_dir / "live-history.json").exists():
|
|
671
|
+
raise OfficialArenaError("native history output must be a fresh episode")
|
|
672
|
+
parsed = self._validate_config(config)
|
|
673
|
+
local_unpriced_models = _install_local_tau_cost_accounting(parsed)
|
|
674
|
+
nl_evaluator_binding = _install_local_tau_nl_evaluator(parsed)
|
|
675
|
+
_install_agentinputlab_retriever(parsed)
|
|
676
|
+
_install_tau_tool_argument_normalizer()
|
|
677
|
+
state = materialized_state(arm_payload)
|
|
678
|
+
budget = require_materialized_budget(state, parsed)
|
|
679
|
+
optimizer_usage = intervention_usage(arm_payload)
|
|
680
|
+
remaining_budget = _remaining_agent_budget(budget, optimizer_usage)
|
|
681
|
+
if remaining_budget["max_model_calls"] < 1 or remaining_budget["max_output_tokens"] < 1:
|
|
682
|
+
return _budget_exhausted_result(
|
|
683
|
+
replay_record=replay_record,
|
|
684
|
+
role=role,
|
|
685
|
+
seed=seed,
|
|
686
|
+
work_dir=work_dir,
|
|
687
|
+
optimizer_usage=optimizer_usage,
|
|
688
|
+
remaining_budget=remaining_budget,
|
|
689
|
+
)
|
|
690
|
+
protected = replay_record["protected"]
|
|
691
|
+
|
|
692
|
+
from tau2.data_model.message import AssistantMessage, ToolMessage, UserMessage
|
|
693
|
+
from tau2.data_model.tasks import InitialState
|
|
694
|
+
from tau2.orchestrator.orchestrator import DEFAULT_FIRST_AGENT_MESSAGE
|
|
695
|
+
from tau2.runner.build import (
|
|
696
|
+
_build_env_kwargs,
|
|
697
|
+
build_environment,
|
|
698
|
+
build_user,
|
|
699
|
+
)
|
|
700
|
+
from tau2.runner.helpers import get_tasks
|
|
701
|
+
from tau2.runner.simulation import run_simulation
|
|
702
|
+
|
|
703
|
+
task_id = protected["task_id"]
|
|
704
|
+
tasks = get_tasks(
|
|
705
|
+
parsed["task_set_name"],
|
|
706
|
+
task_split_name=parsed["task_split_name"],
|
|
707
|
+
task_ids=[task_id],
|
|
708
|
+
)
|
|
709
|
+
if len(tasks) != 1:
|
|
710
|
+
raise OfficialArenaError("tau3 protected task identity is unavailable")
|
|
711
|
+
task = tasks[0]
|
|
712
|
+
raw_history = protected.get("initial_message_history")
|
|
713
|
+
if isinstance(raw_history, list) and raw_history:
|
|
714
|
+
message_types = {
|
|
715
|
+
"assistant": AssistantMessage,
|
|
716
|
+
"user": UserMessage,
|
|
717
|
+
"tool": ToolMessage,
|
|
718
|
+
}
|
|
719
|
+
try:
|
|
720
|
+
initial_history = [
|
|
721
|
+
message_types[str(item["role"])].model_validate(item) for item in raw_history
|
|
722
|
+
]
|
|
723
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
724
|
+
raise OfficialArenaError(
|
|
725
|
+
"tau3 protected initial message history is invalid"
|
|
726
|
+
) from exc
|
|
727
|
+
else:
|
|
728
|
+
initial_user = UserMessage.model_validate(protected["initial_user_message"])
|
|
729
|
+
initial_history = [copy.deepcopy(DEFAULT_FIRST_AGENT_MESSAGE), initial_user]
|
|
730
|
+
original_initial = task.initial_state
|
|
731
|
+
task = task.model_copy(
|
|
732
|
+
update={
|
|
733
|
+
"initial_state": InitialState(
|
|
734
|
+
initialization_data=(
|
|
735
|
+
original_initial.initialization_data
|
|
736
|
+
if original_initial is not None
|
|
737
|
+
else None
|
|
738
|
+
),
|
|
739
|
+
initialization_actions=(
|
|
740
|
+
original_initial.initialization_actions
|
|
741
|
+
if original_initial is not None
|
|
742
|
+
else None
|
|
743
|
+
),
|
|
744
|
+
message_history=copy.deepcopy(initial_history),
|
|
745
|
+
)
|
|
746
|
+
},
|
|
747
|
+
deep=True,
|
|
748
|
+
)
|
|
749
|
+
run_config = self._run_config(parsed, budget=remaining_budget)
|
|
750
|
+
env_kwargs = _build_env_kwargs(run_config, task)
|
|
751
|
+
environment = build_environment(
|
|
752
|
+
"banking_knowledge",
|
|
753
|
+
env_kwargs=env_kwargs,
|
|
754
|
+
)
|
|
755
|
+
agent = _intervened_tau_agent_class()(
|
|
756
|
+
tools=environment.get_tools(),
|
|
757
|
+
domain_policy=environment.get_policy(),
|
|
758
|
+
llm=parsed["llm_agent"],
|
|
759
|
+
llm_args=parsed["llm_args_agent"],
|
|
760
|
+
materialized=state,
|
|
761
|
+
budget=remaining_budget,
|
|
762
|
+
history_transform=self.history_transform,
|
|
763
|
+
)
|
|
764
|
+
user = build_user(
|
|
765
|
+
"user_simulator",
|
|
766
|
+
environment,
|
|
767
|
+
task,
|
|
768
|
+
llm=parsed["llm_user"],
|
|
769
|
+
llm_args=parsed["llm_args_user"],
|
|
770
|
+
)
|
|
771
|
+
from tau2.orchestrator.orchestrator import Orchestrator
|
|
772
|
+
|
|
773
|
+
orchestrator = Orchestrator(
|
|
774
|
+
domain="banking_knowledge",
|
|
775
|
+
agent=agent,
|
|
776
|
+
user=user,
|
|
777
|
+
environment=environment,
|
|
778
|
+
task=task,
|
|
779
|
+
max_steps=_orchestrator_step_cap(
|
|
780
|
+
remaining_budget=remaining_budget,
|
|
781
|
+
materialized=state,
|
|
782
|
+
),
|
|
783
|
+
max_errors=parsed["max_errors"],
|
|
784
|
+
seed=seed,
|
|
785
|
+
simulation_id=f"agentinputlab-{role}-{task_id}-{seed}",
|
|
786
|
+
timeout=parsed["timeout"],
|
|
787
|
+
)
|
|
788
|
+
try:
|
|
789
|
+
simulation = run_simulation(
|
|
790
|
+
orchestrator,
|
|
791
|
+
env_kwargs=env_kwargs,
|
|
792
|
+
)
|
|
793
|
+
except json.JSONDecodeError as error:
|
|
794
|
+
result = _model_output_invalid_result(
|
|
795
|
+
replay_record=replay_record,
|
|
796
|
+
role=role,
|
|
797
|
+
seed=seed,
|
|
798
|
+
work_dir=work_dir,
|
|
799
|
+
error=error,
|
|
800
|
+
messages=orchestrator.get_messages(),
|
|
801
|
+
)
|
|
802
|
+
return self._history_result(result, agent, work_dir)
|
|
803
|
+
except ValueError as error:
|
|
804
|
+
if _is_generated_user_message_validation(error):
|
|
805
|
+
measured_usage = _downstream_agent_usage(
|
|
806
|
+
orchestrator.get_messages(),
|
|
807
|
+
initial_history_length=len(initial_history),
|
|
808
|
+
assistant_message_type=AssistantMessage,
|
|
809
|
+
measured_model_calls=agent._budget_model_calls,
|
|
810
|
+
measured_input_tokens=agent._budget_input_tokens,
|
|
811
|
+
measured_output_tokens=agent._budget_output_tokens,
|
|
812
|
+
)
|
|
813
|
+
result = _model_output_invalid_result(
|
|
814
|
+
replay_record=replay_record,
|
|
815
|
+
role=role,
|
|
816
|
+
seed=seed,
|
|
817
|
+
work_dir=work_dir,
|
|
818
|
+
error=error,
|
|
819
|
+
messages=orchestrator.get_messages(),
|
|
820
|
+
termination_reason="generated_user_message_invalid",
|
|
821
|
+
failed_generation_scope="benchmark_user_simulator",
|
|
822
|
+
measured_usage={
|
|
823
|
+
**measured_usage,
|
|
824
|
+
"cost_usd": 0.0,
|
|
825
|
+
"unpriced_model_calls": measured_usage["model_calls"],
|
|
826
|
+
},
|
|
827
|
+
)
|
|
828
|
+
return self._history_result(result, agent, work_dir)
|
|
829
|
+
if not _is_generated_tool_argument_validation(error):
|
|
830
|
+
raise
|
|
831
|
+
result = _model_output_invalid_result(
|
|
832
|
+
replay_record=replay_record,
|
|
833
|
+
role=role,
|
|
834
|
+
seed=seed,
|
|
835
|
+
work_dir=work_dir,
|
|
836
|
+
error=error,
|
|
837
|
+
messages=orchestrator.get_messages(),
|
|
838
|
+
)
|
|
839
|
+
return self._history_result(result, agent, work_dir)
|
|
840
|
+
finally:
|
|
841
|
+
if getattr(agent, "_history_session", None) is not None:
|
|
842
|
+
agent._history_session.write_receipt(work_dir / "live-history.json",
|
|
843
|
+
target_usage=agent._history_target_usage())
|
|
844
|
+
if simulation.reward_info is None:
|
|
845
|
+
raise OfficialArenaError("tau3 official evaluator did not return reward_info")
|
|
846
|
+
|
|
847
|
+
messages = simulation.get_messages()
|
|
848
|
+
# The replay history contains the fixed initial assistant greeting. It
|
|
849
|
+
# is not a model call made by this arm and must not consume the shared
|
|
850
|
+
# execution budget a second time. The user simulator is benchmark
|
|
851
|
+
# environment state, so its calls and tokens are deliberately excluded.
|
|
852
|
+
agent_usage = _downstream_agent_usage(
|
|
853
|
+
messages,
|
|
854
|
+
initial_history_length=len(initial_history),
|
|
855
|
+
assistant_message_type=AssistantMessage,
|
|
856
|
+
# The intervened agent owns the authoritative counters for real
|
|
857
|
+
# generations. A materialized clarification and the deterministic
|
|
858
|
+
# budget-stop response are AssistantMessages too, but neither is a
|
|
859
|
+
# model call and neither may consume the shared generation budget.
|
|
860
|
+
measured_model_calls=agent._budget_model_calls,
|
|
861
|
+
measured_input_tokens=agent._budget_input_tokens,
|
|
862
|
+
measured_output_tokens=agent._budget_output_tokens,
|
|
863
|
+
)
|
|
864
|
+
if agent_usage["tool_calls"] != agent._budget_tool_calls:
|
|
865
|
+
raise OfficialArenaError(
|
|
866
|
+
"tau3 executed tool-call trace differs from its enforced budget counter"
|
|
867
|
+
)
|
|
868
|
+
trace = {
|
|
869
|
+
"provider": "tau3",
|
|
870
|
+
"source_revision": self.source_revision,
|
|
871
|
+
"task_id": task_id,
|
|
872
|
+
"role": role,
|
|
873
|
+
"seed": seed,
|
|
874
|
+
"cost_accounting": (
|
|
875
|
+
"local_unpriced_zero"
|
|
876
|
+
if str(parsed["llm_agent"]).rsplit("/", 1)[-1] in local_unpriced_models
|
|
877
|
+
else "upstream_litellm"
|
|
878
|
+
),
|
|
879
|
+
"nl_assertion_evaluator": (
|
|
880
|
+
{
|
|
881
|
+
"model": nl_evaluator_binding["model"],
|
|
882
|
+
"provenance": nl_evaluator_binding["provenance"],
|
|
883
|
+
}
|
|
884
|
+
if nl_evaluator_binding is not None
|
|
885
|
+
else {"model": "upstream_default", "provenance": "pinned_tau3"}
|
|
886
|
+
),
|
|
887
|
+
"termination_reason": simulation.termination_reason.value,
|
|
888
|
+
"messages": [message.model_dump(mode="json") for message in messages],
|
|
889
|
+
"reward_info": simulation.reward_info.model_dump(mode="json"),
|
|
890
|
+
"generation_controls": {
|
|
891
|
+
"hard_stops": agent._budget_hard_stops,
|
|
892
|
+
"executed_tool_calls": agent._budget_tool_calls,
|
|
893
|
+
"discarded_tool_calls": agent._budget_discarded_tool_calls,
|
|
894
|
+
},
|
|
895
|
+
}
|
|
896
|
+
work_dir.joinpath("simulation.json").write_text(
|
|
897
|
+
json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
898
|
+
encoding="utf-8",
|
|
899
|
+
)
|
|
900
|
+
total_cost = float(simulation.agent_cost or 0.0) + float(simulation.user_cost or 0.0)
|
|
901
|
+
result = ProviderArmResult(
|
|
902
|
+
metrics={
|
|
903
|
+
"official_reward": float(simulation.reward_info.reward),
|
|
904
|
+
"cost": total_cost,
|
|
905
|
+
"adapter_hard_stops": float(agent._budget_hard_stops),
|
|
906
|
+
},
|
|
907
|
+
trace=trace,
|
|
908
|
+
usage={
|
|
909
|
+
**agent_usage,
|
|
910
|
+
"cost_usd": total_cost,
|
|
911
|
+
"unpriced_model_calls": (
|
|
912
|
+
agent_usage["model_calls"]
|
|
913
|
+
if (
|
|
914
|
+
str(parsed["llm_agent"]).rsplit("/", 1)[-1] in local_unpriced_models
|
|
915
|
+
or (agent_usage["input_tokens"] == 0 and agent_usage["output_tokens"] == 0)
|
|
916
|
+
)
|
|
917
|
+
else 0
|
|
918
|
+
),
|
|
919
|
+
},
|
|
920
|
+
runtime_instance_id=content_hash(
|
|
921
|
+
{
|
|
922
|
+
"provider": "tau3",
|
|
923
|
+
"checkpoint_id": replay_record["checkpoint_id"],
|
|
924
|
+
"role": role,
|
|
925
|
+
"seed": seed,
|
|
926
|
+
"simulation_id": simulation.id,
|
|
927
|
+
"work_dir": str(work_dir),
|
|
928
|
+
}
|
|
929
|
+
),
|
|
930
|
+
)
|
|
931
|
+
return self._history_result(result, agent, work_dir)
|
|
932
|
+
|
|
933
|
+
def _run_config(
|
|
934
|
+
self,
|
|
935
|
+
config: Mapping[str, Any],
|
|
936
|
+
*,
|
|
937
|
+
budget: Mapping[str, Any] | None = None,
|
|
938
|
+
) -> Any:
|
|
939
|
+
from tau2.data_model.simulation import TextRunConfig
|
|
940
|
+
|
|
941
|
+
agent_args = dict(config["llm_args_agent"])
|
|
942
|
+
max_steps = int(config["max_steps"])
|
|
943
|
+
if budget is not None:
|
|
944
|
+
max_steps = min(max_steps, int(budget["max_model_calls"]))
|
|
945
|
+
override = config.get("budget_override")
|
|
946
|
+
declared_per_call = (
|
|
947
|
+
int(override["max_output_tokens_per_call"])
|
|
948
|
+
if isinstance(override, Mapping)
|
|
949
|
+
else max(1, int(budget["max_output_tokens"]) // max_steps)
|
|
950
|
+
)
|
|
951
|
+
per_call = int(
|
|
952
|
+
min(
|
|
953
|
+
declared_per_call,
|
|
954
|
+
max(1, int(budget["max_output_tokens"]) // max_steps),
|
|
955
|
+
)
|
|
956
|
+
)
|
|
957
|
+
agent_args["max_tokens"] = min(
|
|
958
|
+
int(agent_args.get("max_tokens", per_call)),
|
|
959
|
+
per_call,
|
|
960
|
+
int(budget["max_output_tokens"]),
|
|
961
|
+
)
|
|
962
|
+
|
|
963
|
+
return TextRunConfig(
|
|
964
|
+
domain="banking_knowledge",
|
|
965
|
+
agent="llm_agent",
|
|
966
|
+
llm_agent=config["llm_agent"],
|
|
967
|
+
llm_args_agent=agent_args,
|
|
968
|
+
user="user_simulator",
|
|
969
|
+
llm_user=config["llm_user"],
|
|
970
|
+
llm_args_user=dict(config["llm_args_user"]),
|
|
971
|
+
max_steps=max_steps,
|
|
972
|
+
max_errors=config["max_errors"],
|
|
973
|
+
timeout=config["timeout"],
|
|
974
|
+
retrieval_config=config["retrieval_config"],
|
|
975
|
+
retrieval_config_kwargs=dict(config["retrieval_config_kwargs"]),
|
|
976
|
+
)
|
|
977
|
+
|
|
978
|
+
def _validate_config(self, config: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
979
|
+
config = resolve_shared_budget_config(resolve_retriever_config(config))
|
|
980
|
+
if not isinstance(config, Mapping) or not (
|
|
981
|
+
set(config) >= self._CONFIG_FIELDS
|
|
982
|
+
and set(config) <= self._CONFIG_FIELDS | self._OPTIONAL_CONFIG_FIELDS
|
|
983
|
+
):
|
|
984
|
+
raise OfficialArenaError("tau3 provider config fields are invalid")
|
|
985
|
+
for name in (
|
|
986
|
+
"task_set_name",
|
|
987
|
+
"task_split_name",
|
|
988
|
+
"llm_agent",
|
|
989
|
+
"llm_user",
|
|
990
|
+
"retrieval_config",
|
|
991
|
+
):
|
|
992
|
+
if not isinstance(config[name], str) or not config[name].strip():
|
|
993
|
+
raise OfficialArenaError(f"tau3 provider {name} is invalid")
|
|
994
|
+
for name in ("llm_args_agent", "llm_args_user", "retrieval_config_kwargs"):
|
|
995
|
+
if not isinstance(config[name], Mapping):
|
|
996
|
+
raise OfficialArenaError(f"tau3 provider {name} must be an object")
|
|
997
|
+
for name in ("limit", "max_steps", "max_errors"):
|
|
998
|
+
value = config[name]
|
|
999
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|
1000
|
+
raise OfficialArenaError(f"tau3 provider {name} must be positive")
|
|
1001
|
+
timeout = config["timeout"]
|
|
1002
|
+
if timeout is not None and (
|
|
1003
|
+
isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or timeout <= 0
|
|
1004
|
+
):
|
|
1005
|
+
raise OfficialArenaError("tau3 provider timeout is invalid")
|
|
1006
|
+
_capture_attempt_policy(
|
|
1007
|
+
config.get("capture_max_attempts", 1),
|
|
1008
|
+
config.get("capture_text_only_fallback", False),
|
|
1009
|
+
)
|
|
1010
|
+
if config.get("capture_text_only_fallback", False) and not _is_gpt_oss_model_name(config["llm_user"]):
|
|
1011
|
+
raise OfficialArenaError("tau3 text-only capture fallback requires GPT-OSS")
|
|
1012
|
+
if "capture_task_ids" in config:
|
|
1013
|
+
selected = config["capture_task_ids"]
|
|
1014
|
+
if (
|
|
1015
|
+
not isinstance(selected, list)
|
|
1016
|
+
or not selected
|
|
1017
|
+
or len(selected) > config["limit"]
|
|
1018
|
+
or any(not isinstance(value, str) or not value.strip() for value in selected)
|
|
1019
|
+
or len(set(selected)) != len(selected)
|
|
1020
|
+
):
|
|
1021
|
+
raise OfficialArenaError("tau3 capture_task_ids is invalid")
|
|
1022
|
+
validate_dependency_hashes(
|
|
1023
|
+
config["locked_dependency_hashes"],
|
|
1024
|
+
label="tau3",
|
|
1025
|
+
)
|
|
1026
|
+
if "retriever_override" in config:
|
|
1027
|
+
_validate_retriever_override(config["retriever_override"])
|
|
1028
|
+
return config
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
_RETRIEVER_OVERRIDE_FIELDS = frozenset(
|
|
1032
|
+
{
|
|
1033
|
+
"profile_id",
|
|
1034
|
+
"backend",
|
|
1035
|
+
"model_path",
|
|
1036
|
+
"model_revision",
|
|
1037
|
+
"model_artifact_hash",
|
|
1038
|
+
"query_prefix",
|
|
1039
|
+
"document_prefix",
|
|
1040
|
+
"expected_dimensions",
|
|
1041
|
+
"top_k",
|
|
1042
|
+
"device",
|
|
1043
|
+
"prompt_template",
|
|
1044
|
+
"prompt_sha256",
|
|
1045
|
+
}
|
|
1046
|
+
)
|
|
1047
|
+
_LOCAL_ENCODERS: dict[str, Any] = {}
|
|
1048
|
+
RETRIEVER_OVERRIDE_PATH_ENV = "AILAB_TAU3_RETRIEVER_OVERRIDE_PATH"
|
|
1049
|
+
RETRIEVER_OVERRIDE_SHA256_ENV = "AILAB_TAU3_RETRIEVER_OVERRIDE_SHA256"
|
|
1050
|
+
_EMPTY_ASSISTANT_RESPONSE = "[AgentInputLab: empty assistant response]"
|
|
1051
|
+
_CAPTURE_MAX_ATTEMPTS = 16
|
|
1052
|
+
_CAPTURE_MAX_STEPS = 32
|
|
1053
|
+
_CAPTURE_RETRY_SEED_STRIDE = 1_000_003
|
|
1054
|
+
_GPT_OSS_CAPTURE_FALLBACK_ATTEMPTS = 1
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def _capture_attempt_policy(max_attempts: int, allow_text_only_fallback: bool) -> tuple[int, bool]:
|
|
1058
|
+
"""Validate explicit preparation policies before building a simulator."""
|
|
1059
|
+
|
|
1060
|
+
if type(allow_text_only_fallback) is not bool:
|
|
1061
|
+
raise OfficialArenaError("tau3 capture_text_only_fallback must be boolean")
|
|
1062
|
+
maximum = _CAPTURE_MAX_ATTEMPTS + (
|
|
1063
|
+
_GPT_OSS_CAPTURE_FALLBACK_ATTEMPTS if allow_text_only_fallback else 0
|
|
1064
|
+
)
|
|
1065
|
+
if type(max_attempts) is not int or not 1 <= max_attempts <= maximum:
|
|
1066
|
+
raise OfficialArenaError("tau3 capture_max_attempts is invalid")
|
|
1067
|
+
if allow_text_only_fallback and max_attempts != maximum:
|
|
1068
|
+
raise OfficialArenaError("tau3 capture_text_only_fallback requires explicit 17-attempt preparation")
|
|
1069
|
+
return max_attempts, allow_text_only_fallback
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def _capture_retry_seed(base_seed: int, attempt: int) -> int:
|
|
1073
|
+
"""Derive a distinct deterministic seed for a simulator-only capture retry."""
|
|
1074
|
+
|
|
1075
|
+
if attempt < 0:
|
|
1076
|
+
raise ValueError("capture attempt must be non-negative")
|
|
1077
|
+
return base_seed + attempt * _CAPTURE_RETRY_SEED_STRIDE
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def _capture_initial_visible_user(
|
|
1081
|
+
*,
|
|
1082
|
+
build_orchestrator: Any,
|
|
1083
|
+
run_config: Any,
|
|
1084
|
+
task: Any,
|
|
1085
|
+
base_seed: int,
|
|
1086
|
+
assistant_message_type: type,
|
|
1087
|
+
user_message_type: type,
|
|
1088
|
+
max_attempts: int = 1,
|
|
1089
|
+
allow_text_only_fallback: bool = False,
|
|
1090
|
+
) -> tuple[Any, Any, int, int]:
|
|
1091
|
+
"""Capture a visible initial user turn; default to one simulator attempt.
|
|
1092
|
+
|
|
1093
|
+
The benchmark user simulator occasionally emits a long environment-tool
|
|
1094
|
+
preamble, or malformed tool-argument JSON, before the evaluated agent is
|
|
1095
|
+
allowed to run. Those are capture-infrastructure failures, not treatment
|
|
1096
|
+
outcomes. Additional preparation attempts require an explicit bound and
|
|
1097
|
+
retain deterministic attempt/seed metadata. Removing the simulator's tools
|
|
1098
|
+
is a separate explicit opt-in variant, never an automatic repair. Count
|
|
1099
|
+
all native user/agent/judge generations separately: an attempt counter is
|
|
1100
|
+
not token/cost telemetry, and missing usage is not free preparation.
|
|
1101
|
+
"""
|
|
1102
|
+
|
|
1103
|
+
attempts, allow_text_only_fallback = _capture_attempt_policy(max_attempts, allow_text_only_fallback)
|
|
1104
|
+
last_failure: Exception | None = None
|
|
1105
|
+
for attempt in range(attempts):
|
|
1106
|
+
capture_seed = _capture_retry_seed(base_seed, attempt)
|
|
1107
|
+
orchestrator = build_orchestrator(
|
|
1108
|
+
run_config,
|
|
1109
|
+
task,
|
|
1110
|
+
seed=capture_seed,
|
|
1111
|
+
simulation_id=f"agentinputlab-capture-{task.id}-attempt-{attempt + 1}",
|
|
1112
|
+
)
|
|
1113
|
+
capture_user = getattr(orchestrator, "user", None)
|
|
1114
|
+
capture_model = str(getattr(capture_user, "llm", ""))
|
|
1115
|
+
if allow_text_only_fallback and (
|
|
1116
|
+
not _is_gpt_oss_model_name(capture_model) or not hasattr(capture_user, "tools")
|
|
1117
|
+
):
|
|
1118
|
+
orchestrator._cleanup()
|
|
1119
|
+
raise OfficialArenaError("tau3 text-only capture fallback requires a GPT-OSS simulator with tools")
|
|
1120
|
+
text_only_fallback = allow_text_only_fallback and attempt >= _CAPTURE_MAX_ATTEMPTS
|
|
1121
|
+
if text_only_fallback:
|
|
1122
|
+
# GPT-OSS can deterministically loop on user-only environment tools
|
|
1123
|
+
# before emitting the first visible request. After the normal,
|
|
1124
|
+
# bounded capture window is exhausted, make one final simulator
|
|
1125
|
+
# call without those pre-agent tools. The user scenario remains
|
|
1126
|
+
# unchanged, but this is an explicit preparation variant, not the
|
|
1127
|
+
# default capture policy or an evaluated-agent retry.
|
|
1128
|
+
capture_user.tools = []
|
|
1129
|
+
try:
|
|
1130
|
+
orchestrator.initialize()
|
|
1131
|
+
for _capture_step in range(_CAPTURE_MAX_STEPS):
|
|
1132
|
+
orchestrator.step()
|
|
1133
|
+
candidate = orchestrator.message
|
|
1134
|
+
if isinstance(candidate, user_message_type) and _is_replayable_initial_user_message(
|
|
1135
|
+
candidate
|
|
1136
|
+
):
|
|
1137
|
+
return orchestrator, candidate, attempt + 1, capture_seed
|
|
1138
|
+
if isinstance(candidate, assistant_message_type):
|
|
1139
|
+
raise OfficialArenaError(
|
|
1140
|
+
"tau3 agent executed before the first visible user message"
|
|
1141
|
+
)
|
|
1142
|
+
last_failure = OfficialArenaError(
|
|
1143
|
+
"tau3 capture did not produce an initial visible user message"
|
|
1144
|
+
)
|
|
1145
|
+
except json.JSONDecodeError as error:
|
|
1146
|
+
last_failure = error
|
|
1147
|
+
except Exception:
|
|
1148
|
+
orchestrator._cleanup()
|
|
1149
|
+
raise
|
|
1150
|
+
orchestrator._cleanup()
|
|
1151
|
+
raise OfficialArenaError(
|
|
1152
|
+
"tau3 capture did not produce an initial visible user message "
|
|
1153
|
+
f"after {attempts} simulator attempts"
|
|
1154
|
+
) from last_failure
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def _is_gpt_oss_model_name(model_name: str) -> bool:
|
|
1158
|
+
return model_name.casefold().removeprefix("openai/") == "gpt-oss-20b"
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
def _is_gpt_oss_generation_boundary_failure(error: Exception, *, model_name: str) -> bool:
|
|
1162
|
+
"""Recognize model-output failures that should become scored hard stops."""
|
|
1163
|
+
|
|
1164
|
+
if not _is_gpt_oss_model_name(model_name):
|
|
1165
|
+
return False
|
|
1166
|
+
text = str(error).casefold()
|
|
1167
|
+
return (
|
|
1168
|
+
"unexpected tokens remaining in message header" in text
|
|
1169
|
+
or "unexpected token " in text
|
|
1170
|
+
or ("input length" in text and "exceeds model's maximum context length" in text)
|
|
1171
|
+
)
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _file_sha256(path: Path) -> str:
|
|
1175
|
+
digest = hashlib.sha256()
|
|
1176
|
+
with path.open("rb") as handle:
|
|
1177
|
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
1178
|
+
digest.update(block)
|
|
1179
|
+
return digest.hexdigest()
|
|
1180
|
+
|
|
1181
|
+
|
|
1182
|
+
def _validate_retriever_override(value: Any) -> Mapping[str, Any]:
|
|
1183
|
+
if not isinstance(value, Mapping) or set(value) != _RETRIEVER_OVERRIDE_FIELDS:
|
|
1184
|
+
raise OfficialArenaError("tau3 retriever_override fields are invalid")
|
|
1185
|
+
if value["backend"] not in {"bm25", "sentence_transformers_dense"}:
|
|
1186
|
+
raise OfficialArenaError("tau3 retriever_override backend is invalid")
|
|
1187
|
+
for name in ("profile_id", "model_artifact_hash", "prompt_template", "prompt_sha256"):
|
|
1188
|
+
if not isinstance(value[name], str) or not value[name]:
|
|
1189
|
+
raise OfficialArenaError(f"tau3 retriever_override {name} is invalid")
|
|
1190
|
+
for name in ("model_artifact_hash", "prompt_sha256"):
|
|
1191
|
+
if len(value[name]) != 64 or any(
|
|
1192
|
+
character not in "0123456789abcdef" for character in value[name]
|
|
1193
|
+
):
|
|
1194
|
+
raise OfficialArenaError(f"tau3 retriever_override {name} is not SHA-256")
|
|
1195
|
+
for name in ("top_k", "expected_dimensions"):
|
|
1196
|
+
if isinstance(value[name], bool) or not isinstance(value[name], int) or value[name] < 1:
|
|
1197
|
+
raise OfficialArenaError(f"tau3 retriever_override {name} must be positive")
|
|
1198
|
+
raw_prompt = Path(str(value["prompt_template"])).expanduser()
|
|
1199
|
+
prompt = raw_prompt.resolve()
|
|
1200
|
+
if (
|
|
1201
|
+
not prompt.is_file()
|
|
1202
|
+
or raw_prompt.is_symlink()
|
|
1203
|
+
or _file_sha256(prompt) != value["prompt_sha256"]
|
|
1204
|
+
):
|
|
1205
|
+
raise OfficialArenaError("tau3 retriever_override prompt is absent or changed")
|
|
1206
|
+
if value["backend"] == "sentence_transformers_dense":
|
|
1207
|
+
model_path = Path(str(value["model_path"])).expanduser().resolve()
|
|
1208
|
+
if not model_path.is_dir() or not value["model_revision"] or not value["device"]:
|
|
1209
|
+
raise OfficialArenaError("tau3 dense retriever_override model binding is invalid")
|
|
1210
|
+
return value
|
|
1211
|
+
|
|
1212
|
+
|
|
1213
|
+
def resolve_retriever_config(config: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
1214
|
+
"""Resolve a hash-bound process-level τ³ retriever factor.
|
|
1215
|
+
|
|
1216
|
+
Every main-arena method runner uses this provider. The two-variable
|
|
1217
|
+
binding therefore lets RQ4 vary the shared downstream retrieval dependency
|
|
1218
|
+
without changing method logic or editing the pinned tau2 checkout.
|
|
1219
|
+
"""
|
|
1220
|
+
|
|
1221
|
+
raw_path = os.environ.get(RETRIEVER_OVERRIDE_PATH_ENV)
|
|
1222
|
+
raw_sha256 = os.environ.get(RETRIEVER_OVERRIDE_SHA256_ENV)
|
|
1223
|
+
if not raw_path and not raw_sha256:
|
|
1224
|
+
return config
|
|
1225
|
+
if not raw_path or not raw_sha256:
|
|
1226
|
+
raise OfficialArenaError("tau3 retriever override environment binding is incomplete")
|
|
1227
|
+
unresolved = Path(raw_path).expanduser()
|
|
1228
|
+
path = unresolved.resolve()
|
|
1229
|
+
if unresolved.is_symlink() or not path.is_file() or _file_sha256(path) != raw_sha256:
|
|
1230
|
+
raise OfficialArenaError("tau3 retriever override environment artifact differs")
|
|
1231
|
+
try:
|
|
1232
|
+
loaded = json.loads(path.read_text(encoding="utf-8"))
|
|
1233
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
1234
|
+
raise OfficialArenaError("tau3 retriever override artifact is invalid JSON") from exc
|
|
1235
|
+
override = _validate_retriever_override(loaded)
|
|
1236
|
+
existing = config.get("retriever_override")
|
|
1237
|
+
if existing is not None and existing != override:
|
|
1238
|
+
raise OfficialArenaError("tau3 explicit and environment retriever overrides differ")
|
|
1239
|
+
variant_name = (
|
|
1240
|
+
"agentinputlab_rq4_"
|
|
1241
|
+
+ str(override["profile_id"]).replace(".", "_")
|
|
1242
|
+
+ f"_k{override['top_k']}"
|
|
1243
|
+
)
|
|
1244
|
+
return {
|
|
1245
|
+
**config,
|
|
1246
|
+
"retrieval_config": variant_name,
|
|
1247
|
+
"retrieval_config_kwargs": {"top_k": int(override["top_k"])},
|
|
1248
|
+
"retriever_override": override,
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
def _normalized_tool_arguments(value: object) -> str | None:
|
|
1253
|
+
"""Return canonical object JSON for normal or double-encoded arguments."""
|
|
1254
|
+
|
|
1255
|
+
if not isinstance(value, str):
|
|
1256
|
+
return None
|
|
1257
|
+
candidate = value
|
|
1258
|
+
for _ in range(4):
|
|
1259
|
+
try:
|
|
1260
|
+
parsed = json.loads(candidate)
|
|
1261
|
+
except (TypeError, ValueError):
|
|
1262
|
+
return None
|
|
1263
|
+
if isinstance(parsed, Mapping):
|
|
1264
|
+
return json.dumps(parsed, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
|
1265
|
+
if isinstance(parsed, str):
|
|
1266
|
+
candidate = parsed
|
|
1267
|
+
continue
|
|
1268
|
+
return None
|
|
1269
|
+
return None
|
|
1270
|
+
|
|
1271
|
+
|
|
1272
|
+
def _normalized_tool_name(value: object, *, advertised_tool_names: set[str]) -> str | None:
|
|
1273
|
+
"""Strip only a leaked Harmony channel suffix from an advertised tool name."""
|
|
1274
|
+
|
|
1275
|
+
if not isinstance(value, str) or "<|channel|>" not in value:
|
|
1276
|
+
return None
|
|
1277
|
+
candidate = value.split("<|channel|>", 1)[0]
|
|
1278
|
+
if candidate and candidate in advertised_tool_names:
|
|
1279
|
+
return candidate
|
|
1280
|
+
return None
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
def _openai_tool_names(tools: object) -> set[str]:
|
|
1284
|
+
if isinstance(tools, (str, bytes)) or not isinstance(tools, (list, tuple)):
|
|
1285
|
+
return set()
|
|
1286
|
+
names: set[str] = set()
|
|
1287
|
+
for tool in tools:
|
|
1288
|
+
if not isinstance(tool, Mapping):
|
|
1289
|
+
continue
|
|
1290
|
+
function = tool.get("function")
|
|
1291
|
+
if not isinstance(function, Mapping):
|
|
1292
|
+
continue
|
|
1293
|
+
name = function.get("name")
|
|
1294
|
+
if isinstance(name, str) and name:
|
|
1295
|
+
names.add(name)
|
|
1296
|
+
return names
|
|
1297
|
+
|
|
1298
|
+
|
|
1299
|
+
def _normalize_empty_assistant_message(message: Any) -> bool:
|
|
1300
|
+
"""Make an empty OpenAI assistant turn explicit for the pinned tau2 runtime.
|
|
1301
|
+
|
|
1302
|
+
Some local OpenAI-compatible models emit a successful completion whose
|
|
1303
|
+
assistant message contains neither text nor tool calls. The model event is
|
|
1304
|
+
valid at the transport layer, but tau2 rejects it before the official
|
|
1305
|
+
scorer can assign a task outcome. Preserve any refusal text and otherwise
|
|
1306
|
+
insert a deterministic, visibly synthetic sentinel so the trajectory is
|
|
1307
|
+
scored as produced instead of disappearing from the experiment matrix.
|
|
1308
|
+
|
|
1309
|
+
The function returns whether it changed the response. It deliberately
|
|
1310
|
+
does not alter non-empty content, tool calls, finish reasons, or usage.
|
|
1311
|
+
"""
|
|
1312
|
+
|
|
1313
|
+
if message is None or getattr(message, "tool_calls", ()):
|
|
1314
|
+
return False
|
|
1315
|
+
content = getattr(message, "content", None)
|
|
1316
|
+
if content is not None and (not isinstance(content, str) or content.strip()):
|
|
1317
|
+
return False
|
|
1318
|
+
refusal = getattr(message, "refusal", None)
|
|
1319
|
+
message.content = (
|
|
1320
|
+
refusal if isinstance(refusal, str) and refusal.strip() else _EMPTY_ASSISTANT_RESPONSE
|
|
1321
|
+
)
|
|
1322
|
+
return True
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _install_tau_tool_argument_normalizer() -> None:
|
|
1326
|
+
"""Bridge vLLM/OpenAI tool arguments to tau2's dictionary contract.
|
|
1327
|
+
|
|
1328
|
+
OpenAI-compatible servers serialize function arguments as JSON text. A
|
|
1329
|
+
few valid generations quote that JSON one additional time. The pinned
|
|
1330
|
+
tau2 runtime decodes exactly once and then validates a dictionary, so the
|
|
1331
|
+
double-encoded form otherwise aborts an official trajectory. Normalize
|
|
1332
|
+
only this transport representation; tool name, values, and scorer state
|
|
1333
|
+
remain unchanged.
|
|
1334
|
+
"""
|
|
1335
|
+
|
|
1336
|
+
import tau2.utils.llm_utils as llm_utils
|
|
1337
|
+
|
|
1338
|
+
completion = llm_utils.completion
|
|
1339
|
+
if getattr(completion, "_agentinputlab_tool_argument_normalizer", False):
|
|
1340
|
+
return
|
|
1341
|
+
|
|
1342
|
+
def normalized_completion(*args: Any, **kwargs: Any) -> Any:
|
|
1343
|
+
response = completion(*args, **kwargs)
|
|
1344
|
+
advertised_tool_names = _openai_tool_names(kwargs.get("tools"))
|
|
1345
|
+
choices = getattr(response, "choices", ()) or ()
|
|
1346
|
+
for choice in choices:
|
|
1347
|
+
message = getattr(choice, "message", None)
|
|
1348
|
+
_normalize_empty_assistant_message(message)
|
|
1349
|
+
for tool_call in getattr(message, "tool_calls", ()) or ():
|
|
1350
|
+
function = getattr(tool_call, "function", None)
|
|
1351
|
+
raw_name = getattr(function, "name", None)
|
|
1352
|
+
normalized_name = _normalized_tool_name(
|
|
1353
|
+
raw_name,
|
|
1354
|
+
advertised_tool_names=advertised_tool_names,
|
|
1355
|
+
)
|
|
1356
|
+
if normalized_name is not None:
|
|
1357
|
+
function.name = normalized_name
|
|
1358
|
+
raw = getattr(function, "arguments", None)
|
|
1359
|
+
normalized = _normalized_tool_arguments(raw)
|
|
1360
|
+
if normalized is not None:
|
|
1361
|
+
function.arguments = normalized
|
|
1362
|
+
return response
|
|
1363
|
+
|
|
1364
|
+
normalized_completion._agentinputlab_tool_argument_normalizer = True # type: ignore[attr-defined]
|
|
1365
|
+
llm_utils.completion = normalized_completion
|
|
1366
|
+
|
|
1367
|
+
|
|
1368
|
+
def _install_agentinputlab_retriever(config: Mapping[str, Any]) -> None:
|
|
1369
|
+
raw = config.get("retriever_override")
|
|
1370
|
+
if raw is None:
|
|
1371
|
+
return
|
|
1372
|
+
override = _validate_retriever_override(raw)
|
|
1373
|
+
from tau2.domains.banking_knowledge.retrieval import (
|
|
1374
|
+
RETRIEVAL_VARIANTS,
|
|
1375
|
+
PipelineSpec,
|
|
1376
|
+
RetrievalVariant,
|
|
1377
|
+
standard_prompt,
|
|
1378
|
+
)
|
|
1379
|
+
|
|
1380
|
+
variant_name = str(config["retrieval_config"])
|
|
1381
|
+
prompt = Path(str(override["prompt_template"])).expanduser().resolve()
|
|
1382
|
+
if override["backend"] == "bm25":
|
|
1383
|
+
spec = PipelineSpec(type="bm25", top_k=int(override["top_k"]))
|
|
1384
|
+
else:
|
|
1385
|
+
from tau2.knowledge.document_preprocessors.embedding_indexer import (
|
|
1386
|
+
EMBEDDER_REGISTRY as document_registry,
|
|
1387
|
+
)
|
|
1388
|
+
from tau2.knowledge.input_preprocessors.embedding_encoder import (
|
|
1389
|
+
EMBEDDER_REGISTRY as query_registry,
|
|
1390
|
+
)
|
|
1391
|
+
|
|
1392
|
+
registry_key = "agentinputlab_" + str(override["model_artifact_hash"])[:16]
|
|
1393
|
+
|
|
1394
|
+
def embedder_class(*, prefix: str) -> type[Any]:
|
|
1395
|
+
class LocalSentenceTransformerEmbedder:
|
|
1396
|
+
def __init__(self, model: str):
|
|
1397
|
+
if model != override["profile_id"]:
|
|
1398
|
+
raise ValueError("tau3 local retriever profile marker differs")
|
|
1399
|
+
self.query_instruction = prefix
|
|
1400
|
+
identity = str(override["model_artifact_hash"])
|
|
1401
|
+
encoder = _LOCAL_ENCODERS.get(identity)
|
|
1402
|
+
if encoder is None:
|
|
1403
|
+
from sentence_transformers import SentenceTransformer
|
|
1404
|
+
|
|
1405
|
+
encoder = SentenceTransformer(
|
|
1406
|
+
str(override["model_path"]),
|
|
1407
|
+
device=str(override["device"]),
|
|
1408
|
+
local_files_only=True,
|
|
1409
|
+
trust_remote_code=True,
|
|
1410
|
+
)
|
|
1411
|
+
_LOCAL_ENCODERS[identity] = encoder
|
|
1412
|
+
self._encoder = encoder
|
|
1413
|
+
|
|
1414
|
+
def embed(self, texts: list[str]) -> Any:
|
|
1415
|
+
values = [prefix + text for text in texts]
|
|
1416
|
+
vectors = self._encoder.encode(
|
|
1417
|
+
values,
|
|
1418
|
+
batch_size=min(16, max(1, len(values))),
|
|
1419
|
+
normalize_embeddings=True,
|
|
1420
|
+
show_progress_bar=False,
|
|
1421
|
+
convert_to_numpy=True,
|
|
1422
|
+
)
|
|
1423
|
+
if vectors.ndim != 2 or vectors.shape[1] != override["expected_dimensions"]:
|
|
1424
|
+
raise ValueError("tau3 local retriever returned unexpected dimensions")
|
|
1425
|
+
return vectors
|
|
1426
|
+
|
|
1427
|
+
def get_name(self) -> str:
|
|
1428
|
+
return str(override["profile_id"])
|
|
1429
|
+
|
|
1430
|
+
return LocalSentenceTransformerEmbedder
|
|
1431
|
+
|
|
1432
|
+
document_registry[registry_key] = embedder_class(prefix=str(override["document_prefix"]))
|
|
1433
|
+
query_registry[registry_key] = embedder_class(prefix=str(override["query_prefix"]))
|
|
1434
|
+
spec = PipelineSpec(
|
|
1435
|
+
type="embedding",
|
|
1436
|
+
embedder_type=registry_key,
|
|
1437
|
+
embedder_model=str(override["profile_id"]),
|
|
1438
|
+
top_k=int(override["top_k"]),
|
|
1439
|
+
)
|
|
1440
|
+
RETRIEVAL_VARIANTS[variant_name] = RetrievalVariant(
|
|
1441
|
+
name=variant_name,
|
|
1442
|
+
prompt_template=prompt,
|
|
1443
|
+
build_prompt=standard_prompt,
|
|
1444
|
+
kb_search=spec,
|
|
1445
|
+
supports_top_k=True,
|
|
1446
|
+
)
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
def _intervened_tau_agent_class() -> type[Any]:
|
|
1450
|
+
from tau2.agent.llm_agent import LLMAgent, LLMAgentState
|
|
1451
|
+
from tau2.data_model.message import (
|
|
1452
|
+
AssistantMessage,
|
|
1453
|
+
MultiToolMessage,
|
|
1454
|
+
UserMessage,
|
|
1455
|
+
)
|
|
1456
|
+
from tau2.utils.llm_utils import generate
|
|
1457
|
+
|
|
1458
|
+
class IntervenedTauAgent(LLMAgent[LLMAgentState]):
|
|
1459
|
+
def __init__(
|
|
1460
|
+
self,
|
|
1461
|
+
*args: Any,
|
|
1462
|
+
materialized: Mapping[str, Any],
|
|
1463
|
+
budget: Mapping[str, Any],
|
|
1464
|
+
history_transform: Any = None,
|
|
1465
|
+
**kwargs: Any,
|
|
1466
|
+
):
|
|
1467
|
+
super().__init__(*args, **kwargs)
|
|
1468
|
+
self._materialized = materialized
|
|
1469
|
+
self._first_turn = True
|
|
1470
|
+
self._budget = budget
|
|
1471
|
+
self._budget_stopped = False
|
|
1472
|
+
self._budget_model_calls = 0
|
|
1473
|
+
self._budget_tool_calls = 0
|
|
1474
|
+
self._budget_input_tokens = 0
|
|
1475
|
+
self._budget_output_tokens = 0
|
|
1476
|
+
self._budget_hard_stops = 0
|
|
1477
|
+
self._budget_discarded_tool_calls = 0
|
|
1478
|
+
self._history_session = (NativeHistorySession(history_transform, materialized, budget)
|
|
1479
|
+
if history_transform is not None else None)
|
|
1480
|
+
self._budget_max_context_tokens = (
|
|
1481
|
+
2**63 - 1
|
|
1482
|
+
if budget.get("max_context_tokens") is None
|
|
1483
|
+
else int(budget["max_context_tokens"])
|
|
1484
|
+
)
|
|
1485
|
+
|
|
1486
|
+
def _history_target_usage(self):
|
|
1487
|
+
return dict(model_calls=self._budget_model_calls, input_tokens=self._budget_input_tokens,
|
|
1488
|
+
output_tokens=self._budget_output_tokens, tool_calls=self._budget_tool_calls)
|
|
1489
|
+
|
|
1490
|
+
@property
|
|
1491
|
+
def system_prompt(self) -> str:
|
|
1492
|
+
base = super().system_prompt
|
|
1493
|
+
additions = [self._materialized.get("instruction", "")]
|
|
1494
|
+
additions.extend(
|
|
1495
|
+
block.get("text", "")
|
|
1496
|
+
for block in self._materialized.get("context", [])
|
|
1497
|
+
if isinstance(block, Mapping)
|
|
1498
|
+
)
|
|
1499
|
+
visible = "\n\n".join(item for item in additions if item)
|
|
1500
|
+
return base + (
|
|
1501
|
+
"\n\n<optimizer_visible_input>\n" + visible + "\n</optimizer_visible_input>"
|
|
1502
|
+
if visible
|
|
1503
|
+
else ""
|
|
1504
|
+
)
|
|
1505
|
+
|
|
1506
|
+
def generate_next_message(
|
|
1507
|
+
self,
|
|
1508
|
+
message: Any,
|
|
1509
|
+
state: LLMAgentState,
|
|
1510
|
+
) -> tuple[AssistantMessage, LLMAgentState]:
|
|
1511
|
+
metadata = self._materialized.get("metadata", {})
|
|
1512
|
+
clarification = (
|
|
1513
|
+
metadata.get("clarification_request") if isinstance(metadata, Mapping) else None
|
|
1514
|
+
)
|
|
1515
|
+
if self._first_turn and isinstance(clarification, str) and clarification.strip():
|
|
1516
|
+
self._first_turn = False
|
|
1517
|
+
if message is not None:
|
|
1518
|
+
state.messages.append(message)
|
|
1519
|
+
response = AssistantMessage(
|
|
1520
|
+
role="assistant",
|
|
1521
|
+
content=clarification,
|
|
1522
|
+
cost=0.0,
|
|
1523
|
+
)
|
|
1524
|
+
state.messages.append(response)
|
|
1525
|
+
return response, state
|
|
1526
|
+
if self._first_turn and isinstance(message, UserMessage):
|
|
1527
|
+
message = message.model_copy(
|
|
1528
|
+
update={"content": self._materialized["query"]},
|
|
1529
|
+
deep=True,
|
|
1530
|
+
)
|
|
1531
|
+
self._first_turn = False
|
|
1532
|
+
return super().generate_next_message(message, state)
|
|
1533
|
+
|
|
1534
|
+
def _generate_next_message(
|
|
1535
|
+
self,
|
|
1536
|
+
message: Any,
|
|
1537
|
+
state: LLMAgentState,
|
|
1538
|
+
) -> AssistantMessage:
|
|
1539
|
+
if isinstance(message, MultiToolMessage):
|
|
1540
|
+
state.messages.extend(message.tool_messages)
|
|
1541
|
+
else:
|
|
1542
|
+
state.messages.append(message)
|
|
1543
|
+
if self._budget_stopped:
|
|
1544
|
+
return AssistantMessage(
|
|
1545
|
+
role="assistant",
|
|
1546
|
+
content=_SHARED_BUDGET_STOP_MESSAGE,
|
|
1547
|
+
cost=0.0,
|
|
1548
|
+
)
|
|
1549
|
+
messages = state.system_messages + state.messages
|
|
1550
|
+
if self._history_session is not None:
|
|
1551
|
+
remaining = self._history_session.remaining(self._history_target_usage())
|
|
1552
|
+
if remaining["max_model_calls"] < 1 or remaining["max_output_tokens"] < 1:
|
|
1553
|
+
self._budget_stopped = True
|
|
1554
|
+
self._budget_hard_stops += 1
|
|
1555
|
+
return AssistantMessage(role="assistant", content=_SHARED_BUDGET_STOP_MESSAGE, cost=0.0)
|
|
1556
|
+
from tau2.utils.llm_utils import to_litellm_messages
|
|
1557
|
+
messages = self._history_session.messages(messages, encode=to_litellm_messages,
|
|
1558
|
+
agent_usage=self._history_target_usage())
|
|
1559
|
+
try:
|
|
1560
|
+
from litellm import token_counter
|
|
1561
|
+
|
|
1562
|
+
normalized_messages = [
|
|
1563
|
+
item.model_dump(mode="json") if hasattr(item, "model_dump") else item
|
|
1564
|
+
for item in messages
|
|
1565
|
+
]
|
|
1566
|
+
# tau2's internal tool-call objects are not OpenAI-shaped.
|
|
1567
|
+
# Count their canonical JSON as plain text so every argument,
|
|
1568
|
+
# result, system message, and tool schema remains included
|
|
1569
|
+
# without asking LiteLLM to reinterpret the structure.
|
|
1570
|
+
token_messages = [
|
|
1571
|
+
{
|
|
1572
|
+
"role": "user",
|
|
1573
|
+
"content": json.dumps(
|
|
1574
|
+
{
|
|
1575
|
+
"messages": normalized_messages,
|
|
1576
|
+
"tools": [tool.openai_schema for tool in self.tools],
|
|
1577
|
+
},
|
|
1578
|
+
ensure_ascii=False,
|
|
1579
|
+
sort_keys=True,
|
|
1580
|
+
),
|
|
1581
|
+
},
|
|
1582
|
+
]
|
|
1583
|
+
next_input_tokens = int(token_counter(model=self.llm, messages=token_messages))
|
|
1584
|
+
except Exception as exc:
|
|
1585
|
+
raise OfficialArenaError(
|
|
1586
|
+
"tau3 could not measure input tokens before generation"
|
|
1587
|
+
) from exc
|
|
1588
|
+
history_usage = self._history_session.usage if self._history_session is not None else {}
|
|
1589
|
+
history_calls = int(history_usage.get("model_calls", 0))
|
|
1590
|
+
history_input = int(history_usage.get("input_tokens", 0))
|
|
1591
|
+
history_output = int(history_usage.get("output_tokens", 0))
|
|
1592
|
+
remaining_output = int(self._budget["max_output_tokens"]) - self._budget_output_tokens - history_output
|
|
1593
|
+
if (
|
|
1594
|
+
self._budget_model_calls + history_calls >= int(self._budget["max_model_calls"])
|
|
1595
|
+
or self._budget_input_tokens + history_input + next_input_tokens > self._budget_max_context_tokens
|
|
1596
|
+
or remaining_output < 1
|
|
1597
|
+
):
|
|
1598
|
+
self._budget_stopped = True
|
|
1599
|
+
self._budget_hard_stops += 1
|
|
1600
|
+
return AssistantMessage(
|
|
1601
|
+
role="assistant",
|
|
1602
|
+
content=_SHARED_BUDGET_STOP_MESSAGE,
|
|
1603
|
+
cost=0.0,
|
|
1604
|
+
)
|
|
1605
|
+
call_args = dict(self.llm_args)
|
|
1606
|
+
call_args["max_tokens"] = min(
|
|
1607
|
+
int(call_args.get("max_tokens", remaining_output)),
|
|
1608
|
+
remaining_output,
|
|
1609
|
+
)
|
|
1610
|
+
try:
|
|
1611
|
+
response = generate(
|
|
1612
|
+
model=self.llm,
|
|
1613
|
+
tools=self.tools,
|
|
1614
|
+
messages=messages,
|
|
1615
|
+
call_name="agent_response",
|
|
1616
|
+
**call_args,
|
|
1617
|
+
)
|
|
1618
|
+
except Exception as error:
|
|
1619
|
+
if not _is_gpt_oss_generation_boundary_failure(
|
|
1620
|
+
error,
|
|
1621
|
+
model_name=self.llm,
|
|
1622
|
+
):
|
|
1623
|
+
raise
|
|
1624
|
+
# A malformed Harmony header or a request beyond the serving
|
|
1625
|
+
# context window is an evaluated trajectory outcome, not a
|
|
1626
|
+
# missing experiment row. Preserve it as the same scored hard
|
|
1627
|
+
# stop used for an exhausted shared budget.
|
|
1628
|
+
self._budget_model_calls += 1
|
|
1629
|
+
self._budget_input_tokens += next_input_tokens
|
|
1630
|
+
self._budget_stopped = True
|
|
1631
|
+
self._budget_hard_stops += 1
|
|
1632
|
+
return AssistantMessage(
|
|
1633
|
+
role="assistant",
|
|
1634
|
+
content=_SHARED_BUDGET_STOP_MESSAGE,
|
|
1635
|
+
cost=0.0,
|
|
1636
|
+
)
|
|
1637
|
+
usage = getattr(response, "usage", None)
|
|
1638
|
+
if isinstance(usage, Mapping):
|
|
1639
|
+
actual_input = int(usage.get("prompt_tokens", usage.get("input_tokens", 0)) or 0)
|
|
1640
|
+
actual_output = int(
|
|
1641
|
+
usage.get("completion_tokens", usage.get("output_tokens", 0)) or 0
|
|
1642
|
+
)
|
|
1643
|
+
else:
|
|
1644
|
+
actual_input = next_input_tokens
|
|
1645
|
+
actual_output = 0
|
|
1646
|
+
self._budget_model_calls += 1
|
|
1647
|
+
self._budget_input_tokens += actual_input
|
|
1648
|
+
self._budget_output_tokens += actual_output
|
|
1649
|
+
if (
|
|
1650
|
+
self._budget_input_tokens + history_input > self._budget_max_context_tokens
|
|
1651
|
+
or self._budget_output_tokens + history_output > int(self._budget["max_output_tokens"])
|
|
1652
|
+
):
|
|
1653
|
+
raise OfficialArenaError("tau3 provider exceeded its measured shared budget")
|
|
1654
|
+
response, kept_tool_calls, discarded_tool_calls = _apply_tool_call_budget(
|
|
1655
|
+
response,
|
|
1656
|
+
remaining_tool_calls=max(
|
|
1657
|
+
0,
|
|
1658
|
+
int(self._budget["max_tool_calls"]) - self._budget_tool_calls,
|
|
1659
|
+
),
|
|
1660
|
+
)
|
|
1661
|
+
self._budget_tool_calls += kept_tool_calls
|
|
1662
|
+
if discarded_tool_calls:
|
|
1663
|
+
self._budget_stopped = True
|
|
1664
|
+
self._budget_hard_stops += 1
|
|
1665
|
+
self._budget_discarded_tool_calls += discarded_tool_calls
|
|
1666
|
+
return response
|
|
1667
|
+
|
|
1668
|
+
return IntervenedTauAgent
|
|
1669
|
+
|
|
1670
|
+
|
|
1671
|
+
def factory() -> Tau3Provider:
|
|
1672
|
+
return Tau3Provider()
|