agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,960 @@
|
|
|
1
|
+
"""Production provider for the pinned Microsoft STATE-Bench runtime."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import hashlib
|
|
7
|
+
import os
|
|
8
|
+
import subprocess
|
|
9
|
+
from collections.abc import Mapping, Sequence
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ..protocol import (
|
|
14
|
+
OfficialArenaError,
|
|
15
|
+
ProviderArmResult,
|
|
16
|
+
content_hash,
|
|
17
|
+
intervention_usage,
|
|
18
|
+
materialized_state,
|
|
19
|
+
require_materialized_budget,
|
|
20
|
+
resolve_shared_budget_config,
|
|
21
|
+
validate_dependency_hashes,
|
|
22
|
+
visible_shared_budget,
|
|
23
|
+
)
|
|
24
|
+
from ._message_compat import (
|
|
25
|
+
fold_responses_system_messages_into_instructions,
|
|
26
|
+
normalize_qwen38_reasoning_effort,
|
|
27
|
+
requires_qwen38_initial_system_message,
|
|
28
|
+
)
|
|
29
|
+
from ._live_history import ResponsesHistorySession, validate_history_transform
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class _SharedBudgetStop(RuntimeError):
|
|
33
|
+
"""Internal, scored hard stop after a STATE-Bench resource cap is reached."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _EvaluatedAgentOutputInvalid(RuntimeError):
|
|
37
|
+
"""A recognized malformed model generation from the evaluated agent."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, error: Exception) -> None:
|
|
40
|
+
super().__init__(f"{type(error).__name__}: evaluated agent output is invalid")
|
|
41
|
+
self.error = error
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _model_output_invalid_result(
|
|
45
|
+
*,
|
|
46
|
+
replay_record: Mapping[str, Any],
|
|
47
|
+
role: str,
|
|
48
|
+
seed: int,
|
|
49
|
+
work_dir: Path,
|
|
50
|
+
error: Exception,
|
|
51
|
+
measured_usage: Mapping[str, int | float],
|
|
52
|
+
) -> ProviderArmResult:
|
|
53
|
+
"""Materialize malformed generated tool arguments as an official zero score."""
|
|
54
|
+
|
|
55
|
+
protected = replay_record["protected"]
|
|
56
|
+
trace = {
|
|
57
|
+
"provider": "state_bench",
|
|
58
|
+
"source_revision": StateBenchProvider.source_revision,
|
|
59
|
+
"protocol_id": protected["protocol_id"],
|
|
60
|
+
"task_id": protected["task_id"],
|
|
61
|
+
"domain": protected["domain"],
|
|
62
|
+
"role": role,
|
|
63
|
+
"seed": seed,
|
|
64
|
+
"termination_reason": "generated_tool_arguments_invalid",
|
|
65
|
+
"generation_controls": {
|
|
66
|
+
"hard_stops": 1,
|
|
67
|
+
"failure_type": type(error).__name__,
|
|
68
|
+
"failed_generation_scope": "evaluated_agent",
|
|
69
|
+
},
|
|
70
|
+
"score_provenance": "paper_model_output_invalid_zero_action",
|
|
71
|
+
}
|
|
72
|
+
work_dir.joinpath("model-output-invalid.json").write_text(
|
|
73
|
+
json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
74
|
+
encoding="utf-8",
|
|
75
|
+
)
|
|
76
|
+
usage = {
|
|
77
|
+
**measured_usage,
|
|
78
|
+
"cost_usd": 0.0,
|
|
79
|
+
"unpriced_model_calls": int(measured_usage.get("model_calls", 0)),
|
|
80
|
+
}
|
|
81
|
+
return ProviderArmResult(
|
|
82
|
+
metrics={
|
|
83
|
+
"task_completion_pass": 0.0,
|
|
84
|
+
"state_requirements_met": 0.0,
|
|
85
|
+
"task_requirements_met": 0.0,
|
|
86
|
+
"adapter_hard_stops": 1.0,
|
|
87
|
+
},
|
|
88
|
+
trace=trace,
|
|
89
|
+
usage=usage,
|
|
90
|
+
runtime_instance_id=content_hash(
|
|
91
|
+
{
|
|
92
|
+
"provider": "state_bench",
|
|
93
|
+
"checkpoint_id": replay_record["checkpoint_id"],
|
|
94
|
+
"role": role,
|
|
95
|
+
"seed": seed,
|
|
96
|
+
"termination_reason": trace["termination_reason"],
|
|
97
|
+
"work_dir": str(work_dir),
|
|
98
|
+
}
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class StateBenchProvider:
|
|
104
|
+
benchmark_id = "state_bench.common_support"
|
|
105
|
+
source_revision = "4efcbf2d4fe60df04878859b692d9391f3d5b33a"
|
|
106
|
+
scorer_id = "state_bench.official_v0.8.1"
|
|
107
|
+
scorer_revision = "00f5c68a83e55535797435f1fed59e9d334878ef3431f9906d703778dbcf09e8"
|
|
108
|
+
|
|
109
|
+
_CONFIG_FIELDS = frozenset(
|
|
110
|
+
{
|
|
111
|
+
"domains",
|
|
112
|
+
"split",
|
|
113
|
+
"limit",
|
|
114
|
+
"agent_provider",
|
|
115
|
+
"agent_api_key_var",
|
|
116
|
+
"score_reasoning_effort",
|
|
117
|
+
"locked_dependency_hashes",
|
|
118
|
+
}
|
|
119
|
+
)
|
|
120
|
+
_OPENAI_COMPATIBLE_FIELDS = frozenset(
|
|
121
|
+
{
|
|
122
|
+
"agent_model",
|
|
123
|
+
"agent_base_url",
|
|
124
|
+
"evaluator_model",
|
|
125
|
+
"evaluator_base_url",
|
|
126
|
+
"evaluator_api_key_var",
|
|
127
|
+
}
|
|
128
|
+
)
|
|
129
|
+
_OPTIONAL_CONFIG_FIELDS = frozenset({"budget_override"})
|
|
130
|
+
|
|
131
|
+
def __init__(self, *, history_transform=None):
|
|
132
|
+
validate_history_transform(history_transform)
|
|
133
|
+
self.history_transform = history_transform
|
|
134
|
+
|
|
135
|
+
def doctor(self, config: Mapping[str, Any]) -> tuple[str, ...]:
|
|
136
|
+
try:
|
|
137
|
+
self._validate_config(config)
|
|
138
|
+
head = subprocess.check_output(
|
|
139
|
+
["git", "rev-parse", "HEAD"],
|
|
140
|
+
text=True,
|
|
141
|
+
stderr=subprocess.DEVNULL,
|
|
142
|
+
).strip()
|
|
143
|
+
if head != self.source_revision:
|
|
144
|
+
raise OfficialArenaError("STATE-Bench checkout revision differs")
|
|
145
|
+
from state_bench.client import (
|
|
146
|
+
build_llm_client,
|
|
147
|
+
build_locked_judge_client,
|
|
148
|
+
build_user_sim_client,
|
|
149
|
+
)
|
|
150
|
+
from state_bench.domain import get_domain_config
|
|
151
|
+
from state_bench.orchestrator import run_task
|
|
152
|
+
from state_bench.scoring import combine_task_completion
|
|
153
|
+
|
|
154
|
+
del (
|
|
155
|
+
build_llm_client,
|
|
156
|
+
build_locked_judge_client,
|
|
157
|
+
build_user_sim_client,
|
|
158
|
+
get_domain_config,
|
|
159
|
+
run_task,
|
|
160
|
+
combine_task_completion,
|
|
161
|
+
)
|
|
162
|
+
except Exception as exc:
|
|
163
|
+
return (f"{type(exc).__name__}: {exc}",)
|
|
164
|
+
return ()
|
|
165
|
+
|
|
166
|
+
def capture(
|
|
167
|
+
self,
|
|
168
|
+
config: Mapping[str, Any],
|
|
169
|
+
*,
|
|
170
|
+
output_root: Path,
|
|
171
|
+
) -> tuple[Mapping[str, Any], ...]:
|
|
172
|
+
del output_root
|
|
173
|
+
parsed = self._validate_config(config)
|
|
174
|
+
from state_bench.domain import get_domain_config
|
|
175
|
+
from state_bench.paths import domain_tasks_dir
|
|
176
|
+
from state_bench.protocol import load_default_protocol, load_split_task_ids
|
|
177
|
+
from state_bench.schemas import TaskDefinition
|
|
178
|
+
|
|
179
|
+
protocol = load_default_protocol()
|
|
180
|
+
errors = protocol.validate_prompt_hashes()
|
|
181
|
+
if errors:
|
|
182
|
+
raise OfficialArenaError("STATE-Bench protocol prompt hashes differ")
|
|
183
|
+
|
|
184
|
+
rows: list[Mapping[str, Any]] = []
|
|
185
|
+
for domain_name in parsed["domains"]:
|
|
186
|
+
domain = get_domain_config(domain_name)
|
|
187
|
+
task_ids = load_split_task_ids(
|
|
188
|
+
domain_name,
|
|
189
|
+
parsed["split"],
|
|
190
|
+
protocol.split_version,
|
|
191
|
+
)
|
|
192
|
+
for task_id in task_ids:
|
|
193
|
+
task = TaskDefinition.load(domain_tasks_dir(domain_name) / f"{task_id}.json")
|
|
194
|
+
user_id = task.user_id
|
|
195
|
+
if not user_id:
|
|
196
|
+
continue
|
|
197
|
+
instruction = domain.agent_system_prompt.format(
|
|
198
|
+
now=task.now,
|
|
199
|
+
user_id=user_id,
|
|
200
|
+
)
|
|
201
|
+
tool_names = tuple(
|
|
202
|
+
str(tool.get("name"))
|
|
203
|
+
for tool in domain.tool_schemas
|
|
204
|
+
if isinstance(tool, Mapping) and isinstance(tool.get("name"), str)
|
|
205
|
+
)
|
|
206
|
+
if not tool_names:
|
|
207
|
+
raise OfficialArenaError("STATE-Bench domain exposes no tool schemas")
|
|
208
|
+
visible_state = {
|
|
209
|
+
"query": task.opening_message,
|
|
210
|
+
"task_id": task.task_id,
|
|
211
|
+
"instruction": instruction,
|
|
212
|
+
"context": [
|
|
213
|
+
{
|
|
214
|
+
"text": instruction,
|
|
215
|
+
"block_id": "official_policy",
|
|
216
|
+
"source": "state_bench.domain.agent_system_prompt",
|
|
217
|
+
"score": None,
|
|
218
|
+
"metadata": {"domain": domain_name},
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
"text": json.dumps(
|
|
222
|
+
domain.tool_schemas,
|
|
223
|
+
ensure_ascii=False,
|
|
224
|
+
sort_keys=True,
|
|
225
|
+
),
|
|
226
|
+
"block_id": "official_tool_catalog",
|
|
227
|
+
"source": "state_bench.domain.tool_schemas",
|
|
228
|
+
"score": None,
|
|
229
|
+
"metadata": {"domain": domain_name},
|
|
230
|
+
},
|
|
231
|
+
],
|
|
232
|
+
"messages": [
|
|
233
|
+
{
|
|
234
|
+
"role": "user",
|
|
235
|
+
"content": task.opening_message,
|
|
236
|
+
"name": None,
|
|
237
|
+
}
|
|
238
|
+
],
|
|
239
|
+
"tool_names": list(tool_names),
|
|
240
|
+
"metadata": {
|
|
241
|
+
"benchmark_id": self.benchmark_id,
|
|
242
|
+
"domain": domain_name,
|
|
243
|
+
"checkpoint_phase": "turn_zero",
|
|
244
|
+
"clarification_roundtrip_available": True,
|
|
245
|
+
"state_restore_available": True,
|
|
246
|
+
},
|
|
247
|
+
"budget": visible_shared_budget(
|
|
248
|
+
parsed,
|
|
249
|
+
{
|
|
250
|
+
"max_model_calls": domain.max_agent_turns * 9,
|
|
251
|
+
"max_retrieval_calls": 32,
|
|
252
|
+
"max_tool_calls": domain.max_agent_turns * 8,
|
|
253
|
+
"max_output_tokens": 65536,
|
|
254
|
+
"max_context_tokens": None,
|
|
255
|
+
"max_cost_usd": None,
|
|
256
|
+
},
|
|
257
|
+
),
|
|
258
|
+
}
|
|
259
|
+
rows.append(
|
|
260
|
+
{
|
|
261
|
+
"checkpoint_id": f"{domain_name}:{task.task_id}",
|
|
262
|
+
"group_key": f"{domain_name}:{task.task_id}",
|
|
263
|
+
"visible_state": visible_state,
|
|
264
|
+
"eligible_families": [
|
|
265
|
+
"reformulate",
|
|
266
|
+
"decompose",
|
|
267
|
+
"acquire",
|
|
268
|
+
"clarify",
|
|
269
|
+
"curate",
|
|
270
|
+
],
|
|
271
|
+
"protected": {
|
|
272
|
+
"domain": domain_name,
|
|
273
|
+
"task_id": task.task_id,
|
|
274
|
+
"user_id": user_id,
|
|
275
|
+
"protocol_id": protocol.protocol_id,
|
|
276
|
+
"locked_dependency_hashes": parsed["locked_dependency_hashes"],
|
|
277
|
+
},
|
|
278
|
+
}
|
|
279
|
+
)
|
|
280
|
+
if len(rows) >= parsed["limit"]:
|
|
281
|
+
return tuple(rows)
|
|
282
|
+
if not rows:
|
|
283
|
+
raise OfficialArenaError("STATE-Bench capture found no protocol tasks")
|
|
284
|
+
return tuple(rows)
|
|
285
|
+
|
|
286
|
+
def run_arm(
|
|
287
|
+
self, config: Mapping[str, Any], *, replay_record: Mapping[str, Any],
|
|
288
|
+
arm_payload: Mapping[str, Any], seed: int, role: str, work_dir: Path,
|
|
289
|
+
) -> ProviderArmResult:
|
|
290
|
+
context = {}
|
|
291
|
+
try:
|
|
292
|
+
result = self._run_arm(config, replay_record=replay_record, arm_payload=arm_payload,
|
|
293
|
+
seed=seed, role=role, work_dir=work_dir, _history_context=context)
|
|
294
|
+
finally:
|
|
295
|
+
agent = context.get("agent")
|
|
296
|
+
session = getattr(agent, "_history_session", None)
|
|
297
|
+
if session is not None:
|
|
298
|
+
session.write_receipt(work_dir / "live-history.json",
|
|
299
|
+
target_usage=agent.agentinputlab_usage())
|
|
300
|
+
if session is None:
|
|
301
|
+
return result
|
|
302
|
+
receipt_hash = hashlib.sha256((work_dir / "live-history.json").read_bytes()).hexdigest()
|
|
303
|
+
return ProviderArmResult(metrics=result.metrics,
|
|
304
|
+
trace={**result.trace, "live_history_transform": session.hook.fingerprint(),
|
|
305
|
+
"live_history_receipt": {"path": "live-history.json", "sha256": receipt_hash},
|
|
306
|
+
"history_usage": dict(session.usage)},
|
|
307
|
+
usage=session.combined_usage(result.usage),
|
|
308
|
+
runtime_instance_id=content_hash({"native_runtime": result.runtime_instance_id,
|
|
309
|
+
"live_history_receipt_sha256": receipt_hash}))
|
|
310
|
+
|
|
311
|
+
def _run_arm(
|
|
312
|
+
self,
|
|
313
|
+
config: Mapping[str, Any],
|
|
314
|
+
*,
|
|
315
|
+
replay_record: Mapping[str, Any],
|
|
316
|
+
arm_payload: Mapping[str, Any],
|
|
317
|
+
seed: int,
|
|
318
|
+
role: str,
|
|
319
|
+
work_dir: Path,
|
|
320
|
+
_history_context: dict,
|
|
321
|
+
) -> ProviderArmResult:
|
|
322
|
+
parsed = self._validate_config(config)
|
|
323
|
+
state = materialized_state(arm_payload)
|
|
324
|
+
budget = require_materialized_budget(state, parsed)
|
|
325
|
+
optimizer_usage = intervention_usage(arm_payload)
|
|
326
|
+
remaining_model_calls = max(
|
|
327
|
+
0,
|
|
328
|
+
int(budget["max_model_calls"]) - int(optimizer_usage.get("model_calls", 0)),
|
|
329
|
+
)
|
|
330
|
+
remaining_output_tokens = max(
|
|
331
|
+
0,
|
|
332
|
+
int(budget["max_output_tokens"]) - int(optimizer_usage.get("output_tokens", 0)),
|
|
333
|
+
)
|
|
334
|
+
context_cap = budget.get("max_context_tokens")
|
|
335
|
+
remaining_input_tokens = (
|
|
336
|
+
2**63 - 1
|
|
337
|
+
if context_cap is None
|
|
338
|
+
else max(
|
|
339
|
+
0,
|
|
340
|
+
int(context_cap) - int(optimizer_usage.get("input_tokens", 0)),
|
|
341
|
+
)
|
|
342
|
+
)
|
|
343
|
+
remaining_tool_calls = max(
|
|
344
|
+
0,
|
|
345
|
+
int(budget["max_tool_calls"]) - int(optimizer_usage.get("tool_calls", 0)),
|
|
346
|
+
)
|
|
347
|
+
override = parsed.get("budget_override")
|
|
348
|
+
per_call_output_cap = (
|
|
349
|
+
int(override["max_output_tokens_per_call"])
|
|
350
|
+
if isinstance(override, Mapping)
|
|
351
|
+
else min(512, remaining_output_tokens)
|
|
352
|
+
)
|
|
353
|
+
protected = replay_record["protected"]
|
|
354
|
+
domain_name = protected["domain"]
|
|
355
|
+
task_id = protected["task_id"]
|
|
356
|
+
user_id = protected["user_id"]
|
|
357
|
+
|
|
358
|
+
from state_bench.agents.base import AgentRuntimeContext
|
|
359
|
+
from state_bench.client import (
|
|
360
|
+
build_llm_client,
|
|
361
|
+
build_locked_judge_client,
|
|
362
|
+
build_user_sim_client,
|
|
363
|
+
)
|
|
364
|
+
from state_bench.domain import get_domain_config
|
|
365
|
+
from state_bench.env_loader import load_task_environment
|
|
366
|
+
from state_bench.orchestrator import run_task
|
|
367
|
+
from state_bench.paths import domain_tasks_dir
|
|
368
|
+
from state_bench.protocol import load_default_protocol
|
|
369
|
+
from state_bench.schemas import TaskDefinition
|
|
370
|
+
from state_bench.scoring import (
|
|
371
|
+
TaskRequirementsJudge,
|
|
372
|
+
combine_task_completion,
|
|
373
|
+
evaluate_state_requirements,
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
domain = get_domain_config(domain_name)
|
|
377
|
+
task = TaskDefinition.load(domain_tasks_dir(domain_name) / f"{task_id}.json")
|
|
378
|
+
env_data, _ = load_task_environment(domain, task)
|
|
379
|
+
env_copy = env_data.deep_copy()
|
|
380
|
+
env = domain.environment_class(env_copy, now=task.now)
|
|
381
|
+
agent_system_prompt = domain.agent_system_prompt.format(
|
|
382
|
+
now=task.now,
|
|
383
|
+
user_id=user_id,
|
|
384
|
+
)
|
|
385
|
+
if self._OPENAI_COMPATIBLE_FIELDS.issubset(parsed):
|
|
386
|
+
agent_client = _openai_compatible_client(
|
|
387
|
+
model=parsed["agent_model"],
|
|
388
|
+
base_url=parsed["agent_base_url"],
|
|
389
|
+
api_key_var=parsed["agent_api_key_var"],
|
|
390
|
+
)
|
|
391
|
+
simulator_client = _openai_compatible_client(
|
|
392
|
+
model=parsed["evaluator_model"],
|
|
393
|
+
base_url=parsed["evaluator_base_url"],
|
|
394
|
+
api_key_var=parsed["evaluator_api_key_var"],
|
|
395
|
+
)
|
|
396
|
+
judge_client = _openai_compatible_client(
|
|
397
|
+
model=parsed["evaluator_model"],
|
|
398
|
+
base_url=parsed["evaluator_base_url"],
|
|
399
|
+
api_key_var=parsed["evaluator_api_key_var"],
|
|
400
|
+
)
|
|
401
|
+
_install_statebench_judge_json_compat(judge_client)
|
|
402
|
+
else:
|
|
403
|
+
agent_client = build_llm_client(
|
|
404
|
+
provider=parsed["agent_provider"],
|
|
405
|
+
api_key_var=parsed["agent_api_key_var"],
|
|
406
|
+
)
|
|
407
|
+
simulator_client = build_user_sim_client()
|
|
408
|
+
judge_client = build_locked_judge_client()
|
|
409
|
+
runtime_context = AgentRuntimeContext(
|
|
410
|
+
task_id=task.task_id,
|
|
411
|
+
user_id=user_id,
|
|
412
|
+
domain=domain.name,
|
|
413
|
+
now=task.now,
|
|
414
|
+
task_summary=task.task_summary,
|
|
415
|
+
state_requirements=task.state_requirements,
|
|
416
|
+
task_requirements=task.task_requirements,
|
|
417
|
+
)
|
|
418
|
+
agent = _intervened_agent_class()(
|
|
419
|
+
agent_client,
|
|
420
|
+
agent_system_prompt,
|
|
421
|
+
domain.tool_schemas,
|
|
422
|
+
env.tool_handlers,
|
|
423
|
+
runtime_context=runtime_context,
|
|
424
|
+
materialized=state,
|
|
425
|
+
max_api_calls=remaining_model_calls,
|
|
426
|
+
max_output_tokens=remaining_output_tokens,
|
|
427
|
+
max_input_tokens=remaining_input_tokens,
|
|
428
|
+
max_output_tokens_per_call=per_call_output_cap,
|
|
429
|
+
max_tool_calls=remaining_tool_calls,
|
|
430
|
+
history_transform=self.history_transform,
|
|
431
|
+
)
|
|
432
|
+
_history_context["agent"] = agent
|
|
433
|
+
try:
|
|
434
|
+
trajectory = run_task(
|
|
435
|
+
task,
|
|
436
|
+
env_data,
|
|
437
|
+
user_id,
|
|
438
|
+
client=None,
|
|
439
|
+
domain=domain,
|
|
440
|
+
agent=agent,
|
|
441
|
+
env=env,
|
|
442
|
+
trajectory_metadata={
|
|
443
|
+
"agentinputlab": {
|
|
444
|
+
"release_id": arm_payload["release_id"],
|
|
445
|
+
"role": role,
|
|
446
|
+
"seed": seed,
|
|
447
|
+
"output_state_hash": arm_payload["intervention_execution"][
|
|
448
|
+
"output_state_hash"
|
|
449
|
+
],
|
|
450
|
+
}
|
|
451
|
+
},
|
|
452
|
+
simulator_client=simulator_client,
|
|
453
|
+
)
|
|
454
|
+
except _EvaluatedAgentOutputInvalid as failure:
|
|
455
|
+
return _model_output_invalid_result(
|
|
456
|
+
replay_record=replay_record,
|
|
457
|
+
role=role,
|
|
458
|
+
seed=seed,
|
|
459
|
+
work_dir=work_dir,
|
|
460
|
+
error=failure.error,
|
|
461
|
+
measured_usage=agent.agentinputlab_usage(),
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
tool_calls = [
|
|
465
|
+
call
|
|
466
|
+
for message in trajectory.conversation
|
|
467
|
+
for call in (message.get("tool_calls") or [])
|
|
468
|
+
]
|
|
469
|
+
state_result = evaluate_state_requirements(task, trajectory.state_diff)
|
|
470
|
+
protocol = load_default_protocol()
|
|
471
|
+
judge_reasoning_effort = normalize_qwen38_reasoning_effort(
|
|
472
|
+
parsed.get("evaluator_model"),
|
|
473
|
+
parsed["score_reasoning_effort"] or protocol.judge_reasoning_effort,
|
|
474
|
+
)
|
|
475
|
+
judge = TaskRequirementsJudge(
|
|
476
|
+
client=judge_client,
|
|
477
|
+
prompts_dir=domain.prompts_dir,
|
|
478
|
+
system_prompt=domain.judge_system_prompt,
|
|
479
|
+
reasoning_effort=judge_reasoning_effort,
|
|
480
|
+
)
|
|
481
|
+
task_result = judge.evaluate(
|
|
482
|
+
task,
|
|
483
|
+
trajectory.conversation,
|
|
484
|
+
tool_calls,
|
|
485
|
+
trajectory.state_diff,
|
|
486
|
+
)
|
|
487
|
+
completion = combine_task_completion(state_result, task_result)
|
|
488
|
+
if completion is None:
|
|
489
|
+
raise OfficialArenaError("STATE-Bench official task judge did not complete")
|
|
490
|
+
|
|
491
|
+
trace = {
|
|
492
|
+
"provider": "state_bench",
|
|
493
|
+
"source_revision": self.source_revision,
|
|
494
|
+
"protocol_id": protected["protocol_id"],
|
|
495
|
+
"task_id": task_id,
|
|
496
|
+
"domain": domain_name,
|
|
497
|
+
"role": role,
|
|
498
|
+
"seed": seed,
|
|
499
|
+
"conversation": trajectory.conversation,
|
|
500
|
+
"state_diff": trajectory.state_diff.to_dict(),
|
|
501
|
+
"efficiency": (trajectory.efficiency.to_dict() if trajectory.efficiency else None),
|
|
502
|
+
"task_completion_pass": completion,
|
|
503
|
+
"adapter_compatibility_repairs": {
|
|
504
|
+
"qwen3_4b_judge_reasoning_json": int(
|
|
505
|
+
getattr(
|
|
506
|
+
judge_client,
|
|
507
|
+
"_agentinputlab_qwen3_4b_judge_json_repairs",
|
|
508
|
+
0,
|
|
509
|
+
)
|
|
510
|
+
)
|
|
511
|
+
},
|
|
512
|
+
}
|
|
513
|
+
token_usage = trajectory.token_usage.to_dict()
|
|
514
|
+
audited_agent_usage = agent.agentinputlab_usage()
|
|
515
|
+
usage = {
|
|
516
|
+
# Conversation rows are not API-call receipts: a clarification or
|
|
517
|
+
# budget-stop message can be emitted without calling the model.
|
|
518
|
+
# The wrapper counters below are incremented exactly at the model
|
|
519
|
+
# and tool boundaries and are the only counters compatible with
|
|
520
|
+
# the shared remaining-budget calculation above.
|
|
521
|
+
"model_calls": audited_agent_usage["model_calls"],
|
|
522
|
+
"tool_calls": audited_agent_usage["tool_calls"],
|
|
523
|
+
"input_tokens": audited_agent_usage["input_tokens"],
|
|
524
|
+
"output_tokens": audited_agent_usage["output_tokens"],
|
|
525
|
+
"cost_usd": float(token_usage.get("total_cost_usd", 0.0) or 0.0),
|
|
526
|
+
}
|
|
527
|
+
work_dir.joinpath("trajectory.json").write_text(
|
|
528
|
+
json.dumps(trace, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
529
|
+
encoding="utf-8",
|
|
530
|
+
)
|
|
531
|
+
return ProviderArmResult(
|
|
532
|
+
metrics={
|
|
533
|
+
"task_completion_pass": float(completion),
|
|
534
|
+
"state_requirements_met": float(state_result.score),
|
|
535
|
+
"task_requirements_met": float(task_result.score),
|
|
536
|
+
},
|
|
537
|
+
trace=trace,
|
|
538
|
+
usage=usage,
|
|
539
|
+
runtime_instance_id=content_hash(
|
|
540
|
+
{
|
|
541
|
+
"provider": "state_bench",
|
|
542
|
+
"checkpoint_id": replay_record["checkpoint_id"],
|
|
543
|
+
"role": role,
|
|
544
|
+
"seed": seed,
|
|
545
|
+
"work_dir": str(work_dir),
|
|
546
|
+
}
|
|
547
|
+
),
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
def _validate_config(self, config: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
551
|
+
config = resolve_shared_budget_config(config)
|
|
552
|
+
fields = set(config) if isinstance(config, Mapping) else set()
|
|
553
|
+
valid_fields = {
|
|
554
|
+
self._CONFIG_FIELDS,
|
|
555
|
+
self._CONFIG_FIELDS | self._OPENAI_COMPATIBLE_FIELDS,
|
|
556
|
+
self._CONFIG_FIELDS | self._OPTIONAL_CONFIG_FIELDS,
|
|
557
|
+
self._CONFIG_FIELDS | self._OPENAI_COMPATIBLE_FIELDS | self._OPTIONAL_CONFIG_FIELDS,
|
|
558
|
+
}
|
|
559
|
+
if not isinstance(config, Mapping) or fields not in valid_fields:
|
|
560
|
+
raise OfficialArenaError("STATE-Bench provider config fields are invalid")
|
|
561
|
+
domains = config["domains"]
|
|
562
|
+
if (
|
|
563
|
+
isinstance(domains, (str, bytes))
|
|
564
|
+
or not isinstance(domains, Sequence)
|
|
565
|
+
or not domains
|
|
566
|
+
or len(domains) != len(set(domains))
|
|
567
|
+
or any(not isinstance(item, str) or not item.strip() for item in domains)
|
|
568
|
+
):
|
|
569
|
+
raise OfficialArenaError("STATE-Bench domains must be unique strings")
|
|
570
|
+
split = config["split"]
|
|
571
|
+
if split not in {"train", "test"}:
|
|
572
|
+
raise OfficialArenaError("STATE-Bench split must be train or test")
|
|
573
|
+
limit = config["limit"]
|
|
574
|
+
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
|
|
575
|
+
raise OfficialArenaError("STATE-Bench limit must be positive")
|
|
576
|
+
if config["agent_provider"] not in {"openai", "azure_openai"}:
|
|
577
|
+
raise OfficialArenaError("STATE-Bench agent_provider is unsupported")
|
|
578
|
+
key_fields = ["agent_api_key_var"]
|
|
579
|
+
if self._OPENAI_COMPATIBLE_FIELDS.issubset(config):
|
|
580
|
+
if config["agent_provider"] != "openai":
|
|
581
|
+
raise OfficialArenaError(
|
|
582
|
+
"STATE-Bench OpenAI-compatible runtime requires agent_provider=openai"
|
|
583
|
+
)
|
|
584
|
+
for name in (
|
|
585
|
+
"agent_model",
|
|
586
|
+
"agent_base_url",
|
|
587
|
+
"evaluator_model",
|
|
588
|
+
"evaluator_base_url",
|
|
589
|
+
):
|
|
590
|
+
if not isinstance(config[name], str) or not config[name].strip():
|
|
591
|
+
raise OfficialArenaError(f"STATE-Bench {name} is invalid")
|
|
592
|
+
key_fields.append("evaluator_api_key_var")
|
|
593
|
+
for name in key_fields:
|
|
594
|
+
value = config[name]
|
|
595
|
+
if (
|
|
596
|
+
not isinstance(value, str)
|
|
597
|
+
or not value.strip()
|
|
598
|
+
or not value.replace("_", "").isalnum()
|
|
599
|
+
):
|
|
600
|
+
raise OfficialArenaError(f"STATE-Bench {name} is invalid")
|
|
601
|
+
effort = config["score_reasoning_effort"]
|
|
602
|
+
if effort is not None and effort not in {"low", "medium", "high"}:
|
|
603
|
+
raise OfficialArenaError("STATE-Bench score_reasoning_effort is invalid")
|
|
604
|
+
validate_dependency_hashes(
|
|
605
|
+
config["locked_dependency_hashes"],
|
|
606
|
+
label="STATE-Bench",
|
|
607
|
+
)
|
|
608
|
+
return config
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _openai_compatible_client(
|
|
612
|
+
*,
|
|
613
|
+
model: str,
|
|
614
|
+
base_url: str,
|
|
615
|
+
api_key_var: str,
|
|
616
|
+
) -> Any:
|
|
617
|
+
"""Inject only the transport while retaining pinned STATE-Bench client logic."""
|
|
618
|
+
from openai import OpenAI
|
|
619
|
+
from state_bench.client import LLMClient
|
|
620
|
+
|
|
621
|
+
api_key = os.environ.get(api_key_var)
|
|
622
|
+
if not api_key:
|
|
623
|
+
raise OfficialArenaError(
|
|
624
|
+
f"STATE-Bench OpenAI-compatible environment {api_key_var} is missing"
|
|
625
|
+
)
|
|
626
|
+
client = LLMClient.__new__(LLMClient)
|
|
627
|
+
client.provider = "openai"
|
|
628
|
+
client.endpoint = base_url
|
|
629
|
+
client.deployment = model
|
|
630
|
+
client.deployments = [model]
|
|
631
|
+
client._chat_response_history = {}
|
|
632
|
+
client._client = OpenAI(api_key=api_key, base_url=base_url, max_retries=0)
|
|
633
|
+
_install_stateless_tool_history(client)
|
|
634
|
+
return client
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
class _ResponseTextOverride:
|
|
638
|
+
"""Delegate a Responses object while exposing one validated text repair."""
|
|
639
|
+
|
|
640
|
+
def __init__(self, response: Any, output_text: str) -> None:
|
|
641
|
+
self._response = response
|
|
642
|
+
self.output_text = output_text
|
|
643
|
+
|
|
644
|
+
def __getattr__(self, name: str) -> Any:
|
|
645
|
+
return getattr(self._response, name)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _response_field(value: Any, name: str) -> Any:
|
|
649
|
+
if isinstance(value, Mapping):
|
|
650
|
+
return value.get(name)
|
|
651
|
+
return getattr(value, name, None)
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _qwen3_4b_judge_reasoning_json(response: Any, *, model: str) -> str | None:
|
|
655
|
+
"""Recover only a completed Qwen3-4B judge JSON emitted as reasoning text.
|
|
656
|
+
|
|
657
|
+
Some Qwen3-4B vLLM Responses generations place the JSON-object answer in
|
|
658
|
+
a ``reasoning_text`` block while leaving the SDK's ``output_text`` empty.
|
|
659
|
+
STATE-Bench then retries and ultimately drops an otherwise valid official
|
|
660
|
+
judge result. Keep the repair deliberately narrower than generic response
|
|
661
|
+
promotion: it applies only to this served alias, completed responses, one
|
|
662
|
+
reasoning block, and the exact top-level shape consumed by
|
|
663
|
+
``TaskRequirementsJudge``.
|
|
664
|
+
"""
|
|
665
|
+
|
|
666
|
+
alias = model.strip().casefold().replace("_", "-")
|
|
667
|
+
if alias != "qwen3-4b":
|
|
668
|
+
return None
|
|
669
|
+
if _response_field(response, "status") != "completed":
|
|
670
|
+
return None
|
|
671
|
+
if _response_field(response, "output_text") not in {None, ""}:
|
|
672
|
+
return None
|
|
673
|
+
|
|
674
|
+
candidates: list[str] = []
|
|
675
|
+
output = _response_field(response, "output")
|
|
676
|
+
if not isinstance(output, Sequence) or isinstance(output, (str, bytes)):
|
|
677
|
+
return None
|
|
678
|
+
for item in output:
|
|
679
|
+
if _response_field(item, "type") != "reasoning":
|
|
680
|
+
continue
|
|
681
|
+
content = _response_field(item, "content")
|
|
682
|
+
if not isinstance(content, Sequence) or isinstance(content, (str, bytes)):
|
|
683
|
+
continue
|
|
684
|
+
for block in content:
|
|
685
|
+
if _response_field(block, "type") != "reasoning_text":
|
|
686
|
+
continue
|
|
687
|
+
text = _response_field(block, "text")
|
|
688
|
+
if isinstance(text, str) and text.strip():
|
|
689
|
+
candidates.append(text)
|
|
690
|
+
if len(candidates) != 1:
|
|
691
|
+
return None
|
|
692
|
+
try:
|
|
693
|
+
parsed = json.loads(candidates[0])
|
|
694
|
+
except (TypeError, ValueError):
|
|
695
|
+
return None
|
|
696
|
+
if not isinstance(parsed, Mapping) or not isinstance(parsed.get("details"), list):
|
|
697
|
+
return None
|
|
698
|
+
return candidates[0]
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _install_statebench_judge_json_compat(client: Any) -> None:
|
|
702
|
+
"""Install the narrow Qwen3-4B JSON repair on the judge client only."""
|
|
703
|
+
|
|
704
|
+
original = client.complete_json_response
|
|
705
|
+
client._agentinputlab_qwen3_4b_judge_json_repairs = 0
|
|
706
|
+
|
|
707
|
+
def complete_json_response(**call_kwargs: Any) -> Any:
|
|
708
|
+
response = original(**call_kwargs)
|
|
709
|
+
recovered = _qwen3_4b_judge_reasoning_json(
|
|
710
|
+
response,
|
|
711
|
+
model=str(getattr(client, "deployment", "")),
|
|
712
|
+
)
|
|
713
|
+
if recovered is None:
|
|
714
|
+
return response
|
|
715
|
+
client._agentinputlab_qwen3_4b_judge_json_repairs += 1
|
|
716
|
+
return _ResponseTextOverride(response, recovered)
|
|
717
|
+
|
|
718
|
+
client.complete_json_response = complete_json_response
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def _install_stateless_tool_history(client: Any) -> None:
|
|
722
|
+
"""Replay Responses tool history for stateless OpenAI-compatible servers.
|
|
723
|
+
|
|
724
|
+
STATE-Bench natively chains tool turns with ``previous_response_id``.
|
|
725
|
+
Self-hosted vLLM implements the Responses wire format but does not retain
|
|
726
|
+
provider-side response objects. Preserve the upstream turn loop while
|
|
727
|
+
replacing only that transport assumption with explicit input replay.
|
|
728
|
+
"""
|
|
729
|
+
|
|
730
|
+
original = client.complete_with_tools
|
|
731
|
+
histories: dict[str, list[Any]] = {}
|
|
732
|
+
|
|
733
|
+
def replayable(items: list[Any]) -> list[Any]:
|
|
734
|
+
if str(getattr(client, "deployment", "")).casefold() != "gpt-oss-20b":
|
|
735
|
+
return items
|
|
736
|
+
# vLLM's Responses endpoint does not accept its own occasional
|
|
737
|
+
# GPT-OSS ``mcp_call`` output as a subsequent input item. STATE-Bench
|
|
738
|
+
# ignores that item (only function_call is executable), so omit only
|
|
739
|
+
# this inert parser artifact from stateless replay.
|
|
740
|
+
return [
|
|
741
|
+
item
|
|
742
|
+
for item in items
|
|
743
|
+
if (
|
|
744
|
+
item.get("type") if isinstance(item, Mapping) else getattr(item, "type", None)
|
|
745
|
+
)
|
|
746
|
+
!= "mcp_call"
|
|
747
|
+
]
|
|
748
|
+
|
|
749
|
+
def stateless_complete_with_tools(**call_kwargs: Any) -> Any:
|
|
750
|
+
previous = call_kwargs.get("previous_response_id")
|
|
751
|
+
incoming = list(call_kwargs.get("input") or ())
|
|
752
|
+
if previous is not None:
|
|
753
|
+
history = histories.get(str(previous))
|
|
754
|
+
if history is None:
|
|
755
|
+
raise OfficialArenaError(
|
|
756
|
+
"STATE-Bench stateless tool continuation has no local response history"
|
|
757
|
+
)
|
|
758
|
+
incoming = [*replayable(history), *incoming]
|
|
759
|
+
call_kwargs["input"] = incoming
|
|
760
|
+
call_kwargs["previous_response_id"] = None
|
|
761
|
+
else:
|
|
762
|
+
history_view = getattr(client, "_agentinputlab_responses_history", None)
|
|
763
|
+
if history_view is not None:
|
|
764
|
+
call_kwargs = history_view(dict(call_kwargs, input=incoming))
|
|
765
|
+
incoming = list(call_kwargs["input"])
|
|
766
|
+
if requires_qwen38_initial_system_message(client.deployment):
|
|
767
|
+
normalized_input, normalized_instructions = (
|
|
768
|
+
fold_responses_system_messages_into_instructions(
|
|
769
|
+
incoming,
|
|
770
|
+
call_kwargs.get("instructions"),
|
|
771
|
+
)
|
|
772
|
+
)
|
|
773
|
+
call_kwargs["input"] = normalized_input
|
|
774
|
+
call_kwargs["instructions"] = normalized_instructions
|
|
775
|
+
response = original(**call_kwargs)
|
|
776
|
+
response_id = getattr(response, "id", None)
|
|
777
|
+
output = getattr(response, "output", None)
|
|
778
|
+
if not isinstance(response_id, str) or not response_id or not isinstance(output, list):
|
|
779
|
+
raise OfficialArenaError("STATE-Bench Responses result has no replayable output")
|
|
780
|
+
histories[response_id] = [*incoming, *output]
|
|
781
|
+
return response
|
|
782
|
+
|
|
783
|
+
client.complete_with_tools = stateless_complete_with_tools
|
|
784
|
+
client._agentinputlab_stateless_responses = True
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def _is_gpt_oss_harmony_parse_failure(error: Exception, *, model_name: str) -> bool:
|
|
788
|
+
"""Recognize only GPT-OSS response-parser HTTP-500 failures."""
|
|
789
|
+
|
|
790
|
+
if model_name.casefold() != "gpt-oss-20b" or getattr(error, "status_code", None) != 500:
|
|
791
|
+
return False
|
|
792
|
+
body = getattr(error, "body", None)
|
|
793
|
+
message: Any = None
|
|
794
|
+
if isinstance(body, Mapping):
|
|
795
|
+
message = body.get("message")
|
|
796
|
+
nested = body.get("error")
|
|
797
|
+
if message is None and isinstance(nested, Mapping):
|
|
798
|
+
message = nested.get("message")
|
|
799
|
+
if not isinstance(message, str):
|
|
800
|
+
return False
|
|
801
|
+
normalized = message.casefold()
|
|
802
|
+
return normalized.startswith("unexpected tokens remaining in message header") or normalized.startswith(
|
|
803
|
+
"unexpected token "
|
|
804
|
+
)
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def _intervened_agent_class() -> type[Any]:
|
|
808
|
+
from state_bench.agents.state_bench import StateBenchAgent
|
|
809
|
+
|
|
810
|
+
class IntervenedStateBenchAgent(StateBenchAgent):
|
|
811
|
+
def __init__(
|
|
812
|
+
self,
|
|
813
|
+
*args: Any,
|
|
814
|
+
materialized: Mapping[str, Any],
|
|
815
|
+
max_api_calls: int,
|
|
816
|
+
max_output_tokens: int,
|
|
817
|
+
max_input_tokens: int,
|
|
818
|
+
max_output_tokens_per_call: int,
|
|
819
|
+
max_tool_calls: int,
|
|
820
|
+
history_transform=None,
|
|
821
|
+
**kwargs: Any,
|
|
822
|
+
):
|
|
823
|
+
super().__init__(*args, **kwargs)
|
|
824
|
+
self._materialized = materialized
|
|
825
|
+
self._first_turn = True
|
|
826
|
+
self._budget_stopped = False
|
|
827
|
+
self._budget_api_calls = 0
|
|
828
|
+
self._budget_input_tokens = 0
|
|
829
|
+
self._budget_output_tokens = 0
|
|
830
|
+
self._budget_tool_calls = 0
|
|
831
|
+
self._max_api_calls = max_api_calls
|
|
832
|
+
self._max_input_tokens = max_input_tokens
|
|
833
|
+
self._max_output_tokens = max_output_tokens
|
|
834
|
+
self._max_output_tokens_per_call = max_output_tokens_per_call
|
|
835
|
+
self._max_tool_calls = max_tool_calls
|
|
836
|
+
self._history_session = (ResponsesHistorySession(history_transform, materialized,
|
|
837
|
+
dict(max_model_calls=max_api_calls, max_output_tokens=max_output_tokens))
|
|
838
|
+
if history_transform is not None else None)
|
|
839
|
+
if self._history_session is not None:
|
|
840
|
+
if not hasattr(self.client, "_agentinputlab_stateless_responses"):
|
|
841
|
+
raise OfficialArenaError("STATE history requires the native stateless Responses transport")
|
|
842
|
+
|
|
843
|
+
def history_view(call_kwargs):
|
|
844
|
+
call_kwargs["input"], call_kwargs["instructions"] = self._history_session.responses(
|
|
845
|
+
call_kwargs["input"], instructions=call_kwargs.get("instructions"),
|
|
846
|
+
agent_usage=self.agentinputlab_usage())
|
|
847
|
+
remaining = self._history_session.remaining(self.agentinputlab_usage())
|
|
848
|
+
if (remaining["max_model_calls"] < 1 or remaining["max_output_tokens"] < 1
|
|
849
|
+
or self._budget_input_tokens + self._history_session.usage.get("input_tokens", 0)
|
|
850
|
+
>= self._max_input_tokens):
|
|
851
|
+
raise _SharedBudgetStop
|
|
852
|
+
call_kwargs["max_tokens"] = min(int(call_kwargs["max_tokens"]),
|
|
853
|
+
remaining["max_output_tokens"])
|
|
854
|
+
return call_kwargs
|
|
855
|
+
|
|
856
|
+
self.client._agentinputlab_responses_history = history_view
|
|
857
|
+
original = self.client.complete_with_tools
|
|
858
|
+
|
|
859
|
+
def budgeted_complete_with_tools(**call_kwargs: Any) -> Any:
|
|
860
|
+
remaining_output = self._max_output_tokens - self._budget_output_tokens
|
|
861
|
+
calls = self._budget_api_calls
|
|
862
|
+
if self._history_session is not None:
|
|
863
|
+
remaining = self._history_session.remaining(self.agentinputlab_usage())
|
|
864
|
+
remaining_output = remaining["max_output_tokens"]
|
|
865
|
+
calls += int(self._history_session.usage.get("model_calls", 0))
|
|
866
|
+
if calls >= self._max_api_calls or remaining_output < 1:
|
|
867
|
+
raise _SharedBudgetStop
|
|
868
|
+
requested = int(call_kwargs.get("max_tokens", 4096))
|
|
869
|
+
call_kwargs["max_tokens"] = min(
|
|
870
|
+
requested,
|
|
871
|
+
self._max_output_tokens_per_call,
|
|
872
|
+
remaining_output,
|
|
873
|
+
)
|
|
874
|
+
response = original(**call_kwargs)
|
|
875
|
+
self._budget_api_calls += 1
|
|
876
|
+
usage = getattr(response, "usage", None)
|
|
877
|
+
self._budget_input_tokens += int(getattr(usage, "input_tokens", 0) or 0)
|
|
878
|
+
self._budget_output_tokens += int(getattr(usage, "output_tokens", 0) or 0)
|
|
879
|
+
if (
|
|
880
|
+
self._budget_input_tokens + (self._history_session.usage.get("input_tokens", 0)
|
|
881
|
+
if self._history_session is not None else 0) > self._max_input_tokens
|
|
882
|
+
or self._budget_output_tokens + (self._history_session.usage.get("output_tokens", 0)
|
|
883
|
+
if self._history_session is not None else 0) > self._max_output_tokens
|
|
884
|
+
):
|
|
885
|
+
raise _SharedBudgetStop
|
|
886
|
+
return response
|
|
887
|
+
|
|
888
|
+
self.client.complete_with_tools = budgeted_complete_with_tools
|
|
889
|
+
original_handlers = dict(self.tool_handlers)
|
|
890
|
+
for tool_name, handler in original_handlers.items():
|
|
891
|
+
|
|
892
|
+
def budgeted_handler(
|
|
893
|
+
arguments: Any,
|
|
894
|
+
*,
|
|
895
|
+
_handler: Any = handler,
|
|
896
|
+
) -> Any:
|
|
897
|
+
if self._budget_tool_calls >= self._max_tool_calls:
|
|
898
|
+
raise _SharedBudgetStop
|
|
899
|
+
self._budget_tool_calls += 1
|
|
900
|
+
return _handler(arguments)
|
|
901
|
+
|
|
902
|
+
self.tool_handlers[tool_name] = budgeted_handler
|
|
903
|
+
|
|
904
|
+
def prepare_conversation(self, conversation: list[Any]) -> list[Any]:
|
|
905
|
+
prepared = [dict(item) if isinstance(item, Mapping) else item for item in conversation]
|
|
906
|
+
if prepared and isinstance(prepared[0], Mapping):
|
|
907
|
+
prepared[0] = {**prepared[0], "content": self._materialized["query"]}
|
|
908
|
+
additions = [self._materialized.get("instruction", "")]
|
|
909
|
+
additions.extend(
|
|
910
|
+
item.get("text", "")
|
|
911
|
+
for item in self._materialized.get("context", [])
|
|
912
|
+
if isinstance(item, Mapping)
|
|
913
|
+
)
|
|
914
|
+
injected = "\n\n".join(item for item in additions if item)
|
|
915
|
+
return self.inject_system_message(prepared, injected)
|
|
916
|
+
|
|
917
|
+
def act(self, conversation: list[Any]) -> tuple[str, list[dict[str, Any]], list[Any]]:
|
|
918
|
+
metadata = self._materialized.get("metadata", {})
|
|
919
|
+
clarification = (
|
|
920
|
+
metadata.get("clarification_request") if isinstance(metadata, Mapping) else None
|
|
921
|
+
)
|
|
922
|
+
if self._first_turn and isinstance(clarification, str) and clarification.strip():
|
|
923
|
+
self._first_turn = False
|
|
924
|
+
return clarification, [], [{"role": "assistant", "content": clarification}]
|
|
925
|
+
self._first_turn = False
|
|
926
|
+
if self._budget_stopped:
|
|
927
|
+
message = "I could not complete the task within the shared run budget."
|
|
928
|
+
return message, [], [{"role": "assistant", "content": message}]
|
|
929
|
+
try:
|
|
930
|
+
return super().act(conversation)
|
|
931
|
+
except _SharedBudgetStop:
|
|
932
|
+
self._budget_stopped = True
|
|
933
|
+
message = "I could not complete the task within the shared run budget."
|
|
934
|
+
return message, [], [{"role": "assistant", "content": message}]
|
|
935
|
+
except json.JSONDecodeError as error:
|
|
936
|
+
raise _EvaluatedAgentOutputInvalid(error) from error
|
|
937
|
+
except Exception as error:
|
|
938
|
+
if not _is_gpt_oss_harmony_parse_failure(
|
|
939
|
+
error,
|
|
940
|
+
model_name=self.client.deployment,
|
|
941
|
+
):
|
|
942
|
+
raise
|
|
943
|
+
self._budget_api_calls += 1
|
|
944
|
+
raise _EvaluatedAgentOutputInvalid(error) from None
|
|
945
|
+
|
|
946
|
+
def agentinputlab_usage(self) -> Mapping[str, int]:
|
|
947
|
+
"""Return counters collected at the actual model/tool boundaries."""
|
|
948
|
+
|
|
949
|
+
return {
|
|
950
|
+
"model_calls": self._budget_api_calls,
|
|
951
|
+
"tool_calls": self._budget_tool_calls,
|
|
952
|
+
"input_tokens": self._budget_input_tokens,
|
|
953
|
+
"output_tokens": self._budget_output_tokens,
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
return IntervenedStateBenchAgent
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def factory() -> StateBenchProvider:
|
|
960
|
+
return StateBenchProvider()
|