agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
"""Shared, dependency-light contracts for official arena providers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Protocol, runtime_checkable
|
|
14
|
+
|
|
15
|
+
REPLAY_INVENTORY_PROTOCOL = "agentinputlab.official-arena-replay-inventory.v1"
|
|
16
|
+
EXECUTABLE_PAYLOAD_PROTOCOL = "agentinputlab.executable-release-payload.v1"
|
|
17
|
+
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
18
|
+
SHARED_BUDGET_PATH_ENV = "AILAB_SHARED_BUDGET_OVERRIDE_PATH"
|
|
19
|
+
SHARED_BUDGET_SHA256_ENV = "AILAB_SHARED_BUDGET_OVERRIDE_SHA256"
|
|
20
|
+
_BUDGET_FIELDS = frozenset(
|
|
21
|
+
{
|
|
22
|
+
"profile_id",
|
|
23
|
+
"max_model_calls",
|
|
24
|
+
"max_retrieval_calls",
|
|
25
|
+
"max_tool_calls",
|
|
26
|
+
"max_output_tokens",
|
|
27
|
+
"max_context_tokens",
|
|
28
|
+
"max_output_tokens_per_call",
|
|
29
|
+
}
|
|
30
|
+
)
|
|
31
|
+
_STANDARD_SHARED_BUDGET = {
|
|
32
|
+
"max_model_calls": 8,
|
|
33
|
+
"max_retrieval_calls": 8,
|
|
34
|
+
"max_tool_calls": 8,
|
|
35
|
+
"max_output_tokens": 4096,
|
|
36
|
+
# ExecutionBudget.max_context_tokens is a cumulative input-token budget,
|
|
37
|
+
# not the per-request model context window. Eight calls may each carry a
|
|
38
|
+
# growing interactive history, so the frozen run-level cap is eight times
|
|
39
|
+
# the 32,768-token per-call context control.
|
|
40
|
+
"max_context_tokens": 262144,
|
|
41
|
+
"max_output_tokens_per_call": 512,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class OfficialArenaError(RuntimeError):
|
|
46
|
+
"""Stable fail-closed error raised by an official runtime provider."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True, slots=True)
|
|
50
|
+
class ProviderArmResult:
|
|
51
|
+
"""One completed official episode plus scorer and usage evidence."""
|
|
52
|
+
|
|
53
|
+
metrics: Mapping[str, float]
|
|
54
|
+
trace: Mapping[str, Any]
|
|
55
|
+
usage: Mapping[str, int | float] = field(default_factory=dict)
|
|
56
|
+
runtime_instance_id: str = ""
|
|
57
|
+
|
|
58
|
+
def __post_init__(self) -> None:
|
|
59
|
+
if not isinstance(self.metrics, Mapping) or not self.metrics:
|
|
60
|
+
raise OfficialArenaError("official provider metrics must be non-empty")
|
|
61
|
+
for name, value in self.metrics.items():
|
|
62
|
+
if (
|
|
63
|
+
not isinstance(name, str)
|
|
64
|
+
or not name.strip()
|
|
65
|
+
or isinstance(value, bool)
|
|
66
|
+
or not isinstance(value, (int, float))
|
|
67
|
+
or not math.isfinite(float(value))
|
|
68
|
+
):
|
|
69
|
+
raise OfficialArenaError("official provider metrics are invalid")
|
|
70
|
+
if not isinstance(self.trace, Mapping) or not self.trace:
|
|
71
|
+
raise OfficialArenaError("official provider trace must be non-empty")
|
|
72
|
+
if not isinstance(self.usage, Mapping):
|
|
73
|
+
raise OfficialArenaError("official provider usage must be an object")
|
|
74
|
+
if not isinstance(self.runtime_instance_id, str) or len(self.runtime_instance_id) != 64:
|
|
75
|
+
raise OfficialArenaError(
|
|
76
|
+
"official provider runtime_instance_id must be a SHA-256 digest"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@runtime_checkable
|
|
81
|
+
class OfficialArenaProvider(Protocol):
|
|
82
|
+
"""Runtime-specific capture and continuation boundary."""
|
|
83
|
+
|
|
84
|
+
benchmark_id: str
|
|
85
|
+
source_revision: str
|
|
86
|
+
scorer_id: str
|
|
87
|
+
scorer_revision: str
|
|
88
|
+
|
|
89
|
+
def doctor(self, config: Mapping[str, Any]) -> tuple[str, ...]: ...
|
|
90
|
+
|
|
91
|
+
def capture(
|
|
92
|
+
self,
|
|
93
|
+
config: Mapping[str, Any],
|
|
94
|
+
*,
|
|
95
|
+
output_root: Path,
|
|
96
|
+
) -> tuple[Mapping[str, Any], ...]: ...
|
|
97
|
+
|
|
98
|
+
def run_arm(
|
|
99
|
+
self,
|
|
100
|
+
config: Mapping[str, Any],
|
|
101
|
+
*,
|
|
102
|
+
replay_record: Mapping[str, Any],
|
|
103
|
+
arm_payload: Mapping[str, Any],
|
|
104
|
+
seed: int,
|
|
105
|
+
role: str,
|
|
106
|
+
work_dir: Path,
|
|
107
|
+
) -> ProviderArmResult: ...
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def canonical_json(value: Any) -> str:
|
|
111
|
+
return json.dumps(
|
|
112
|
+
value,
|
|
113
|
+
ensure_ascii=False,
|
|
114
|
+
sort_keys=True,
|
|
115
|
+
separators=(",", ":"),
|
|
116
|
+
allow_nan=False,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def content_hash(value: Any) -> str:
|
|
121
|
+
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def sha256_file(path: Path) -> str:
|
|
125
|
+
digest = hashlib.sha256()
|
|
126
|
+
with path.open("rb") as handle:
|
|
127
|
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
128
|
+
digest.update(block)
|
|
129
|
+
return digest.hexdigest()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def validate_shared_budget(value: Any) -> Mapping[str, int | str]:
|
|
133
|
+
"""Validate one paper-wide resource budget shared by every arena arm."""
|
|
134
|
+
|
|
135
|
+
if not isinstance(value, Mapping) or set(value) != _BUDGET_FIELDS:
|
|
136
|
+
raise OfficialArenaError("shared budget override fields are invalid")
|
|
137
|
+
if not isinstance(value["profile_id"], str) or not value["profile_id"].strip():
|
|
138
|
+
raise OfficialArenaError("shared budget profile_id must be non-empty")
|
|
139
|
+
for name in _BUDGET_FIELDS - {"profile_id"}:
|
|
140
|
+
raw = value[name]
|
|
141
|
+
if isinstance(raw, bool) or not isinstance(raw, int) or raw < 1:
|
|
142
|
+
raise OfficialArenaError(f"shared budget {name} must be a positive integer")
|
|
143
|
+
if value["max_output_tokens_per_call"] > value["max_output_tokens"]:
|
|
144
|
+
raise OfficialArenaError(
|
|
145
|
+
"shared budget per-call output cap exceeds the cumulative output cap"
|
|
146
|
+
)
|
|
147
|
+
return value
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def standard_shared_budget(multiplier: float = 1.0) -> dict[str, int | str]:
|
|
151
|
+
"""Materialize the frozen paper control profile at one resource multiplier."""
|
|
152
|
+
|
|
153
|
+
if (
|
|
154
|
+
isinstance(multiplier, bool)
|
|
155
|
+
or not isinstance(multiplier, (int, float))
|
|
156
|
+
or not math.isfinite(float(multiplier))
|
|
157
|
+
or multiplier <= 0
|
|
158
|
+
):
|
|
159
|
+
raise OfficialArenaError("shared budget multiplier must be finite and positive")
|
|
160
|
+
return {
|
|
161
|
+
"profile_id": f"paper-shared-budget-x{multiplier:g}",
|
|
162
|
+
**{
|
|
163
|
+
name: max(1, round(value * multiplier))
|
|
164
|
+
for name, value in _STANDARD_SHARED_BUDGET.items()
|
|
165
|
+
},
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def resolve_shared_budget_config(config: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
170
|
+
"""Attach an authenticated process-level budget factor to a provider config.
|
|
171
|
+
|
|
172
|
+
The factor is read by every official provider before capture. Consequently
|
|
173
|
+
the exact same budget becomes part of the captured visible state, the
|
|
174
|
+
intervention input hash, and the fair-comparison dependency locks.
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
raw_path = os.environ.get(SHARED_BUDGET_PATH_ENV)
|
|
178
|
+
raw_sha256 = os.environ.get(SHARED_BUDGET_SHA256_ENV)
|
|
179
|
+
if not raw_path and not raw_sha256:
|
|
180
|
+
return config
|
|
181
|
+
if not raw_path or not raw_sha256 or _SHA256.fullmatch(raw_sha256) is None:
|
|
182
|
+
raise OfficialArenaError("shared budget environment binding is incomplete")
|
|
183
|
+
unresolved = Path(raw_path).expanduser()
|
|
184
|
+
path = unresolved.resolve()
|
|
185
|
+
if unresolved.is_symlink() or not path.is_file() or sha256_file(path) != raw_sha256:
|
|
186
|
+
raise OfficialArenaError("shared budget environment artifact differs")
|
|
187
|
+
try:
|
|
188
|
+
loaded = json.loads(path.read_text(encoding="utf-8"))
|
|
189
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
190
|
+
raise OfficialArenaError("shared budget artifact is invalid JSON") from exc
|
|
191
|
+
override = validate_shared_budget(loaded)
|
|
192
|
+
existing = config.get("budget_override")
|
|
193
|
+
if existing is not None and existing != override:
|
|
194
|
+
raise OfficialArenaError("explicit and environment budget overrides differ")
|
|
195
|
+
return {**config, "budget_override": dict(override)}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def visible_shared_budget(config: Mapping[str, Any], fallback: Mapping[str, Any]) -> dict[str, Any]:
|
|
199
|
+
"""Return the exact budget exposed to intervention code at capture time."""
|
|
200
|
+
|
|
201
|
+
raw = config.get("budget_override")
|
|
202
|
+
if raw is None:
|
|
203
|
+
return dict(fallback)
|
|
204
|
+
override = validate_shared_budget(raw)
|
|
205
|
+
return {
|
|
206
|
+
"max_model_calls": override["max_model_calls"],
|
|
207
|
+
"max_retrieval_calls": override["max_retrieval_calls"],
|
|
208
|
+
"max_tool_calls": override["max_tool_calls"],
|
|
209
|
+
"max_output_tokens": override["max_output_tokens"],
|
|
210
|
+
"max_context_tokens": override["max_context_tokens"],
|
|
211
|
+
"max_cost_usd": None,
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def require_materialized_budget(
|
|
216
|
+
state: Mapping[str, Any],
|
|
217
|
+
config: Mapping[str, Any],
|
|
218
|
+
) -> Mapping[str, Any]:
|
|
219
|
+
"""Fail closed if a materialized arm changed the shared resource budget."""
|
|
220
|
+
|
|
221
|
+
raw = config.get("budget_override")
|
|
222
|
+
budget = state.get("budget")
|
|
223
|
+
if not isinstance(budget, Mapping):
|
|
224
|
+
raise OfficialArenaError("materialized state has no execution budget")
|
|
225
|
+
if raw is None:
|
|
226
|
+
return budget
|
|
227
|
+
expected = visible_shared_budget(config, {})
|
|
228
|
+
if dict(budget) != expected:
|
|
229
|
+
raise OfficialArenaError("materialized arm changed the shared budget factor")
|
|
230
|
+
return budget
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def validate_dependency_hashes(
|
|
234
|
+
value: Any,
|
|
235
|
+
*,
|
|
236
|
+
label: str,
|
|
237
|
+
) -> Mapping[str, str]:
|
|
238
|
+
if (
|
|
239
|
+
not isinstance(value, Mapping)
|
|
240
|
+
or not value
|
|
241
|
+
or any(
|
|
242
|
+
not isinstance(name, str)
|
|
243
|
+
or not name
|
|
244
|
+
or not isinstance(digest, str)
|
|
245
|
+
or _SHA256.fullmatch(digest) is None
|
|
246
|
+
for name, digest in value.items()
|
|
247
|
+
)
|
|
248
|
+
):
|
|
249
|
+
raise OfficialArenaError(f"{label} dependency locks are invalid")
|
|
250
|
+
return value
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def materialized_state(arm_payload: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
254
|
+
if (
|
|
255
|
+
not isinstance(arm_payload, Mapping)
|
|
256
|
+
or arm_payload.get("schema_version") != 1
|
|
257
|
+
or arm_payload.get("protocol") != EXECUTABLE_PAYLOAD_PROTOCOL
|
|
258
|
+
):
|
|
259
|
+
raise OfficialArenaError(
|
|
260
|
+
"official arena requires an executable materialized release payload"
|
|
261
|
+
)
|
|
262
|
+
execution = arm_payload.get("intervention_execution")
|
|
263
|
+
if not isinstance(execution, Mapping):
|
|
264
|
+
raise OfficialArenaError("intervention_execution is missing")
|
|
265
|
+
state = execution.get("materialized_state")
|
|
266
|
+
if not isinstance(state, Mapping):
|
|
267
|
+
raise OfficialArenaError("materialized intervention state is missing")
|
|
268
|
+
if content_hash(state) != execution.get("output_state_hash"):
|
|
269
|
+
raise OfficialArenaError("materialized intervention state hash differs")
|
|
270
|
+
if execution.get("input_state_hash") != arm_payload.get("visible_state_hash"):
|
|
271
|
+
raise OfficialArenaError("intervention input state differs from the checkpoint")
|
|
272
|
+
return state
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def intervention_usage(arm_payload: Mapping[str, Any]) -> Mapping[str, int | float]:
|
|
276
|
+
execution = arm_payload.get("intervention_execution")
|
|
277
|
+
if not isinstance(execution, Mapping) or not isinstance(execution.get("usage"), Mapping):
|
|
278
|
+
raise OfficialArenaError("intervention usage is missing")
|
|
279
|
+
return execution["usage"]
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def merge_usage(*values: Mapping[str, Any]) -> dict[str, int | float]:
|
|
283
|
+
integer_fields = (
|
|
284
|
+
"model_calls",
|
|
285
|
+
"retrieval_calls",
|
|
286
|
+
"tool_calls",
|
|
287
|
+
"embedding_calls",
|
|
288
|
+
"unpriced_model_calls",
|
|
289
|
+
"input_tokens",
|
|
290
|
+
"output_tokens",
|
|
291
|
+
)
|
|
292
|
+
float_fields = (
|
|
293
|
+
"cpu_seconds",
|
|
294
|
+
"accelerator_seconds",
|
|
295
|
+
"cost_usd",
|
|
296
|
+
"compute_cost_usd",
|
|
297
|
+
)
|
|
298
|
+
result: dict[str, int | float] = {name: 0 for name in integer_fields}
|
|
299
|
+
result.update({name: 0.0 for name in float_fields})
|
|
300
|
+
for value in values:
|
|
301
|
+
if not isinstance(value, Mapping):
|
|
302
|
+
raise OfficialArenaError("usage contribution must be an object")
|
|
303
|
+
for name in integer_fields:
|
|
304
|
+
raw = value.get(name, 0)
|
|
305
|
+
if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0:
|
|
306
|
+
raise OfficialArenaError(f"usage field {name} must be non-negative integer")
|
|
307
|
+
result[name] += raw
|
|
308
|
+
for name in float_fields:
|
|
309
|
+
raw = value.get(name, 0.0)
|
|
310
|
+
if (
|
|
311
|
+
isinstance(raw, bool)
|
|
312
|
+
or not isinstance(raw, (int, float))
|
|
313
|
+
or not math.isfinite(float(raw))
|
|
314
|
+
or raw < 0
|
|
315
|
+
):
|
|
316
|
+
raise OfficialArenaError(f"usage field {name} must be non-negative finite")
|
|
317
|
+
result[name] += float(raw)
|
|
318
|
+
return result
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
__all__ = [
|
|
322
|
+
"EXECUTABLE_PAYLOAD_PROTOCOL",
|
|
323
|
+
"REPLAY_INVENTORY_PROTOCOL",
|
|
324
|
+
"OfficialArenaError",
|
|
325
|
+
"OfficialArenaProvider",
|
|
326
|
+
"ProviderArmResult",
|
|
327
|
+
"canonical_json",
|
|
328
|
+
"content_hash",
|
|
329
|
+
"intervention_usage",
|
|
330
|
+
"materialized_state",
|
|
331
|
+
"merge_usage",
|
|
332
|
+
"sha256_file",
|
|
333
|
+
"validate_dependency_hashes",
|
|
334
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Official arena provider implementations."""
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""Optional native-message history view; no native state/tool object is rewritten."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import copy
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from ..protocol import OfficialArenaError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def validate_history_transform(hook):
|
|
13
|
+
if hook is not None and not all(callable(getattr(hook, name, None))
|
|
14
|
+
for name in ("transform", "fingerprint")):
|
|
15
|
+
raise OfficialArenaError("native history requires transform and fingerprint methods")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class NativeHistorySession:
|
|
19
|
+
"""Separate helper usage from target usage, enforcing their shared caps."""
|
|
20
|
+
|
|
21
|
+
def __init__(self, hook, materialized, budget):
|
|
22
|
+
validate_history_transform(hook)
|
|
23
|
+
self.hook = hook
|
|
24
|
+
self.materialized = copy.deepcopy(dict(materialized))
|
|
25
|
+
self.budget = dict(budget)
|
|
26
|
+
self.usage = {}
|
|
27
|
+
|
|
28
|
+
def remaining(self, agent_usage):
|
|
29
|
+
fields = {"max_model_calls": "model_calls", "max_output_tokens": "output_tokens"}
|
|
30
|
+
return {cap: int(self.budget[cap]) - int(agent_usage.get(name, 0))
|
|
31
|
+
- int(self.usage.get(name, 0)) for cap, name in fields.items()}
|
|
32
|
+
|
|
33
|
+
def messages(self, native, *, encode, agent_usage):
|
|
34
|
+
"""Map only system/first-user text; reuse retained native objects exactly."""
|
|
35
|
+
from agentinputlab.integrations.live_history import _balanced_tool_frames
|
|
36
|
+
|
|
37
|
+
raw = encode(native)
|
|
38
|
+
logical = self._transform(raw, agent_usage=agent_usage)
|
|
39
|
+
first_user = next((i for i, row in enumerate(raw) if row.get("role") == "user"), None)
|
|
40
|
+
if (first_user is None or first_user == 0 or raw[0].get("role") != "system"
|
|
41
|
+
or len(logical) <= first_user or logical[0].get("role") != "system"
|
|
42
|
+
or logical[first_user].get("role") != "user"):
|
|
43
|
+
raise OfficialArenaError("native history changed the system/first-user framing")
|
|
44
|
+
if list(logical[1:first_user]) != raw[1:first_user]:
|
|
45
|
+
raise OfficialArenaError("native history changed a pre-user native frame")
|
|
46
|
+
text_slots = {0, first_user}
|
|
47
|
+
result, cursor = [], 0
|
|
48
|
+
for i, row in enumerate(logical):
|
|
49
|
+
row = dict(row)
|
|
50
|
+
if i in text_slots:
|
|
51
|
+
original = raw[i]
|
|
52
|
+
if ({k: v for k, v in row.items() if k != "content"}
|
|
53
|
+
!= {k: v for k, v in original.items() if k != "content"}
|
|
54
|
+
or not isinstance(row.get("content"), str)):
|
|
55
|
+
raise OfficialArenaError("native history changed non-text first-turn fields")
|
|
56
|
+
result.append(native[i].model_copy(update={"content": row["content"]}, deep=True))
|
|
57
|
+
cursor = i + 1
|
|
58
|
+
else:
|
|
59
|
+
index = next((j for j in range(cursor, len(raw)) if raw[j] == row), None)
|
|
60
|
+
if index is None:
|
|
61
|
+
raise OfficialArenaError("native history invented or reordered a retained frame")
|
|
62
|
+
result.append(copy.deepcopy(native[index]))
|
|
63
|
+
cursor = index + 1
|
|
64
|
+
if encode(result) != list(logical):
|
|
65
|
+
raise OfficialArenaError("native history message roundtrip changed wire fields")
|
|
66
|
+
if not _balanced_tool_frames(logical):
|
|
67
|
+
raise OfficialArenaError("native history orphaned tool frames")
|
|
68
|
+
return result
|
|
69
|
+
|
|
70
|
+
def _transform(self, raw, *, agent_usage):
|
|
71
|
+
"""Shared measured helper budget; protocol-specific roundtrips stay separate."""
|
|
72
|
+
from agentinputlab.observability import external_usage_counters, normalize_external_usage
|
|
73
|
+
|
|
74
|
+
remaining = self.remaining(agent_usage)
|
|
75
|
+
answer = self.hook.transform(copy.deepcopy(raw), state=copy.deepcopy(self.materialized),
|
|
76
|
+
remaining_budget=remaining)
|
|
77
|
+
if not isinstance(answer, Mapping) or set(answer) != {"messages", "usage"}:
|
|
78
|
+
raise OfficialArenaError("native history response fields differ")
|
|
79
|
+
usage = answer["usage"]
|
|
80
|
+
if not isinstance(usage, Mapping) or not {"model_calls", "output_tokens"} <= usage.keys():
|
|
81
|
+
raise OfficialArenaError("native history requires actual call/output usage")
|
|
82
|
+
if usage["model_calls"] and not {"input_tokens", "retrieval_calls"} <= usage.keys():
|
|
83
|
+
raise OfficialArenaError("native history calls require complete actual usage")
|
|
84
|
+
usage = external_usage_counters(normalize_external_usage({
|
|
85
|
+
"model_calls": 0, "retrieval_calls": 0, "input_tokens": 0, "output_tokens": 0, **usage}))
|
|
86
|
+
for key, value in usage.items():
|
|
87
|
+
self.usage[key] = self.usage.get(key, 0) + value
|
|
88
|
+
if not usage["model_calls"] and (usage["input_tokens"] or usage["output_tokens"]):
|
|
89
|
+
raise OfficialArenaError("skipped native history cannot report model tokens")
|
|
90
|
+
if usage["tool_calls"] or usage["retrieval_calls"] or usage["embedding_calls"]:
|
|
91
|
+
raise OfficialArenaError("native history cannot execute tools or retrieval")
|
|
92
|
+
if usage["model_calls"] >= remaining["max_model_calls"] or usage["output_tokens"] >= remaining["max_output_tokens"]:
|
|
93
|
+
raise OfficialArenaError("native history failed to reserve downstream budget")
|
|
94
|
+
logical = answer["messages"]
|
|
95
|
+
if not isinstance(logical, (list, tuple)) or any(not isinstance(row, Mapping) for row in logical):
|
|
96
|
+
raise OfficialArenaError("native history messages are invalid")
|
|
97
|
+
return list(logical)
|
|
98
|
+
|
|
99
|
+
def combined_usage(self, target_usage):
|
|
100
|
+
result = dict(target_usage)
|
|
101
|
+
for key, value in self.usage.items():
|
|
102
|
+
result[key] = result.get(key, 0) + value
|
|
103
|
+
return result
|
|
104
|
+
|
|
105
|
+
def write_receipt(self, path: Path, *, target_usage):
|
|
106
|
+
value = dict(fingerprint=self.hook.fingerprint(), records=list(getattr(self.hook, "records", ())),
|
|
107
|
+
history_usage=dict(self.usage), target_usage=dict(target_usage),
|
|
108
|
+
agent_and_history_usage=self.combined_usage(target_usage),
|
|
109
|
+
original_shared_budget=self.budget,
|
|
110
|
+
native_environment_history_NOT_modified=True)
|
|
111
|
+
if hasattr(self, "responses_records"):
|
|
112
|
+
value["responses_view"] = dict(protocol="native.responses.user_boundary.v1",
|
|
113
|
+
records=list(self.responses_records))
|
|
114
|
+
if path.exists():
|
|
115
|
+
if json.loads(path.read_text(encoding="utf-8")) != value:
|
|
116
|
+
raise OfficialArenaError("native history receipt changed before finalization")
|
|
117
|
+
return
|
|
118
|
+
with path.open("x", encoding="utf-8") as handle:
|
|
119
|
+
json.dump(value, handle, sort_keys=True, indent=2, allow_nan=False)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class ResponsesHistorySession(NativeHistorySession):
|
|
123
|
+
"""A reversible optimizer view, never a Chat-Completions wire conversion.
|
|
124
|
+
|
|
125
|
+
Real instructions and injected system items are immutable. Each complete
|
|
126
|
+
assistant/tool episode between actual user messages is one atomic text view
|
|
127
|
+
for ACON. Retained Items are deep-copied from the original native objects,
|
|
128
|
+
including reasoning, annotations, call IDs, arguments and tool results.
|
|
129
|
+
Invoke only at new user turns, not inside a function-call continuation.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def __init__(self, *args, **kwargs):
|
|
133
|
+
super().__init__(*args, **kwargs)
|
|
134
|
+
self.responses_records = []
|
|
135
|
+
|
|
136
|
+
def responses(self, native, *, instructions, agent_usage):
|
|
137
|
+
from agentinputlab.contracts.serialization import canonical_json, content_hash
|
|
138
|
+
|
|
139
|
+
if not isinstance(instructions, str) or not instructions:
|
|
140
|
+
raise OfficialArenaError("Responses history requires real text instructions")
|
|
141
|
+
if not isinstance(native, (list, tuple)):
|
|
142
|
+
raise OfficialArenaError("Responses history requires native input Items")
|
|
143
|
+
rows = []
|
|
144
|
+
for item in native:
|
|
145
|
+
row = dict(item) if isinstance(item, Mapping) else item.model_dump(mode="json")
|
|
146
|
+
if not isinstance(row, dict):
|
|
147
|
+
raise OfficialArenaError("Responses history Item is not serializable")
|
|
148
|
+
rows.append(row)
|
|
149
|
+
system_indices = [i for i, row in enumerate(rows) if row.get("role") in {"system", "developer"}]
|
|
150
|
+
policy = canonical_json(dict(instructions=instructions,
|
|
151
|
+
input_policy_items=[rows[i] for i in system_indices]))
|
|
152
|
+
raw = [dict(role="system", content=policy)]
|
|
153
|
+
groups = []
|
|
154
|
+
pending_group = []
|
|
155
|
+
|
|
156
|
+
def flush():
|
|
157
|
+
if not pending_group:
|
|
158
|
+
return
|
|
159
|
+
pending, seen = set(), set()
|
|
160
|
+
for i in pending_group:
|
|
161
|
+
row = rows[i]
|
|
162
|
+
if row.get("type") == "function_call":
|
|
163
|
+
call = row.get("call_id")
|
|
164
|
+
if not isinstance(call, str) or not call or call in seen:
|
|
165
|
+
raise OfficialArenaError("Responses history has duplicate or missing call_id")
|
|
166
|
+
pending.add(call)
|
|
167
|
+
seen.add(call)
|
|
168
|
+
elif row.get("type") == "function_call_output":
|
|
169
|
+
call = row.get("call_id")
|
|
170
|
+
if call not in pending:
|
|
171
|
+
raise OfficialArenaError("Responses history has an orphaned function result")
|
|
172
|
+
pending.remove(call)
|
|
173
|
+
if pending:
|
|
174
|
+
raise OfficialArenaError("Responses history has an incomplete function episode")
|
|
175
|
+
raw.append(dict(role="assistant", content=canonical_json([rows[i] for i in pending_group])))
|
|
176
|
+
groups.append(list(pending_group))
|
|
177
|
+
pending_group.clear()
|
|
178
|
+
|
|
179
|
+
for i, row in enumerate(rows):
|
|
180
|
+
if i in system_indices:
|
|
181
|
+
continue # Preserved on the native wire and included in the policy view.
|
|
182
|
+
if row.get("role") == "user":
|
|
183
|
+
flush()
|
|
184
|
+
content = row.get("content")
|
|
185
|
+
if isinstance(content, list) and len(content) == 1 and isinstance(content[0], Mapping):
|
|
186
|
+
if content[0].get("type") != "input_text":
|
|
187
|
+
raise OfficialArenaError("Responses history user Item is not plain text")
|
|
188
|
+
content = content[0].get("text")
|
|
189
|
+
if not isinstance(content, str):
|
|
190
|
+
raise OfficialArenaError("Responses history user Item is not plain text")
|
|
191
|
+
raw.append(dict(role="user", content=content))
|
|
192
|
+
groups.append([i])
|
|
193
|
+
else:
|
|
194
|
+
pending_group.append(i)
|
|
195
|
+
flush()
|
|
196
|
+
if len(raw) < 2 or raw[1]["role"] != "user" or raw[-1]["role"] != "user":
|
|
197
|
+
raise OfficialArenaError("Responses history requires an actual new user-turn boundary")
|
|
198
|
+
logical = self._transform(raw, agent_usage=agent_usage)
|
|
199
|
+
if (len(logical) < 2 or logical[0] != raw[0]
|
|
200
|
+
or set(logical[1]) != {"role", "content"}
|
|
201
|
+
or logical[1].get("role") != "user" or not isinstance(logical[1].get("content"), str)):
|
|
202
|
+
raise OfficialArenaError("Responses history changed immutable policy or first-user framing")
|
|
203
|
+
selected = set(system_indices + groups[0])
|
|
204
|
+
cursor = 2
|
|
205
|
+
for row in logical[2:]:
|
|
206
|
+
index = next((j for j in range(cursor, len(raw)) if raw[j] == row), None)
|
|
207
|
+
if index is None:
|
|
208
|
+
raise OfficialArenaError("Responses history invented or reordered native Items")
|
|
209
|
+
selected.update(groups[index - 1])
|
|
210
|
+
cursor = index + 1
|
|
211
|
+
# The current real user request must remain; no model-generated turn is inserted.
|
|
212
|
+
if not set(groups[-1]) <= selected:
|
|
213
|
+
raise OfficialArenaError("Responses history removed the current user turn")
|
|
214
|
+
result = [copy.deepcopy(item) for i, item in enumerate(native) if i in selected]
|
|
215
|
+
first_index = groups[0][0]
|
|
216
|
+
first_output = sorted(selected).index(first_index)
|
|
217
|
+
original = rows[first_index]
|
|
218
|
+
content = logical[1]["content"]
|
|
219
|
+
if isinstance(original["content"], list):
|
|
220
|
+
content = [{**original["content"][0], "text": content}]
|
|
221
|
+
if isinstance(result[first_output], Mapping):
|
|
222
|
+
result[first_output] = {**result[first_output], "content": content}
|
|
223
|
+
else:
|
|
224
|
+
result[first_output] = result[first_output].model_copy(update={"content": content}, deep=True)
|
|
225
|
+
self.responses_records.append(dict(native_input_hash=content_hash(rows),
|
|
226
|
+
optimizer_view_hash=content_hash(raw), retained_source_item_indices=sorted(selected),
|
|
227
|
+
policy_unchanged=True, native_items_NOT_chat_messages=True))
|
|
228
|
+
return result, instructions
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Narrow message-shape compatibility helpers for self-hosted model templates."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def requires_qwen38_initial_system_message(model: Any) -> bool:
|
|
10
|
+
"""Return whether the served alias uses Qwen3.8's strict system placement."""
|
|
11
|
+
|
|
12
|
+
if not isinstance(model, str):
|
|
13
|
+
return False
|
|
14
|
+
alias = model.strip().casefold().replace("_", "-")
|
|
15
|
+
return alias == "qwen3.8" or alias.startswith("qwen3.8-")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def normalize_qwen38_reasoning_effort(model: Any, effort: Any) -> Any:
|
|
19
|
+
"""Map STATE-Bench's strongest effort label to Qwen3.8's equivalent."""
|
|
20
|
+
|
|
21
|
+
if requires_qwen38_initial_system_message(model) and effort == "high":
|
|
22
|
+
return "xhigh"
|
|
23
|
+
return effort
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def coalesce_chat_system_messages(messages: Sequence[Any]) -> list[Any]:
|
|
27
|
+
"""Move chat system content into one leading message without reordering peers."""
|
|
28
|
+
|
|
29
|
+
copied = [dict(item) if isinstance(item, Mapping) else item for item in messages]
|
|
30
|
+
system_items: list[dict[str, Any]] = []
|
|
31
|
+
non_system_items: list[Any] = []
|
|
32
|
+
for item in copied:
|
|
33
|
+
if isinstance(item, Mapping) and item.get("role") == "system":
|
|
34
|
+
content = item.get("content")
|
|
35
|
+
if not isinstance(content, str):
|
|
36
|
+
raise TypeError("Qwen3.8 system message content must be text")
|
|
37
|
+
system_items.append(dict(item))
|
|
38
|
+
else:
|
|
39
|
+
non_system_items.append(item)
|
|
40
|
+
if not system_items:
|
|
41
|
+
return copied
|
|
42
|
+
merged = dict(system_items[0])
|
|
43
|
+
merged["content"] = "\n\n".join(item["content"] for item in system_items)
|
|
44
|
+
return [merged, *non_system_items]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def fold_responses_system_messages_into_instructions(
|
|
48
|
+
input_items: Sequence[Any],
|
|
49
|
+
instructions: Any,
|
|
50
|
+
) -> tuple[list[Any], Any]:
|
|
51
|
+
"""Fold Responses input system items into its single instructions field.
|
|
52
|
+
|
|
53
|
+
Qwen3.8's chat template accepts only the leading system slot. The Responses
|
|
54
|
+
transport renders ``instructions`` there, so later system-role input items
|
|
55
|
+
must be removed and appended to that field in their original order.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
copied = [dict(item) if isinstance(item, Mapping) else item for item in input_items]
|
|
59
|
+
system_contents: list[str] = []
|
|
60
|
+
non_system_items: list[Any] = []
|
|
61
|
+
for item in copied:
|
|
62
|
+
if isinstance(item, Mapping) and item.get("role") == "system":
|
|
63
|
+
content = item.get("content")
|
|
64
|
+
if not isinstance(content, str):
|
|
65
|
+
raise TypeError("Qwen3.8 Responses system item content must be text")
|
|
66
|
+
system_contents.append(content)
|
|
67
|
+
else:
|
|
68
|
+
non_system_items.append(item)
|
|
69
|
+
if not system_contents:
|
|
70
|
+
return copied, instructions
|
|
71
|
+
if instructions is not None and not isinstance(instructions, str):
|
|
72
|
+
raise TypeError("Qwen3.8 Responses instructions must be text or None")
|
|
73
|
+
parts = []
|
|
74
|
+
if instructions:
|
|
75
|
+
parts.append(instructions)
|
|
76
|
+
parts.extend(system_contents)
|
|
77
|
+
return non_system_items, "\n\n".join(parts)
|