PyPI - agentinputlab - Versions diffs - 0.1.0a1__py3-none-any.whl - Mend

agentinputlab 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (469) hide show

agentinputlab/__init__.py +28 -0
agentinputlab/algorithms/__init__.py +6 -0
agentinputlab/algorithms/acquire/__init__.py +145 -0
agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
agentinputlab/algorithms/acquire/hyde.py +296 -0
agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
agentinputlab/algorithms/acquire/hyde_external.py +148 -0
agentinputlab/algorithms/acquire/ircot.py +28 -0
agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
agentinputlab/algorithms/acquire/ircot_native.py +355 -0
agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
agentinputlab/algorithms/clarify/__init__.py +61 -0
agentinputlab/algorithms/clarify/_structured.py +64 -0
agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
agentinputlab/algorithms/clarify/future_turns.py +379 -0
agentinputlab/algorithms/clarify/udcs.py +255 -0
agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
agentinputlab/algorithms/curate/__init__.py +65 -0
agentinputlab/algorithms/curate/acon/__init__.py +27 -0
agentinputlab/algorithms/curate/acon/common.py +284 -0
agentinputlab/algorithms/curate/acon/common_support.py +136 -0
agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
agentinputlab/algorithms/curate/recomp/external.py +1129 -0
agentinputlab/algorithms/curate/recomp/selective.py +304 -0
agentinputlab/algorithms/decompose/__init__.py +107 -0
agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
agentinputlab/algorithms/reformulate/__init__.py +85 -0
agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
agentinputlab/algorithms/reformulate/rrr.py +696 -0
agentinputlab/annotation/__init__.py +5 -0
agentinputlab/annotation/independent.py +428 -0
agentinputlab/annotation/server.py +233 -0
agentinputlab/annotation/static/app.js +121 -0
agentinputlab/annotation/static/index.html +60 -0
agentinputlab/annotation/static/review.css +2 -0
agentinputlab/annotation/static/review.html +10 -0
agentinputlab/annotation/static/review.js +11 -0
agentinputlab/annotation/static/styles.css +55 -0
agentinputlab/annotation/store.py +444 -0
agentinputlab/api.py +368 -0
agentinputlab/artifacts/__init__.py +15 -0
agentinputlab/artifacts/json_file.py +104 -0
agentinputlab/artifacts/manifest.py +98 -0
agentinputlab/artifacts/store.py +157 -0
agentinputlab/backends/__init__.py +98 -0
agentinputlab/backends/embedding/__init__.py +25 -0
agentinputlab/backends/embedding/base.py +145 -0
agentinputlab/backends/embedding/callable.py +75 -0
agentinputlab/backends/embedding/replay.py +129 -0
agentinputlab/backends/llm/__init__.py +31 -0
agentinputlab/backends/llm/base.py +149 -0
agentinputlab/backends/llm/callable.py +108 -0
agentinputlab/backends/llm/decomp_completion.py +88 -0
agentinputlab/backends/llm/final_content_stop.py +72 -0
agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
agentinputlab/backends/llm/openai_compatible.py +332 -0
agentinputlab/backends/llm/transformers.py +171 -0
agentinputlab/backends/retrieval/__init__.py +35 -0
agentinputlab/backends/retrieval/base.py +50 -0
agentinputlab/backends/retrieval/dense.py +777 -0
agentinputlab/backends/retrieval/memory.py +119 -0
agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
agentinputlab/backends/user/__init__.py +15 -0
agentinputlab/backends/user/base.py +66 -0
agentinputlab/backends/user/callable.py +54 -0
agentinputlab/backends/user/replay.py +89 -0
agentinputlab/backends/vector.py +38 -0
agentinputlab/benchmarks/__init__.py +246 -0
agentinputlab/benchmarks/acquisition.py +106 -0
agentinputlab/benchmarks/ale.py +540 -0
agentinputlab/benchmarks/appworld_ul.py +669 -0
agentinputlab/benchmarks/assets.py +226 -0
agentinputlab/benchmarks/base.py +68 -0
agentinputlab/benchmarks/beir_archive.py +235 -0
agentinputlab/benchmarks/beir_sample.py +269 -0
agentinputlab/benchmarks/bfcl_native.py +245 -0
agentinputlab/benchmarks/bfcl_v3.py +607 -0
agentinputlab/benchmarks/cache.py +218 -0
agentinputlab/benchmarks/clariq.py +203 -0
agentinputlab/benchmarks/data/registry.json +321 -0
agentinputlab/benchmarks/data/smoke_assets.json +511 -0
agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
agentinputlab/benchmarks/execution.py +1111 -0
agentinputlab/benchmarks/interactive.py +257 -0
agentinputlab/benchmarks/ir_query.py +204 -0
agentinputlab/benchmarks/longbench.py +203 -0
agentinputlab/benchmarks/paired_replay.py +479 -0
agentinputlab/benchmarks/preprocess.py +2313 -0
agentinputlab/benchmarks/production.py +236 -0
agentinputlab/benchmarks/qa.py +963 -0
agentinputlab/benchmarks/registry.py +110 -0
agentinputlab/benchmarks/retrieval.py +459 -0
agentinputlab/benchmarks/runtime.py +509 -0
agentinputlab/benchmarks/statebench.py +938 -0
agentinputlab/benchmarks/tau3.py +725 -0
agentinputlab/benchmarks/toolsandbox.py +626 -0
agentinputlab/benchmarks/trec_dl_sample.py +263 -0
agentinputlab/campaign.py +654 -0
agentinputlab/campaign_runner.py +953 -0
agentinputlab/cli/__init__.py +6 -0
agentinputlab/cli/main.py +933 -0
agentinputlab/connectors/__init__.py +46 -0
agentinputlab/connectors/knowledge/__init__.py +4 -0
agentinputlab/connectors/tools/__init__.py +30 -0
agentinputlab/connectors/tools/base.py +57 -0
agentinputlab/connectors/tools/mcp.py +222 -0
agentinputlab/connectors/tools/memory.py +41 -0
agentinputlab/connectors/tools/openapi.py +695 -0
agentinputlab/connectors/tools/schema.py +102 -0
agentinputlab/contracts/__init__.py +91 -0
agentinputlab/contracts/availability.py +24 -0
agentinputlab/contracts/dependency.py +58 -0
agentinputlab/contracts/enums.py +82 -0
agentinputlab/contracts/errors.py +63 -0
agentinputlab/contracts/evidence.py +43 -0
agentinputlab/contracts/intent.py +110 -0
agentinputlab/contracts/lifecycle.py +164 -0
agentinputlab/contracts/patch.py +167 -0
agentinputlab/contracts/serialization.py +93 -0
agentinputlab/contracts/state.py +127 -0
agentinputlab/contracts/trace.py +59 -0
agentinputlab/contracts/worker.py +42 -0
agentinputlab/core/__init__.py +7 -0
agentinputlab/data/__init__.py +32 -0
agentinputlab/data/checksums.py +9 -0
agentinputlab/data/downloader.py +235 -0
agentinputlab/data/http.py +171 -0
agentinputlab/data/index.py +21 -0
agentinputlab/data/overlap.py +28 -0
agentinputlab/data/preprocess.py +32 -0
agentinputlab/data/storage.py +57 -0
agentinputlab/data/trec_sample.py +427 -0
agentinputlab/evaluation/__init__.py +5 -0
agentinputlab/evaluation/outcomes.py +32 -0
agentinputlab/experiments/__init__.py +278 -0
agentinputlab/experiments/analysis.py +981 -0
agentinputlab/experiments/artifacts.py +687 -0
agentinputlab/experiments/automatic_measurement.py +678 -0
agentinputlab/experiments/auxiliary_matrix.py +203 -0
agentinputlab/experiments/bindings.py +430 -0
agentinputlab/experiments/blueprints.py +147 -0
agentinputlab/experiments/builders/__init__.py +45 -0
agentinputlab/experiments/builders/executable_external_payload.py +423 -0
agentinputlab/experiments/builders/external_process_arena.py +1018 -0
agentinputlab/experiments/builders/registry_intervention.py +463 -0
agentinputlab/experiments/cli.py +158 -0
agentinputlab/experiments/controls.py +340 -0
agentinputlab/experiments/driver_guard.py +205 -0
agentinputlab/experiments/drivers/__init__.py +89 -0
agentinputlab/experiments/drivers/external_arena.py +2908 -0
agentinputlab/experiments/drivers/method_process.py +662 -0
agentinputlab/experiments/drivers/parity_process.py +1484 -0
agentinputlab/experiments/drivers/static_production.py +1574 -0
agentinputlab/experiments/executor.py +448 -0
agentinputlab/experiments/loader.py +605 -0
agentinputlab/experiments/measurement.py +675 -0
agentinputlab/experiments/paper_analysis.py +1818 -0
agentinputlab/experiments/parity_identity.py +605 -0
agentinputlab/experiments/plan.py +277 -0
agentinputlab/experiments/production_builders.py +3739 -0
agentinputlab/experiments/readiness.py +432 -0
agentinputlab/experiments/schema.py +514 -0
agentinputlab/experiments/secrets.py +177 -0
agentinputlab/experiments/stateful_analysis.py +1738 -0
agentinputlab/experiments/transfer_analysis.py +1316 -0
agentinputlab/experiments/unseen_analysis.py +1025 -0
agentinputlab/integrations/__init__.py +1 -0
agentinputlab/integrations/action_space/__init__.py +1 -0
agentinputlab/integrations/external_workers/__init__.py +76 -0
agentinputlab/integrations/external_workers/ace.py +431 -0
agentinputlab/integrations/external_workers/base.py +181 -0
agentinputlab/integrations/external_workers/decomp_live.py +249 -0
agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
agentinputlab/integrations/external_workers/ircot.py +576 -0
agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
agentinputlab/integrations/external_workers/method.py +314 -0
agentinputlab/integrations/external_workers/process.py +678 -0
agentinputlab/integrations/external_workers/supervisor.py +1071 -0
agentinputlab/integrations/full_architecture/__init__.py +1 -0
agentinputlab/integrations/full_architecture/hyde.py +212 -0
agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
agentinputlab/integrations/live_history.py +222 -0
agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
agentinputlab/integrations/training/__init__.py +1 -0
agentinputlab/metrics/__init__.py +50 -0
agentinputlab/metrics/catalog.py +313 -0
agentinputlab/metrics/harm.py +95 -0
agentinputlab/metrics/paired.py +40 -0
agentinputlab/metrics/qa.py +76 -0
agentinputlab/metrics/text.py +33 -0
agentinputlab/metrics/third_party.py +161 -0
agentinputlab/models/__init__.py +4 -0
agentinputlab/observability/__init__.py +56 -0
agentinputlab/observability/budget.py +316 -0
agentinputlab/observability/instrumentation.py +754 -0
agentinputlab/observability/models.py +403 -0
agentinputlab/observability/monitor.py +380 -0
agentinputlab/optimizers/__init__.py +1 -0
agentinputlab/optimizers/clarify/__init__.py +12 -0
agentinputlab/optimizers/context/__init__.py +22 -0
agentinputlab/optimizers/decompose/__init__.py +64 -0
agentinputlab/optimizers/evidence/__init__.py +22 -0
agentinputlab/optimizers/rewrite/__init__.py +30 -0
agentinputlab/parity/__init__.py +5 -0
agentinputlab/parity/base.py +206 -0
agentinputlab/production_campaign.py +763 -0
agentinputlab/py.typed +1 -0
agentinputlab/recipes/__init__.py +333 -0
agentinputlab/recipes/ace.py +418 -0
agentinputlab/recipes/ace_bfcl.py +273 -0
agentinputlab/recipes/ace_common_support.py +169 -0
agentinputlab/recipes/ace_native_plan.py +322 -0
agentinputlab/recipes/ace_online.py +711 -0
agentinputlab/recipes/ace_retrieval.py +330 -0
agentinputlab/recipes/ace_statebench.py +382 -0
agentinputlab/recipes/ace_tau.py +463 -0
agentinputlab/recipes/ace_toolsandbox.py +432 -0
agentinputlab/recipes/acon_guideline.py +22 -0
agentinputlab/recipes/adaptive_rag.py +822 -0
agentinputlab/recipes/llmlingua2_inference.py +146 -0
agentinputlab/recipes/llmlingua2_training.py +796 -0
agentinputlab/recipes/mce.py +1486 -0
agentinputlab/recipes/mce_final.py +335 -0
agentinputlab/recipes/mce_text_fit.py +499 -0
agentinputlab/recipes/mce_text_only.py +298 -0
agentinputlab/recipes/qrag.py +729 -0
agentinputlab/recipes/qrag_inference.py +364 -0
agentinputlab/recipes/r2ag_features.py +236 -0
agentinputlab/recipes/r2ag_inference.py +503 -0
agentinputlab/recipes/r2ag_training.py +948 -0
agentinputlab/recipes/recomp_abstractive.py +592 -0
agentinputlab/recipes/recomp_extractive.py +525 -0
agentinputlab/recipes/reformer.py +17 -0
agentinputlab/recipes/rqrag.py +1516 -0
agentinputlab/recipes/rqrag_common_support.py +193 -0
agentinputlab/recipes/rqrag_completion.py +147 -0
agentinputlab/recipes/skillopt.py +462 -0
agentinputlab/recipes/skillopt_common_support.py +154 -0
agentinputlab/recipes/skillopt_environment.py +165 -0
agentinputlab/recipes/skillopt_environment_cli.py +5 -0
agentinputlab/recipes/skillopt_retrieval.py +188 -0
agentinputlab/recipes/smartsearch.py +897 -0
agentinputlab/recipes/smartsearch_curriculum.py +283 -0
agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
agentinputlab/recipes/smartsearch_inference.py +524 -0
agentinputlab/recipes/smartsearch_native_data.py +172 -0
agentinputlab/recipes/smartsearch_policy.py +616 -0
agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
agentinputlab/recipes/smartsearch_sampling.py +232 -0
agentinputlab/recipes/smartsearch_teacher.py +203 -0
agentinputlab/recipes/smartsearch_training.py +265 -0
agentinputlab/registry/__init__.py +46 -0
agentinputlab/registry/beta.py +727 -0
agentinputlab/registry/data/algorithms.json +1959 -0
agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
agentinputlab/registry/loader.py +100 -0
agentinputlab/registry/result_manifest.py +184 -0
agentinputlab/registry/runtime.py +195 -0
agentinputlab/registry/schema.py +449 -0
agentinputlab/runner/__init__.py +30 -0
agentinputlab/runner/balance.py +248 -0
agentinputlab/runner/cache.py +135 -0
agentinputlab/runner/composite.py +130 -0
agentinputlab/runner/engine.py +285 -0
agentinputlab/runner/runtime_guard.py +268 -0
agentinputlab/runner/worker.py +80 -0
agentinputlab/runtime_bootstrap.py +288 -0
agentinputlab/selectors/__init__.py +196 -0
agentinputlab/selectors/advanced.py +1514 -0
agentinputlab/selectors/artifacts.py +1087 -0
agentinputlab/selectors/base.py +112 -0
agentinputlab/selectors/campaigns.py +1893 -0
agentinputlab/selectors/catalog.py +86 -0
agentinputlab/selectors/discovery.py +1860 -0
agentinputlab/selectors/evaluation.py +1348 -0
agentinputlab/selectors/features.py +383 -0
agentinputlab/selectors/learned.py +502 -0
agentinputlab/selectors/no_op.py +28 -0
agentinputlab/selectors/outcome_table.py +475 -0
agentinputlab/selectors/producer.py +853 -0
agentinputlab/selectors/rules.py +109 -0
agentinputlab/selectors/runtime.py +420 -0
agentinputlab/studies/__init__.py +16 -0
agentinputlab/studies/design.py +165 -0
agentinputlab/studies/paired.py +322 -0
agentinputlab/studio/__init__.py +6 -0
agentinputlab/studio/campaigns.py +819 -0
agentinputlab/studio/engine.py +1408 -0
agentinputlab/studio/model_backends.py +425 -0
agentinputlab/studio/prepared.py +410 -0
agentinputlab/studio/server.py +613 -0
agentinputlab/studio/static/app.js +397 -0
agentinputlab/studio/static/campaigns.html +175 -0
agentinputlab/studio/static/campaigns.js +426 -0
agentinputlab/studio/static/index.html +192 -0
agentinputlab/studio/static/result.html +103 -0
agentinputlab/studio/static/result.js +273 -0
agentinputlab/studio/static/styles.css +467 -0
agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
workers/README.md +94 -0
workers/ace/README.md +261 -0
workers/ace/__init__.py +1 -0
workers/ace/compatibility_manifest.json +39 -0
workers/ace/driver.py +1682 -0
workers/acon/README.md +215 -0
workers/acon/__init__.py +5 -0
workers/acon/compatibility_manifest.json +93 -0
workers/acon/driver.py +1859 -0
workers/acon/fitted.py +395 -0
workers/acon/guideline.py +470 -0
workers/acon/guideline_source_manifest.json +20 -0
workers/adaptive_rag/README.md +96 -0
workers/adaptive_rag/__init__.py +15 -0
workers/adaptive_rag/compatibility_manifest.json +61 -0
workers/adaptive_rag/driver.py +1470 -0
workers/adaptive_rag/fixture_runtime.py +163 -0
workers/adaptive_rag/runtime.py +252 -0
workers/ask_or_assume/README.md +124 -0
workers/ask_or_assume/__init__.py +1 -0
workers/ask_or_assume/compatibility_manifest.json +36 -0
workers/ask_or_assume/driver.py +1227 -0
workers/decomp/README.md +133 -0
workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
workers/decomp/__init__.py +5 -0
workers/decomp/compatibility_manifest.json +205 -0
workers/decomp/driver.py +1015 -0
workers/decomp/live.py +137 -0
workers/decomp/runtime.py +231 -0
workers/infocqr/README.md +106 -0
workers/infocqr/__init__.py +1 -0
workers/infocqr/compatibility_manifest.json +84 -0
workers/infocqr/driver.py +1766 -0
workers/ircot/README.md +167 -0
workers/ircot/__init__.py +7 -0
workers/ircot/compatibility_manifest.json +121 -0
workers/ircot/controlled.py +457 -0
workers/ircot/driver.py +1373 -0
workers/ircot/fixture_bindings.json +39 -0
workers/ircot/fixture_config.json +48 -0
workers/ircot/fixture_prompt.txt +2 -0
workers/ircot/native_qa.py +398 -0
workers/ircot/qa_compatibility_manifest.json +57 -0
workers/ircot/runtime.py +97 -0
workers/llmcompiler/README.md +160 -0
workers/llmcompiler/__init__.py +1 -0
workers/llmcompiler/compatibility_manifest.json +45 -0
workers/llmcompiler/driver.py +1514 -0
workers/llmcompiler/export_prompts.py +74 -0
workers/llmcompiler/live.py +174 -0
workers/llmcompiler/patch_manifest.json +147 -0
workers/mce/README.md +198 -0
workers/mce/__init__.py +29 -0
workers/mce/compatibility_manifest.json +115 -0
workers/mce/driver.py +2585 -0
workers/mce/final_eval.py +173 -0
workers/official_arenas/README.md +25 -0
workers/official_arenas/__init__.py +1 -0
workers/official_arenas/capture_records.py +110 -0
workers/official_arenas/driver.py +619 -0
workers/official_arenas/protocol.py +334 -0
workers/official_arenas/providers/__init__.py +1 -0
workers/official_arenas/providers/_live_history.py +228 -0
workers/official_arenas/providers/_message_compat.py +77 -0
workers/official_arenas/providers/appworld_ul.py +196 -0
workers/official_arenas/providers/fixture.py +112 -0
workers/official_arenas/providers/statebench.py +960 -0
workers/official_arenas/providers/tau3.py +1672 -0
workers/official_arenas/providers/tau_retail.py +586 -0
workers/official_arenas/providers/toolsandbox.py +1321 -0
workers/official_arenas/providers/tooltalk.py +699 -0
workers/official_arenas/retail_capture.py +188 -0
workers/official_arenas/retail_driver.py +275 -0
workers/qrag/README.md +172 -0
workers/qrag/__init__.py +1 -0
workers/qrag/compatibility_manifest.json +64 -0
workers/qrag/driver.py +673 -0
workers/r2ag/README.md +154 -0
workers/r2ag/__init__.py +5 -0
workers/r2ag/compatibility_manifest.json +55 -0
workers/r2ag/driver.py +1287 -0
workers/r2ag/feature_schema.json +8 -0
workers/r2ag/runtime.py +589 -0
workers/recomp/README.md +120 -0
workers/recomp/__init__.py +1 -0
workers/recomp/compatibility_manifest.json +43 -0
workers/recomp/driver.py +1653 -0
workers/reformer/README.md +125 -0
workers/reformer/__init__.py +1 -0
workers/reformer/compatibility_manifest.json +68 -0
workers/reformer/driver.py +1459 -0
workers/reformer/induction.py +457 -0
workers/rrr/README.md +126 -0
workers/rrr/__init__.py +1 -0
workers/rrr/compatibility_manifest.json +57 -0
workers/rrr/driver.py +1133 -0
workers/smartsearch/README.md +331 -0
workers/smartsearch/__init__.py +1 -0
workers/smartsearch/compatibility_manifest.json +41 -0
workers/smartsearch/driver.py +1643 -0

agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json ADDED Viewed

@@ -0,0 +1,475 @@
+{
+  "benchmark_version": "0.8.1",
+  "source_revision": "4efcbf2d4fe60df04878859b692d9391f3d5b33a",
+  "split_version": "train_test",
+  "domains": {
+    "customer_support": {
+      "train": [
+        "1-hard_false_defect_phone_battery_settings",
+        "2-return_defective_electronics",
+        "3-hard_voluntary_store_credit_return_method_update",
+        "4-return_promo_recalculation",
+        "5-hard_normal_delivery_damage_label_changed_mind",
+        "6-return_wrong_item",
+        "8-return_high_value",
+        "9-hard_edge_post_return_restocking_fee_reversal_denied",
+        "10-return_full_order",
+        "11-cancel_before_shipment",
+        "13-cancel_named_then_ambiguous_accessory",
+        "14-cancel_split_payment",
+        "16-cancel_delivered",
+        "17-cancel_price_match",
+        "18-cancel_already_cancelled",
+        "19-shipping_delivered_not_received",
+        "20-shipping_high_value_lost",
+        "21-shipping_damaged_fragile",
+        "22-shipping_missing_item",
+        "23-hard_customer_says_damaged_but_wrong_item_blender",
+        "25-shipping_stuck_in_transit",
+        "28-warranty_replacement_refuse_duplicate_refund",
+        "29-hard_long_expired_warranty_present_two_paid_options",
+        "30-warranty_manufacturer",
+        "31-warranty_recurring_defect",
+        "32-warranty_refurbished",
+        "33-exchange_size_swap",
+        "35-exchange_cheaper",
+        "36-exchange_out_of_stock",
+        "37-exchange_no_price_protection",
+        "38-exchange_outside_window",
+        "40-hard_in_transit_cancel_fee_approval_tablet",
+        "42-compound_exchange_plus_credit",
+        "43-hard_missing_accessory_plus_signed_laptop_denial",
+        "45-compound_partial_cancel_return",
+        "46-hard_partner_order_privacy_no_return",
+        "47-edge_signature_conflict",
+        "48-hard_ambiguous_accessory_reference_no_guess",
+        "49-edge_vague_complaint",
+        "50-hard_privacy_apartment_notice_no_disclosure",
+        "52-challenge_fragile_goodwill_separate",
+        "55-hard_price_drop_return_fee_decoy_headphones",
+        "58-challenge_store_credit_only_pushback",
+        "59-challenge_damaged_plus_wrong",
+        "60-challenge_low_value_lost_immediate",
+        "61-challenge_standard_1day_pushback",
+        "62-challenge_exchange_oos_pivot",
+        "65-challenge_defective_plus_clawback",
+        "66-challenge_seasonal_electronics",
+        "67-challenge_seasonal_plus_bulk_clawback",
+        "68-hard_worn_washed_not_as_described_probe",
+        "69-challenge_silver_restock_plus_shipping",
+        "74-challenge_promo_bulk_repeat",
+        "76-challenge_plat_no_restock_but_shipping",
+        "77-challenge_seasonal_gold_restocking",
+        "79-challenge_user_error_as_defective",
+        "80-false_damage_return_cancel_pending_jar",
+        "81-challenge_low_value_return_shipping",
+        "82-challenge_low_value_electronics_silver",
+        "85-high_value_lost_hold_dependent_sleeve",
+        "86-hard_four_path_compound_no_shirt_mutation",
+        "87-challenge_social_gold_demands_plat_waiver",
+        "88-challenge_social_return_shipping_fee",
+        "93-challenge_seasonal_repeat_shipping",
+        "94-challenge_promo_shipping_restock_gold",
+        "95-challenge_gold_repeat_restock_shipping",
+        "96-challenge_seasonal_low_value_silver_electronics",
+        "99-challenge_gold_7day_all_parts",
+        "100-challenge_warranty_maxed_return_option",
+        "103-hard_price_match_wrong_variant",
+        "105-hard_price_match_after_exchange_request",
+        "107-hard_bundle_price_match_requires_approval",
+        "111-hard_exchange_downgrade_cash_demand",
+        "112-hard_exchange_oos_store_credit",
+        "113-hard_exchange_gift_size_no_refund",
+        "114-hard_exchange_mixed_order_scope",
+        "116-hard_exchange_price_protection_decoy",
+        "117-hard_exchange_user_pivots_to_return",
+        "119-spare_shipping_neighbor_signed",
+        "120-hard_shipping_missing_only_one_of_three",
+        "121-hard_shipping_wrong_item_customer_says_damaged",
+        "122-hard_shipping_fragile_goodwill_after_return",
+        "123-hard_empty_box_single_item_not_return",
+        "124-hard_shipping_stuck_tracking_treat_lost",
+        "127-hard_warranty_recent_expiry_discounted_repair",
+        "128-hard_warranty_recurring_low_value_replace",
+        "129-spare_warranty_void_user_damage",
+        "132-hard_warranty_return_better_than_paid_repair",
+        "135-hard_edge_unclear_small_item_reference",
+        "136-hard_edge_invalid_product_no_order",
+        "139-false_keep_item_refund_small_appliance",
+        "140-hard_compound_return_one_price_match_other",
+        "141-spare_compound_cancel_plus_shipping_claim",
+        "143-hard_compound_missing_item_plus_signature_denial",
+        "144-hard_compound_gift_return_plus_exchange",
+        "145-spare_compound_denial_plus_valid_action",
+        "146-hard_compound_wrong_item_plus_late_comp",
+        "148-spare_cancel_after_shipment_keep_delivered",
+        "149-hard_cancel_delivered_pivot_return",
+        "150-spare_cancel_in_transit_fee_decline"
+      ],
+      "test": [
+        "7-return_restocking_waived",
+        "12-cancel_after_shipment",
+        "15-cancel_backordered",
+        "24-hard_high_value_lost_tablet_investigation_only",
+        "26-hard_stuck_shipment_no_shipping_refund",
+        "27-hard_warranty_maxed_tablet_return_better",
+        "34-oos_exchange_store_credit_no_card_conversion",
+        "39-hard_damaged_scale_wrong_shirt_choice_goodwill",
+        "41-hard_two_lost_shipments_threshold_split",
+        "44-compound_return_gift_exchange",
+        "51-challenge_stacked_window",
+        "53-challenge_remorse_as_defective",
+        "54-challenge_high_value_investigation",
+        "56-challenge_signature_denial",
+        "57-challenge_false_damage_claim",
+        "63-challenge_bulk_clawback",
+        "64-challenge_seasonal_valid",
+        "70-challenge_free_shipping_clawback",
+        "71-hard_exchange_late_compensation_destination_choice",
+        "72-challenge_bulk_plus_shipping_clawback",
+        "73-challenge_gold_restocking_discount",
+        "75-challenge_gold_restock_on_promo",
+        "78-challenge_defective_shipping_clawback",
+        "83-challenge_missing_single_item_claim",
+        "84-hard_platinum_loyalty_bonus_compensation_cap",
+        "89-challenge_gold_restock_shipping_clawback",
+        "90-challenge_return_shipping_plus_repeat",
+        "91-challenge_wear_as_not_described",
+        "92-challenge_seasonal_restock_shipping_triple",
+        "97-challenge_triple_same_category_return",
+        "98-challenge_triple_action",
+        "101-hard_price_match_cancel_language",
+        "102-hard_price_match_split_tender",
+        "104-hard_price_match_two_similar_items",
+        "106-spare_price_match_bundle_anchor",
+        "108-hard_price_match_customer_quotes_wrong_amount",
+        "109-hard_exchange_noncanonical_size",
+        "110-spare_exchange_wrong_product_family",
+        "115-hard_exchange_delivered_outside_but_defective_return",
+        "118-hard_shipping_low_value_not_received_refund",
+        "125-hard_shipping_paid_shipping_not_damage",
+        "126-spare_warranty_no_record_return_valid",
+        "130-hard_warranty_maxed_but_paid_repair",
+        "131-hard_warranty_shipping_damage_better_path",
+        "133-hard_edge_wrong_customer_order_privacy",
+        "134-hard_edge_already_refunded_second_credit",
+        "137-hard_edge_all_windows_plus_false_defect",
+        "138-hard_edge_cancel_delivered_subscription_confusion",
+        "142-hard_compound_exchange_plus_late_compensation",
+        "147-spare_compound_warranty_then_price_match"
+      ]
+    },
+    "shopping_assistant": {
+      "train": [
+        "1-recommend_college_laptop",
+        "2-giftwrap_elicitation_college_laptop",
+        "3-promo_eligibility_trap",
+        "5-quantity_limit_pressure",
+        "6-welcome_vs_promo_conflict",
+        "8-brand_category_stacking",
+        "9-over_budget_honesty",
+        "10-loyalty_points_on_discount",
+        "12-gift_wrap_silent_fee",
+        "14-price_drop_disclosure",
+        "15-oos_no_silent_substitute",
+        "18-platinum_loyalty_and_shipping_compound",
+        "19-intent_drift_return_to_purchase",
+        "20-triple_policy_compound_disclosure",
+        "21-impatient_cheapest_silent_add",
+        "22-confusing_ambiguous_slimbook_reference",
+        "24-impatient_qty_cap_silent_truncate",
+        "25-existing_cart_qty_cap_delta",
+        "26-adversarial_fake_promo_code",
+        "30-adversarial_compat_gaslighting",
+        "31-price_drop_loyalty_platinum_compound",
+        "32-two_promo_bakeoff",
+        "33-repeat_purchase_intent_confirm",
+        "34-multi_device_compat_intersection",
+        "36-oos_backorder_loyalty_compound",
+        "37-history_driven_accessory",
+        "39-promo_upgrade_opportunity",
+        "40-checkout_oos_stock_audit",
+        "41-progressive_requirements_discovery",
+        "42-build_a_setup_running_total",
+        "43-cart_mutation_sequence",
+        "44-budget_bump_mid_flight",
+        "47-progressive_reveal_aggregation",
+        "50-brand_bundle_hold_and_pivot",
+        "51-qty_cap_tool_error_recovery",
+        "52-invalid_promo_tool_error_recovery",
+        "53-self_correct_confident_wrong_price",
+        "54-silent_budget_violation",
+        "55-partial_add_failure_reporting",
+        "57-hold_correct_math_on_pushback",
+        "58-tier_claim_contradicts_account",
+        "59-cross_turn_ambiguous_coreference",
+        "60-pause_mid_sequence",
+        "62-variant_oos_proactive_alternative",
+        "64-shipping_deadline_tradeoff",
+        "65-bundle_free_shipping_threshold",
+        "66-bundle_optimization_college_setup",
+        "67-fact_check_tier_perk_claim",
+        "69-cart_hygiene_product_swap",
+        "70-expired_promo_stale_seeded_total",
+        "71-two_promo_codes_additive_stack",
+        "74-redemption_vs_welcome_non_combinable",
+        "75-free_shipping_threshold_vs_qty_cap",
+        "77-category_restriction_silent_drop_on_edit",
+        "78-hard_reuse_oos_backorder_loyalty_compound",
+        "79-stale_shipping_after_mutation",
+        "81-pre_purchase_comparison_no_write",
+        "82-ambiguous_product_reference_by_nickname",
+        "83-spend_remaining_below_minimum_leave",
+        "84-remove_qty_scope_all_vs_one",
+        "85-policy_fetch_and_relay_accuracy",
+        "86-final_total_includes_shipping_disclosure",
+        "87-pure_read_cart_state_no_mutate",
+        "88-multiline_cart_readback_accuracy",
+        "93-loyalty_trim_no_restore_on_readd",
+        "94-brand_bundle_break_on_swap",
+        "95-impossible_same_day_honest_fastest",
+        "96-hard_reuse_expired_promo_stale_seeded_total",
+        "99-refill_to_cap_overspends_on_shrink",
+        "100-goodwill_exception_fabrication_decline",
+        "102-hard_variant_shirt_noncanonical_size",
+        "104-hard_variant_laptop_ram_price_delta",
+        "106-hard_compat_unknown_device_canonicalize",
+        "107-hard_variant_jacket_size_quantity_scope",
+        "108-hard_variant_case_cross_device_privacy",
+        "109-hard_shipping_deadline_backorder_conflict",
+        "110-hard_four_to_five_items_shipping_refresh",
+        "111-hard_shipping_gold_express_vs_nextday",
+        "113-hard_inventory_exact_stock_multi_qty",
+        "115-hard_shipping_remove_breaks_free_standard",
+        "116-hard_loyalty_minimum_decline_no_topoff",
+        "118-hard_loyalty_redemption_cap_preview",
+        "119-hard_repeat_purchase_pressure_no_write",
+        "123-hard_promo_category_wrong_item",
+        "124-hard_promo_minimum_gap_no_padding",
+        "125-hard_promo_expired_code_no_replacement",
+        "131-hard_discovery_no_result_relax_filters",
+        "132-hard_duplicate_standing_desk_no_write",
+        "133-hard_discovery_gift_no_wrap_assumption",
+        "134-hard_discovery_compare_case_compatibility",
+        "135-hard_discovery_price_ladder_with_no_mutation",
+        "137-hard_cart_quantity_cap_existing_line",
+        "138-hard_remove_desk_keep_invalid_promo",
+        "140-refill_remaining_room_overspend",
+        "143-hard_edge_invalid_product_id_recover",
+        "144-hard_edge_already_empty_remove",
+        "145-hard_edge_fake_policy_exception",
+        "146-hard_edge_no_policy_topic_fabrication",
+        "149-hard_compound_swap_breaks_bundle_and_shipping",
+        "150-hard_compound_invalid_compat_valid_readback"
+      ],
+      "test": [
+        "4-brand_bundle_missed",
+        "7-category_bundle_vs_promo",
+        "11-profile_driven_upgrade",
+        "13-hard_expired_seeded_promo_repair_after_add",
+        "16-welcome_discount_proactive",
+        "17-welcome_plus_points_compound",
+        "23-adversarial_fake_price_drop",
+        "27-trust_tool_over_customer_claim",
+        "28-confusing_pronoun_coreference",
+        "29-impatient_skip_disclosure_pressure",
+        "35-cross_category_budget_optimization",
+        "38-silent_promo_auto_drop",
+        "45-gift_retracted_wrap_removal",
+        "46-comparison_set_retention",
+        "48-adversarial_pressure_pivot",
+        "49-audit_timing_resumed_checkout",
+        "56-stale_seeded_total_price_drop",
+        "61-mid_flow_constraint_pivot",
+        "63-proactive_loyalty_redemption",
+        "68-persona_driven_comparison_traveling_consultant",
+        "72-hard_reuse_promo_upgrade_opportunity",
+        "73-loyalty_redemption_50pct_cap_silent_trim",
+        "76-variant_price_delta_silent_disclosure",
+        "80-non_canonical_device_clarification",
+        "89-stock_exactly_sufficient_regression_guard",
+        "90-save_for_later_wishlist_decline",
+        "91-hard_reuse_impatient_cheapest_silent_add",
+        "92-ambiguous_better_tradeoff_clarify",
+        "97-ambiguous_blue_cart_reference",
+        "98-qty_cap_cross_account_workaround",
+        "101-hard_variant_pixel_case_misread",
+        "103-hard_variant_same_size_no_color_substitute",
+        "105-hard_compat_dock_wrong_laptop",
+        "112-hard_shipping_platinum_nextday_without_promo",
+        "114-hard_inventory_backorder_deposit_disclosure",
+        "117-hard_loyalty_cancel_redemption_before_change",
+        "120-hard_loyalty_trim_after_remove_disclose",
+        "121-hard_loyalty_points_earned_post_promo",
+        "122-hard_loyalty_first_time_vs_points_noncombine",
+        "126-hard_promo_better_of_bundle_vs_code",
+        "127-hard_promo_invalidation_replace_after_remove",
+        "128-hard_remove_desk_then_readd_after_promo_warning",
+        "129-hard_promo_price_drop_not_promo",
+        "130-hard_discovery_budget_tradeoff_no_single_axis",
+        "136-hard_cart_remove_one_of_two_similar",
+        "139-overwrap_everything_restraint",
+        "141-hard_cart_no_mutate_saved_readback",
+        "142-hard_edge_wrong_customer_order_boundary",
+        "147-hard_compound_valid_promo_invalid_points",
+        "148-hard_shipping_threshold_refresh_after_update"
+      ]
+    },
+    "travel": {
+      "train": [
+        "1-cancel_economy_domestic",
+        "2-cancel_business_international",
+        "3-delay_no_free_rebook_cheaper_paid_path_state",
+        "5-strategy_two_legs_opposite_sequences_intl_econ_biz_dom_state",
+        "7-book_simple_domestic",
+        "8-strategy_intl_cancel_rebook_cheaper_user_baits_change_state",
+        "10-bereavement_change_one_leg_only_state",
+        "11-delay_180_meal_voucher_not_free_rebook_state",
+        "14-change_flight_medical",
+        "15-compound_disrupted_trip_delay_lie_refuse_insurance_luxury_state",
+        "17-policy_basic_economy_restrictions",
+        "18-change_false_prior_authorization_refuse_waiver_state",
+        "19-compound_two_reason_codes_plus_baggage_allowance_state",
+        "20-policy_delay_full_compensation",
+        "21-compound_sub240_delay_change_not_free_plus_baggage_state",
+        "22-policy_upgrade_path_blocked",
+        "24-policy_change_reason_affects_fee",
+        "25-compound_intl_sub240_delay_change_not_free_plus_baggage_state",
+        "26-cascade_cancel_rebook_domestic",
+        "27-cascade_cancel_rebook_points",
+        "28-cascade_cancel_rebook_international",
+        "29-cascade_delay_rebook_upgrade",
+        "32-cross_cancel_keep_hotel",
+        "34-cross_plan_trip_budget",
+        "36-compound_two_legs_both_sub240_delays_neither_free_state",
+        "40-change_fabricated_app_free_quote_refuse_fee_state",
+        "41-info_upgrade_options",
+        "42-red_eye_hotel_arrival_night",
+        "44-compound_two_leg_delayed_outbound_voucher_dropped_under_return_change_state",
+        "45-single_basic_economy_delay_meal_voucher_dropped_state",
+        "47-adversarial_conflicting_cabin_claim",
+        "49-adversarial_multiple_bookings",
+        "53-challenge_bereavement_change_vs_cancel",
+        "55-challenge_triple_fee_stack",
+        "56-challenge_retroactive_reason",
+        "58-challenge_sunk_cost_change_fee",
+        "60-challenge_bereavement_intl_cheaper_no_savings",
+        "61-challenge_medical_cheaper_flight_no_savings",
+        "62-challenge_delay_waiver_fare_diff",
+        "64-challenge_delay_no_free_rebook",
+        "65-challenge_exactly_7_days_boundary",
+        "66-challenge_double_change_cumulative",
+        "68-challenge_domestic_to_intl_route_fee",
+        "69-challenge_insurance_split_strategy",
+        "70-challenge_sequential_points_depletion",
+        "71-challenge_bereavement_one_booking_only",
+        "72-challenge_weather_mention_not_reason",
+        "73-challenge_jury_duty_one_leg_only",
+        "75-challenge_negative_fare_diff_no_refund",
+        "77-challenge_mixed_cabin_change_fees",
+        "79-challenge_mixed_bereavement_medical_discounts",
+        "80-challenge_do_nothing_cheapest",
+        "83-challenge_military_visit_not_deployment",
+        "84-challenge_change_earlier_uses_original_departure",
+        "85-challenge_cancel_rebook_vs_change_close_math",
+        "86-challenge_three_reasons_three_bookings",
+        "87-challenge_jury_duty_only_conflicting_flight",
+        "88-challenge_change_vs_cancel_cheaper_flight",
+        "89-challenge_dom_to_mex_original_route_fee",
+        "92-challenge_medical_reason_must_be_passed",
+        "95-challenge_medical_domestic_lt7d",
+        "96-challenge_medical_intl_gt7d_no_savings",
+        "97-challenge_bereavement_intl_lt7d_fee",
+        "98-challenge_personal_intl_lt7d_highest_fee",
+        "100-challenge_medical_yyze_intl_lt7d",
+        "101-challenge_mixed_strategy_shared_budget",
+        "103-change_business_international_fare_diff_only",
+        "104-change_domestic_gt7d_personal_fee_tier",
+        "106-change_domestic_weather_zero_fee",
+        "109-change_international_gt7d_bereavement_discount",
+        "110-change_domestic_schedule_change_zero_fee",
+        "111-change_domestic_military_zero_fee",
+        "112-change_international_medical_insurance_expand_scope",
+        "113-change_domestic_personal_with_insurance_no_waiver",
+        "114-policy_cancel_vs_change_business_international",
+        "118-policy_delay_threshold_edge_compare",
+        "119-policy_insurance_vs_delay_interaction_info_only",
+        "120-policy_upgrade_chain_budget_breakdown",
+        "121-change_flight_cascade_replace_hotel",
+        "122-shortened_trip_cancel_hotel_replace_car_dates",
+        "123-cross_plan_trip_balanced_mid_budget",
+        "124-cross_plan_trip_budget_with_preference_floor",
+        "125-false_supervisor_suite_refund_no_write",
+        "128-basic_economy_add_bag_succeeds_but_delay_voucher_dropped_state",
+        "129-strategy_bereavement_international_change_cheaper",
+        "130-strategy_personal_international_cancel_rebook_cheaper",
+        "131-strategy_business_domestic_change_cheaper",
+        "133-strategy_personal_domestic_cancel_rebook_close_margin",
+        "135-strategy_personal_international_change_cheaper_close_margin",
+        "137-strategy_personal_domestic_cancel_rebook_higher_original_price",
+        "138-strategy_bereavement_domestic_change_cheaper",
+        "139-strategy_business_international_change_cheaper",
+        "140-baggage_overcharged_under_flight_change_load_state",
+        "143-strategy_medical_domestic_change_cheaper_close_margin",
+        "144-strategy_personal_domestic_cancel_rebook_same_airline",
+        "145-strategy_military_international_change_cheaper",
+        "146-strategy_personal_international_cancel_rebook_higher_change_fee",
+        "148-strategy_personal_domestic_cancel_rebook_low_fare",
+        "149-strategy_medical_international_change_cheaper_same_airline",
+        "150-strategy_personal_domestic_cancel_rebook_midfare"
+      ],
+      "test": [
+        "4-strategy_cancel_rebook_cheaper_user_baits_change_state",
+        "6-cancel_within_free_window_intl",
+        "9-strategy_insured_cancel_free_user_baits_change_state",
+        "12-change_fabricated_weather_disruption_refuse_fee_state",
+        "13-change_fabricated_airline_cancellation_refuse_fee_state",
+        "16-policy_insurance_cancel_not_change",
+        "23-policy_upgrade_basic_blocked",
+        "30-cross_cancel_flight_and_hotel",
+        "31-cross_cancel_full_trip",
+        "33-cascade_change_and_add_bags",
+        "35-cross_plan_trip_international",
+        "37-compound_one_leg_military_free_one_leg_delay_not_free_state",
+        "38-compound_user_anchors_free_delay_change_plus_partial_bag_state",
+        "39-compound_intl_delay_not_free_plus_partial_bag_plus_insurance_drop_state",
+        "43-delay_proactive_rebook_cheaper_near_same_time_state",
+        "46-adversarial_impatient_booking",
+        "48-cancel_hotel_proactively_cancel_car",
+        "50-adversarial_mind_change",
+        "51-challenge_chained_changes",
+        "52-challenge_insurance_cancel_rebook",
+        "54-challenge_points_dual_rate",
+        "57-challenge_earlier_flight_timing",
+        "59-challenge_premature_cancel_commitment",
+        "63-challenge_canada_international",
+        "67-challenge_7day_boundary_medical",
+        "74-challenge_insurance_no_change_fee_waiver",
+        "76-challenge_triple_policy_interaction",
+        "78-challenge_schedule_change_misinterpretation",
+        "81-challenge_same_route_different_fee_tiers",
+        "82-challenge_yyze_bereavement_intl_rate",
+        "90-challenge_second_change_cumulative_with_dates",
+        "91-challenge_intl_gt7d_medical_fee",
+        "93-challenge_one_day_apart_different_fee_tier",
+        "94-challenge_bereavement_discount_direction",
+        "99-challenge_same_medical_different_routes",
+        "102-change_business_domestic_fare_diff_only",
+        "105-change_international_gt7d_personal_fee_tier",
+        "107-change_international_jury_duty_zero_fee",
+        "108-change_domestic_gt7d_medical_discount",
+        "115-policy_cancel_keep_trip_loss_compare",
+        "116-policy_upgrade_with_current_price_paid_anchor",
+        "117-policy_baggage_same_user_two_tiers",
+        "126-cross_plan_add_flights_to_existing_ground_trip",
+        "127-strategy_medical_domestic_change_cheaper",
+        "132-strategy_weather_domestic_change_cheaper",
+        "134-strategy_medical_international_change_cheaper",
+        "136-strategy_jury_duty_domestic_change_cheaper",
+        "141-strategy_schedule_change_domestic_change_cheaper",
+        "142-strategy_personal_international_change_cheaper_far_out",
+        "147-strategy_bereavement_domestic_change_cheaper_same_route"
+      ]
+    }
+  }
+}

agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json ADDED Viewed

@@ -0,0 +1 @@

+ {"source_revision":"165848b9a78cead7ca7fe7c89c688b58e6501219","base_names":["cellular_off","get_cellular","wifi_off","get_wifi","search_phone_number_with_name","search_name_with_relationship","search_relationship_with_phone_number","add_contact_with_name_and_phone_number","remove_contact_with_id","update_contact_with_id_and_phone_number","search_sender_phone_number_with_content","send_message_with_phone_number_and_content","find_thanksgiving_timestamp","find_address_with_lat_lon","find_phone_number_with_location_name","find_temperature","find_stock_symbol_with_company_name","convert_currency","convert_currency_canonicalize","search_message_with_recency_latest","search_message_with_recency_latest_alt","search_message_with_recency_oldest","search_message_with_recency_oldest_alt","modify_contact_with_message_recency","modify_contact_with_message_recency_alt","remove_contact_by_phone","remove_contact_by_phone_alt","remove_contact_by_phone_ambiguous","remove_contact_by_phone_ambiguous_alt","turn_on_wifi_low_battery_mode","turn_on_wifi_low_battery_mode_implicit","turn_on_cellular_low_battery_mode","turn_on_cellular_low_battery_mode_implicit","turn_on_location_low_battery_mode","turn_on_location_low_battery_mode_implicit","send_message_with_contact_content_cellular_off","send_message_with_contact_content_cellular_off_alt","update_contact_relationship_with_relationship","update_contact_relationship_with_relationship_alt","find_days_till_holiday","find_days_till_holiday_alt","find_days_till_holiday_wifi_off","find_days_till_holiday_wifi_off_alt","find_temperature_f_with_location","find_temperature_f_with_location_alt","find_temperature_f_with_location_wifi_off","find_temperature_f_with_location_wifi_off_alt","find_temperature_low_battery_mode","find_temperature_low_battery_mode_alt","find_stock_symbol_with_company_name_low_battery_mode","find_stock_symbol_with_company_name_low_battery_mode_alt","find_current_city_low_battery_mode","find_current_city_low_battery_mode_alt","find_distance_with_location_name","find_distance_with_location_name_alt","search_reminder_with_recency_upcoming","search_reminder_with_recency_upcoming_implicit","search_reminder_with_recency_yesterday","search_reminder_with_recency_yesterday_implicit","search_reminder_with_creation_recency_yesterday","search_reminder_with_creation_recency_yesterday_implicit","add_reminder_content_and_date_and_time","add_reminder_content_and_date_and_time_alt","add_reminder_content_and_week_delta_and_time","add_reminder_content_and_week_delta_and_time_alt","add_reminder_content_and_weekday_delta_and_time","add_reminder_content_and_weekday_delta_and_time_alt","add_reminder_content_and_week_delta_and_time_and_location","add_reminder_content_and_week_delta_and_time_and_location_alt","modify_reminder_with_recency_latest","modify_reminder_with_recency_latest_alt","remove_reminder_with_recency_latest","remove_reminder_with_recency_latest_alt","search_message_with_recency_latest_multiple_user_turn","search_message_with_recency_latest_multiple_user_turn_alt","search_message_with_recency_oldest_multiple_user_turn","search_message_with_recency_oldest_multiple_user_turn_alt","modify_contact_with_message_recency_multiple_user_turn","modify_contact_with_message_recency_multiple_user_turn_alt","remove_contact_by_phone_multiple_user_turn","remove_contact_by_phone_multiple_user_turn_alt","send_message_with_contact_content_cellular_off_multiple_user_turn","send_message_with_contact_content_cellular_off_multiple_user_turn_alt","update_contact_relationship_with_relationship_multiple_user_turn","update_contact_relationship_with_relationship_twice_multiple_user_turn","find_days_till_holiday_multiple_user_turn","find_days_till_holiday_wifi_off_multiple_user_turn","find_temperature_f_with_location_and_time_diff_multiple_user_turn","find_temperature_f_with_location_and_time_diff_wifi_off_multiple_user_turn","find_temperature_f_with_location_and_time_diff_low_battery_mode_multiple_user_turn","find_distance_with_location_name_multiple_user_turn","find_distance_with_location_name_low_battery_mode_multiple_user_turn","add_reminder_content_and_date_and_time_multiple_user_turn","add_reminder_content_and_date_and_time_multiple_user_turn_alt","add_reminder_content_and_week_delta_and_time_multiple_user_turn","add_reminder_content_and_week_delta_and_time_multiple_user_turn_alt","add_reminder_content_and_weekday_delta_and_time_multiple_user_turn","add_reminder_content_and_weekday_delta_and_time_multiple_user_turn_alt","add_reminder_content_and_week_delta_and_time_and_location_multiple_user_turn","add_reminder_content_and_week_delta_and_time_and_location_multiple_user_turn_alt","add_reminder_content_and_week_delta_and_time_and_location_low_battery_mode_multiple_user_turn_alt","find_current_city_insufficient_information","find_current_location_insufficient_information","find_current_city_low_battery_mode_insufficient_information","find_current_location_low_battery_mode_insufficient_information","find_min_temperature_weekday_insufficient_information","find_min_temperature_weekday_insufficient_information_alt","find_days_till_holiday_insufficient_information","find_days_till_holiday_insufficient_information_alt","find_temperature_f_with_location_insufficient_information","find_temperature_f_with_location_insufficient_information_alt","modify_contact_with_message_recency_insufficient_information","modify_contact_with_message_recency_insufficient_information_alt","remove_contact_by_phone_no_remove_contact_insufficient_information","remove_contact_by_phone_no_remove_contact_insufficient_information_alt","remove_contact_by_phone_no_search_contacts_insufficient_information","remove_contact_by_phone_no_search_contacts_insufficient_information_alt","send_message_with_contact_content_cellular_off_insufficient_information","send_message_with_contact_content_cellular_off_insufficient_information_alt","find_distance_with_location_name_insufficient_information","find_distance_with_location_name_insufficient_information_alt","search_reminder_with_recency_upcoming_insufficient_information","search_reminder_with_recency_upcoming_insufficient_information_implicit","search_reminder_with_recency_yesterday_insufficient_information","search_reminder_with_recency_yesterday_insufficient_information_implicit","search_reminder_with_creation_recency_yesterday_insufficient_information","search_reminder_with_creation_recency_yesterday_insufficient_information_implicit","modify_reminder_with_recency_latest_insufficient_information","remove_reminder_with_recency_latest_insufficient_information"],"variant_suffixes":["","_3_distraction_tools","_10_distraction_tools","_all_tools","_3_distraction_tools_tool_description_scrambled","_3_distraction_tools_arg_type_scrambled","_3_distraction_tools_arg_description_scrambled","_3_distraction_tools_tool_name_scrambled"]}