agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
"""ACE feedback on frozen Tau tool plans, using native tools and reward logic.
|
|
2
|
+
|
|
3
|
+
Initial user messages are existing captures, not generated by this adapter.
|
|
4
|
+
Retail uses the official training split; Banking uses declared disjoint groups.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import copy
|
|
10
|
+
import inspect
|
|
11
|
+
import time
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
|
+
from contextlib import nullcontext
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from urllib.parse import urlsplit
|
|
17
|
+
|
|
18
|
+
from agentinputlab.api import _state
|
|
19
|
+
from agentinputlab.contracts import ExternalDependencyError, ValidationError
|
|
20
|
+
from agentinputlab.contracts.serialization import canonical_json, content_hash, to_jsonable
|
|
21
|
+
|
|
22
|
+
from .ace import ACEExample
|
|
23
|
+
from .ace_native_plan import (
|
|
24
|
+
NativePlanFeedback,
|
|
25
|
+
_save,
|
|
26
|
+
_sha,
|
|
27
|
+
native_process_scope,
|
|
28
|
+
source_file_pins,
|
|
29
|
+
verify_native_checkout,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
SCHEMA = "agentinputlab.ace.Tau.native_feedback_input.v1"
|
|
33
|
+
PLAN_INSTRUCTION = (
|
|
34
|
+
"Solve the supplied task as one noninteractive tool plan. Preserve all visible task "
|
|
35
|
+
"rules and function schemas. Put exactly one JSON object in your final_answer string: "
|
|
36
|
+
'{"actions":[{"name":"visible_function_name","arguments":{}}],"response":"message to user"}. '
|
|
37
|
+
"Use only advertised tools with their required arguments. Actions execute in order without "
|
|
38
|
+
"another model call, access to tool replies while planning, or another user turn. The response "
|
|
39
|
+
"ends this declared plan. Do not output Python, code fences, placeholder variables or extra fields.\n"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _provider(domain):
|
|
44
|
+
if domain == "retail":
|
|
45
|
+
from workers.official_arenas.providers.tau_retail import TauRetailProvider
|
|
46
|
+
|
|
47
|
+
return TauRetailProvider()
|
|
48
|
+
if domain == "banking_knowledge":
|
|
49
|
+
from workers.official_arenas.providers.tau3 import Tau3Provider
|
|
50
|
+
|
|
51
|
+
return Tau3Provider()
|
|
52
|
+
raise ValidationError("ACE Tau supports explicit retail or banking_knowledge domains")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _same_backbone(configuration, target):
|
|
56
|
+
model = target.get("model_id") if isinstance(target, Mapping) else None
|
|
57
|
+
agent = configuration.get("llm_agent")
|
|
58
|
+
if (
|
|
59
|
+
not isinstance(model, str)
|
|
60
|
+
or not model
|
|
61
|
+
or not isinstance(agent, str)
|
|
62
|
+
or agent not in (model, "openai/" + model)
|
|
63
|
+
or configuration.get("llm_user") != agent
|
|
64
|
+
or configuration.get("llm_judge", agent) != agent
|
|
65
|
+
):
|
|
66
|
+
raise ValidationError("ACE Tau user and judge must follow the declared target backbone")
|
|
67
|
+
seed = configuration.get("llm_args_agent", {}).get("seed")
|
|
68
|
+
if type(seed) is not int:
|
|
69
|
+
raise ValidationError("ACE Tau needs an explicit integer evaluated-agent seed")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _native_inputs(checkout, domain, rows, provider):
|
|
73
|
+
from tau2.data_model.message import AssistantMessage, ToolMessage, UserMessage
|
|
74
|
+
from tau2.data_model.tasks import InitialState
|
|
75
|
+
from tau2.runner.build import _build_env_kwargs, build_environment
|
|
76
|
+
from tau2.runner.helpers import get_tasks, load_task_splits
|
|
77
|
+
from tau2.utils.utils import DATA_DIR
|
|
78
|
+
|
|
79
|
+
if Path(DATA_DIR).resolve() != checkout / "data":
|
|
80
|
+
raise ExternalDependencyError("ACE Tau data must belong to the verified native checkout")
|
|
81
|
+
split = load_task_splits(domain)
|
|
82
|
+
if domain == "retail":
|
|
83
|
+
if not isinstance(split, dict) or not {"train", "test"} <= split.keys():
|
|
84
|
+
raise ValidationError("Retail official task split is unavailable")
|
|
85
|
+
train = set(map(str, split["train"]))
|
|
86
|
+
if train & set(map(str, split["test"])):
|
|
87
|
+
raise ValidationError("Retail native training and test splits overlap")
|
|
88
|
+
elif split is not None:
|
|
89
|
+
raise ValidationError("Banking split semantics changed; re-declare the fitting protocol")
|
|
90
|
+
tasks, histories, schemas, kwargs, configs = {}, {}, {}, {}, {}
|
|
91
|
+
kinds = {"assistant": AssistantMessage, "tool": ToolMessage, "user": UserMessage}
|
|
92
|
+
for row in rows:
|
|
93
|
+
key = row["case_id"]
|
|
94
|
+
cfg = provider._validate_config(row["native_configuration"])
|
|
95
|
+
state = _state(row["state"])
|
|
96
|
+
protected = row["protected_checkpoint"]["protected"]
|
|
97
|
+
if (
|
|
98
|
+
cfg["task_set_name"] != domain
|
|
99
|
+
or cfg["task_split_name"] != "train"
|
|
100
|
+
or (domain == "retail" and key not in train)
|
|
101
|
+
or protected["task_id"] != key
|
|
102
|
+
or state.task_id != key
|
|
103
|
+
or protected["locked_dependency_hashes"] != cfg["locked_dependency_hashes"]
|
|
104
|
+
):
|
|
105
|
+
raise ValidationError("Tau task, training origin or dependency binding differs")
|
|
106
|
+
matches = get_tasks(domain, task_split_name=cfg["task_split_name"], task_ids=[key])
|
|
107
|
+
if len(matches) != 1 or str(matches[0].id) != key:
|
|
108
|
+
raise ValidationError("Tau fitting task is missing from the native inventory")
|
|
109
|
+
task = matches[0]
|
|
110
|
+
history = [kinds[m["role"]].model_validate(m) for m in protected["initial_message_history"]]
|
|
111
|
+
if (
|
|
112
|
+
not history
|
|
113
|
+
or not isinstance(history[-1], UserMessage)
|
|
114
|
+
or history[-1].is_tool_call()
|
|
115
|
+
or state.query != history[-1].content
|
|
116
|
+
):
|
|
117
|
+
raise ValidationError("Tau feedback requires its frozen initial visible user message")
|
|
118
|
+
initial = task.initial_state
|
|
119
|
+
task = task.model_copy(
|
|
120
|
+
update=dict(
|
|
121
|
+
initial_state=InitialState(
|
|
122
|
+
initialization_data=initial.initialization_data if initial else None,
|
|
123
|
+
initialization_actions=initial.initialization_actions if initial else None,
|
|
124
|
+
message_history=copy.deepcopy(history),
|
|
125
|
+
)
|
|
126
|
+
),
|
|
127
|
+
deep=True,
|
|
128
|
+
)
|
|
129
|
+
env_args = _build_env_kwargs(
|
|
130
|
+
provider._run_config(cfg, budget=to_jsonable(state.budget)), task
|
|
131
|
+
)
|
|
132
|
+
environment = build_environment(domain, env_kwargs=env_args)
|
|
133
|
+
tool_schema = [to_jsonable(t.openai_schema) for t in environment.get_tools()]
|
|
134
|
+
names = {t.get("name") or t.get("function", {}).get("name") for t in tool_schema}
|
|
135
|
+
if (
|
|
136
|
+
not names
|
|
137
|
+
or names != set(state.tool_names)
|
|
138
|
+
or state.instruction != environment.get_policy()
|
|
139
|
+
):
|
|
140
|
+
raise ValidationError("Tau visible native policy or tool inventory differs")
|
|
141
|
+
catalogs = [c for c in state.context if c.block_id == "official_tool_catalog"]
|
|
142
|
+
if len(catalogs) != 1:
|
|
143
|
+
raise ValidationError("Tau frozen visible state needs its native tool catalog")
|
|
144
|
+
import json
|
|
145
|
+
|
|
146
|
+
if to_jsonable(json.loads(catalogs[0].text)) != tool_schema:
|
|
147
|
+
raise ValidationError("Tau frozen function schemas differ from the native environment")
|
|
148
|
+
tasks[key], histories[key], schemas[key] = task, history, tool_schema
|
|
149
|
+
configs[key], kwargs[key] = cfg, env_args
|
|
150
|
+
files = list((checkout / "src/tau2").rglob("*.py"))
|
|
151
|
+
data = checkout / "data/tau2/domains" / domain
|
|
152
|
+
if domain == "retail":
|
|
153
|
+
files.extend(
|
|
154
|
+
data / name for name in ("db.json", "policy.md", "tasks.json", "split_tasks.json")
|
|
155
|
+
)
|
|
156
|
+
else:
|
|
157
|
+
files.append(data / "db.json")
|
|
158
|
+
files.extend(p for p in (data / "documents").rglob("*") if p.is_file())
|
|
159
|
+
files.extend(data / "tasks" / (key + ".json") for key in tasks)
|
|
160
|
+
return tasks, histories, schemas, kwargs, configs, source_file_pins(checkout, files)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def build_ace_tau_feedback_input(
|
|
164
|
+
*,
|
|
165
|
+
checkout,
|
|
166
|
+
domain: str,
|
|
167
|
+
captures: Sequence[Mapping],
|
|
168
|
+
native_configurations: Mapping,
|
|
169
|
+
train_ids: Sequence[str],
|
|
170
|
+
validation_ids: Sequence[str],
|
|
171
|
+
heldout_ids: Sequence[str],
|
|
172
|
+
partition_label: str,
|
|
173
|
+
target_model: Mapping,
|
|
174
|
+
output_path,
|
|
175
|
+
):
|
|
176
|
+
"""Prepare existing captures, without user/agent/judge model calls."""
|
|
177
|
+
provider = _provider(domain)
|
|
178
|
+
root = verify_native_checkout(checkout, revision=provider.source_revision)
|
|
179
|
+
ids = [*train_ids, *validation_ids]
|
|
180
|
+
if (
|
|
181
|
+
not train_ids
|
|
182
|
+
or not validation_ids
|
|
183
|
+
or any(not isinstance(k, str) or not k for k in ids)
|
|
184
|
+
or len(ids) != len(set(ids))
|
|
185
|
+
or set(ids) & set(heldout_ids)
|
|
186
|
+
or not isinstance(partition_label, str)
|
|
187
|
+
or not partition_label.strip()
|
|
188
|
+
or not isinstance(native_configurations, Mapping)
|
|
189
|
+
or set(ids) != set(native_configurations)
|
|
190
|
+
):
|
|
191
|
+
raise ValidationError(
|
|
192
|
+
"Tau needs disjoint fitting groups, per-case configurations and a label"
|
|
193
|
+
)
|
|
194
|
+
selected = [c for c in captures if c["checkpoint_id"] in ids]
|
|
195
|
+
if len(selected) != len(ids) or {c["checkpoint_id"] for c in selected} != set(ids):
|
|
196
|
+
raise ValidationError("Tau capture IDs are missing or duplicated")
|
|
197
|
+
by_id = {c["checkpoint_id"]: to_jsonable(c) for c in selected}
|
|
198
|
+
rows = []
|
|
199
|
+
for role, group in (("train", train_ids), ("validation", validation_ids)):
|
|
200
|
+
for key in group:
|
|
201
|
+
cfg = to_jsonable(native_configurations[key])
|
|
202
|
+
_same_backbone(cfg, target_model)
|
|
203
|
+
state = _state(by_id[key]["visible_state"])
|
|
204
|
+
rows.append(
|
|
205
|
+
dict(
|
|
206
|
+
case_id=key,
|
|
207
|
+
role=role,
|
|
208
|
+
state=to_jsonable(state),
|
|
209
|
+
state_hash=state.state_hash,
|
|
210
|
+
protected_checkpoint=by_id[key],
|
|
211
|
+
native_configuration=cfg,
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
with native_process_scope(root, package="tau2"):
|
|
215
|
+
_, _, schemas, _, _, pins = _native_inputs(root, domain, rows, provider)
|
|
216
|
+
value = dict(
|
|
217
|
+
schema=SCHEMA,
|
|
218
|
+
benchmark=provider.benchmark_id,
|
|
219
|
+
domain=domain,
|
|
220
|
+
native_source_revision=provider.source_revision,
|
|
221
|
+
source_files=pins,
|
|
222
|
+
fitting_rows=rows,
|
|
223
|
+
heldout_ids=list(heldout_ids),
|
|
224
|
+
source_official_split="train" if domain == "retail" else None,
|
|
225
|
+
partition_source=partition_label,
|
|
226
|
+
target_model=to_jsonable(target_model),
|
|
227
|
+
environment_model=to_jsonable(target_model),
|
|
228
|
+
native_function_schemas=schemas,
|
|
229
|
+
initial_user_captures_reused_NOT_regenerated=True,
|
|
230
|
+
builder_model_calls=0,
|
|
231
|
+
ToolTalk_not_used=True,
|
|
232
|
+
)
|
|
233
|
+
path = Path(output_path).expanduser().absolute()
|
|
234
|
+
if path.resolve().is_relative_to(root) or path.exists() or path.is_symlink():
|
|
235
|
+
raise ValidationError("Tau input must be a new file outside the clean native checkout")
|
|
236
|
+
_save(path, value)
|
|
237
|
+
return dict(path=str(path), sha256=_sha(path))
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class ACETauFeedback(NativePlanFeedback):
|
|
241
|
+
"""Native Tau reward for a frozen, explicitly noninteractive tool plan."""
|
|
242
|
+
|
|
243
|
+
processor_id = "agentinputlab.ace.Tau.native_tool_plan_feedback.v1"
|
|
244
|
+
metric = "official_reward"
|
|
245
|
+
|
|
246
|
+
def __init__(
|
|
247
|
+
self, *, checkout, source_input, output_root, mode="offline", environment_base_url=None
|
|
248
|
+
):
|
|
249
|
+
self._initialize(
|
|
250
|
+
checkout=checkout, source_input=source_input, output_root=output_root, mode=mode
|
|
251
|
+
)
|
|
252
|
+
self.domain = self.source.get("domain")
|
|
253
|
+
self.provider = _provider(self.domain)
|
|
254
|
+
if (
|
|
255
|
+
self.source.get("schema") != SCHEMA
|
|
256
|
+
or self.source.get("benchmark") != self.provider.benchmark_id
|
|
257
|
+
):
|
|
258
|
+
raise ValidationError("Tau feedback schema or benchmark differs")
|
|
259
|
+
self.checkout = verify_native_checkout(checkout, revision=self.provider.source_revision)
|
|
260
|
+
self._verify_source()
|
|
261
|
+
expected_split = "train" if self.domain == "retail" else None
|
|
262
|
+
if self.source["source_official_split"] != expected_split or not self.source.get(
|
|
263
|
+
"partition_source"
|
|
264
|
+
):
|
|
265
|
+
raise ValidationError("Tau official/caller-defined split provenance differs")
|
|
266
|
+
for row in self.rows.values():
|
|
267
|
+
_same_backbone(row["native_configuration"], self.source["target_model"])
|
|
268
|
+
if self.source["target_model"] != self.source["environment_model"]:
|
|
269
|
+
raise ValidationError("Tau environment and target backbone bindings differ")
|
|
270
|
+
with native_process_scope(self.checkout, package="tau2"):
|
|
271
|
+
self.tasks, self.histories, self.schemas, self.env_kwargs, self.configs, pins = (
|
|
272
|
+
_native_inputs(self.checkout, self.domain, list(self.rows.values()), self.provider)
|
|
273
|
+
)
|
|
274
|
+
if (
|
|
275
|
+
pins != self.source["source_files"]
|
|
276
|
+
or self.schemas != self.source["native_function_schemas"]
|
|
277
|
+
):
|
|
278
|
+
raise ValidationError("Tau native source or visible function schemas changed")
|
|
279
|
+
self.environment_base_url = environment_base_url
|
|
280
|
+
if environment_base_url is not None:
|
|
281
|
+
url = urlsplit(environment_base_url) if isinstance(environment_base_url, str) else None
|
|
282
|
+
if (
|
|
283
|
+
not url
|
|
284
|
+
or url.scheme not in ("http", "https")
|
|
285
|
+
or not url.hostname
|
|
286
|
+
or url.username
|
|
287
|
+
or url.password
|
|
288
|
+
):
|
|
289
|
+
raise ValidationError(
|
|
290
|
+
"Tau environment endpoint must be an explicit credential-free HTTP URL"
|
|
291
|
+
)
|
|
292
|
+
self.tool_names = {k: set(s.tool_names) for k, s in self.states.items()}
|
|
293
|
+
self._create_output()
|
|
294
|
+
|
|
295
|
+
def fingerprint(self):
|
|
296
|
+
self._verify_source()
|
|
297
|
+
return dict(
|
|
298
|
+
processor_id=self.processor_id,
|
|
299
|
+
mode=self.mode,
|
|
300
|
+
source_input=self.source_input,
|
|
301
|
+
source_files=self.source["source_files"],
|
|
302
|
+
source_revision=self.provider.source_revision,
|
|
303
|
+
scorer_id=self.provider.scorer_id,
|
|
304
|
+
scorer_revision=self.provider.scorer_revision,
|
|
305
|
+
provider_source_sha256=_sha(inspect.getfile(type(self.provider))),
|
|
306
|
+
task_adapter_source_sha256=_sha(__file__),
|
|
307
|
+
target_model=self.source["target_model"],
|
|
308
|
+
environment_model=self.source["environment_model"],
|
|
309
|
+
environment_base_url=self.environment_base_url,
|
|
310
|
+
source_official_split=self.source["source_official_split"],
|
|
311
|
+
partition_source=self.source["partition_source"],
|
|
312
|
+
groups={
|
|
313
|
+
role: [
|
|
314
|
+
dict(case_id=k, state_hash=r["state_hash"])
|
|
315
|
+
for k, r in self.rows.items()
|
|
316
|
+
if r["role"] == role
|
|
317
|
+
]
|
|
318
|
+
for role in ("train", "validation")
|
|
319
|
+
},
|
|
320
|
+
visible_tool_schemas_sha256={k: content_hash(v) for k, v in self.schemas.items()},
|
|
321
|
+
feedback_metric=self.metric,
|
|
322
|
+
initial_user_captures_reused_NOT_regenerated=True,
|
|
323
|
+
noninteractive_plan_NOT_native_closed_loop_agent=True,
|
|
324
|
+
heldout_feedback_NOT_used=True,
|
|
325
|
+
transductive_NOT_independent_heldout=self.mode == "online",
|
|
326
|
+
ToolTalk_NOT_used=True,
|
|
327
|
+
native_user_simulator_calls=0,
|
|
328
|
+
native_judge_controls_and_costs_separate=True,
|
|
329
|
+
automatic_retry=False,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
def _example(self, key, split):
|
|
333
|
+
state = self.states[key]
|
|
334
|
+
visible = dict(
|
|
335
|
+
query=state.query,
|
|
336
|
+
instruction=state.instruction,
|
|
337
|
+
messages=state.messages,
|
|
338
|
+
context=state.context,
|
|
339
|
+
tool_names=state.tool_names,
|
|
340
|
+
native_function_tools=self.schemas[key],
|
|
341
|
+
)
|
|
342
|
+
return ACEExample(
|
|
343
|
+
example_id=key,
|
|
344
|
+
input=dict(question=PLAN_INSTRUCTION + canonical_json(visible)),
|
|
345
|
+
expected_output=key,
|
|
346
|
+
split=split,
|
|
347
|
+
metadata=dict(
|
|
348
|
+
source_state_hash=state.state_hash,
|
|
349
|
+
task_processor=self.processor_id,
|
|
350
|
+
evaluator_target_is_opaque_case_id=True,
|
|
351
|
+
),
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
def _native_score(self, key, plan, folder):
|
|
355
|
+
with native_process_scope(self.checkout, package="tau2"):
|
|
356
|
+
from tau2.data_model.message import AssistantMessage, ToolCall
|
|
357
|
+
from tau2.data_model.simulation import SimulationRun, TerminationReason
|
|
358
|
+
from tau2.evaluator import evaluator_nl_assertions
|
|
359
|
+
from tau2.evaluator.evaluator import EvaluationType, evaluate_simulation
|
|
360
|
+
from tau2.runner.build import build_environment
|
|
361
|
+
from tau2.utils import llm_utils
|
|
362
|
+
|
|
363
|
+
from workers.official_arenas.providers import tau3, tau_retail
|
|
364
|
+
|
|
365
|
+
started, start_time = time.monotonic(), datetime.now(timezone.utc).isoformat()
|
|
366
|
+
task, cfg = self.tasks[key], copy.deepcopy(self.configs[key])
|
|
367
|
+
history = copy.deepcopy(self.histories[key])
|
|
368
|
+
if self.environment_base_url is not None:
|
|
369
|
+
for role in ("agent", "user", "judge"):
|
|
370
|
+
if "llm_args_" + role in cfg:
|
|
371
|
+
cfg["llm_args_" + role]["api_base"] = self.environment_base_url
|
|
372
|
+
original_cost = llm_utils.get_response_cost
|
|
373
|
+
original_judge = (
|
|
374
|
+
evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS,
|
|
375
|
+
evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS_ARGS,
|
|
376
|
+
)
|
|
377
|
+
try:
|
|
378
|
+
tau3._install_local_tau_cost_accounting(cfg)
|
|
379
|
+
if self.domain == "banking_knowledge":
|
|
380
|
+
tau3._install_local_tau_nl_evaluator(cfg)
|
|
381
|
+
environment = build_environment(self.domain, env_kwargs=self.env_kwargs[key])
|
|
382
|
+
initial = task.initial_state
|
|
383
|
+
environment.set_state(
|
|
384
|
+
initial.initialization_data,
|
|
385
|
+
initial.initialization_actions,
|
|
386
|
+
copy.deepcopy(history),
|
|
387
|
+
strict=True,
|
|
388
|
+
)
|
|
389
|
+
for index, action in enumerate(plan["actions"]):
|
|
390
|
+
call = ToolCall(
|
|
391
|
+
id="ace_plan_" + str(index),
|
|
392
|
+
name=action["name"],
|
|
393
|
+
arguments=action["arguments"],
|
|
394
|
+
requestor="assistant",
|
|
395
|
+
)
|
|
396
|
+
history.append(AssistantMessage.text("", tool_calls=[call]))
|
|
397
|
+
history.append(environment.get_response(call))
|
|
398
|
+
history.append(AssistantMessage.text(plan["response"]))
|
|
399
|
+
simulation = SimulationRun(
|
|
400
|
+
id="ace-fitting-" + key,
|
|
401
|
+
task_id=key,
|
|
402
|
+
start_time=start_time,
|
|
403
|
+
end_time=datetime.now(timezone.utc).isoformat(),
|
|
404
|
+
duration=time.monotonic() - started,
|
|
405
|
+
termination_reason=TerminationReason.AGENT_STOP,
|
|
406
|
+
seed=cfg["llm_args_agent"]["seed"],
|
|
407
|
+
messages=history,
|
|
408
|
+
agent_cost=0.0,
|
|
409
|
+
user_cost=0.0,
|
|
410
|
+
)
|
|
411
|
+
_save(
|
|
412
|
+
folder / "before-native-scorer.json",
|
|
413
|
+
dict(
|
|
414
|
+
case_id=key,
|
|
415
|
+
plan=plan,
|
|
416
|
+
messages=[m.model_dump(mode="json") for m in history],
|
|
417
|
+
source_state_hash=self.states[key].state_hash,
|
|
418
|
+
native_noninteractive_plan_NOT_interactive_rollout=True,
|
|
419
|
+
evaluated_agent_model_calls=0,
|
|
420
|
+
native_user_simulator_calls=0,
|
|
421
|
+
),
|
|
422
|
+
)
|
|
423
|
+
context = (
|
|
424
|
+
tau_retail.bound_retail_judge(
|
|
425
|
+
cfg, list(task.evaluation_criteria.nl_assertions or [])
|
|
426
|
+
)
|
|
427
|
+
if self.domain == "retail"
|
|
428
|
+
else nullcontext()
|
|
429
|
+
)
|
|
430
|
+
with context:
|
|
431
|
+
reward = evaluate_simulation(
|
|
432
|
+
simulation,
|
|
433
|
+
task,
|
|
434
|
+
evaluation_type=EvaluationType.ALL,
|
|
435
|
+
solo_mode=False,
|
|
436
|
+
domain=self.domain,
|
|
437
|
+
env_kwargs=self.env_kwargs[key],
|
|
438
|
+
strict_replay=True,
|
|
439
|
+
)
|
|
440
|
+
finally:
|
|
441
|
+
llm_utils.get_response_cost = original_cost
|
|
442
|
+
(
|
|
443
|
+
evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS,
|
|
444
|
+
evaluator_nl_assertions.DEFAULT_LLM_NL_ASSERTIONS_ARGS,
|
|
445
|
+
) = original_judge
|
|
446
|
+
result = dict(
|
|
447
|
+
status="native_Tau_noninteractive_tool_plan_scored",
|
|
448
|
+
case_id=key,
|
|
449
|
+
native_metrics=dict(official_reward=float(reward.reward)),
|
|
450
|
+
native_reward_info=reward.model_dump(mode="json"),
|
|
451
|
+
ordered_action_count=len(plan["actions"]),
|
|
452
|
+
source_state_hash=self.states[key].state_hash,
|
|
453
|
+
evaluated_agent_model_calls=0,
|
|
454
|
+
native_user_simulator_calls=0,
|
|
455
|
+
noninteractive_plan_agent_stop_explicit_NOT_native_interactive_success_claim=True,
|
|
456
|
+
)
|
|
457
|
+
_save(folder / "native-score.json", result)
|
|
458
|
+
return result
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def create_tau_feedback(**configuration):
|
|
462
|
+
"""Flat-JSON startup factory for the isolated native ACE worker."""
|
|
463
|
+
return ACETauFeedback(**configuration)
|