agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"question": "Fixture entity relation?",
|
|
3
|
+
"generator_outputs": [
|
|
4
|
+
{
|
|
5
|
+
"text": "Evidence points to the second fixture entity.",
|
|
6
|
+
"input_tokens": 12,
|
|
7
|
+
"output_tokens": 8
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"text": "Therefore, the answer is fixture-answer.",
|
|
11
|
+
"input_tokens": 18,
|
|
12
|
+
"output_tokens": 7
|
|
13
|
+
}
|
|
14
|
+
],
|
|
15
|
+
"retrieval_rounds": [
|
|
16
|
+
{
|
|
17
|
+
"query": "Fixture entity relation?",
|
|
18
|
+
"documents": [
|
|
19
|
+
{
|
|
20
|
+
"id": "fixture-d1",
|
|
21
|
+
"title": "Fixture One",
|
|
22
|
+
"paragraph_text": "The first fixture fact points toward another entity.",
|
|
23
|
+
"score": 0.9
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"query": "Evidence points to the second fixture entity.",
|
|
29
|
+
"documents": [
|
|
30
|
+
{
|
|
31
|
+
"id": "fixture-d2",
|
|
32
|
+
"title": "Fixture Two",
|
|
33
|
+
"paragraph_text": "The second fixture fact establishes fixture-answer.",
|
|
34
|
+
"score": 0.8
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
}
|
|
38
|
+
]
|
|
39
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"start_state": "step_by_step_bm25_retriever",
|
|
3
|
+
"end_state": "[EOQ]",
|
|
4
|
+
"models": {
|
|
5
|
+
"step_by_step_bm25_retriever": {
|
|
6
|
+
"name": "retrieve_and_reset_paragraphs",
|
|
7
|
+
"next_model": "step_by_step_cot_reasoning_gen",
|
|
8
|
+
"retrieval_type": "bm25",
|
|
9
|
+
"retriever_host": "http://fixture.invalid",
|
|
10
|
+
"retriever_port": "0",
|
|
11
|
+
"retrieval_count": 2,
|
|
12
|
+
"global_max_num_paras": 8,
|
|
13
|
+
"query_source": "question_or_last_generated_sentence",
|
|
14
|
+
"source_corpus_name": "fixture-corpus-v1",
|
|
15
|
+
"document_type": "title_paragraph_text",
|
|
16
|
+
"return_pids": false,
|
|
17
|
+
"cumulate_titles": true,
|
|
18
|
+
"end_state": "[EOQ]"
|
|
19
|
+
},
|
|
20
|
+
"step_by_step_cot_reasoning_gen": {
|
|
21
|
+
"name": "step_by_step_cot_gen",
|
|
22
|
+
"next_model": "step_by_step_exit_controller",
|
|
23
|
+
"prompt_file": "__AGENTINPUTLAB_LOCKED_PROMPT__",
|
|
24
|
+
"prompt_reader_args": {},
|
|
25
|
+
"generation_type": "sentences",
|
|
26
|
+
"reset_queries_as_sentences": false,
|
|
27
|
+
"add_context": true,
|
|
28
|
+
"shuffle_paras": false,
|
|
29
|
+
"terminal_return_type": null,
|
|
30
|
+
"disable_exit": true,
|
|
31
|
+
"end_state": "[EOQ]",
|
|
32
|
+
"gen_model": "gpt3",
|
|
33
|
+
"engine": "code-davinci-002",
|
|
34
|
+
"max_tokens": 64
|
|
35
|
+
},
|
|
36
|
+
"step_by_step_exit_controller": {
|
|
37
|
+
"name": "step_by_step_exit_controller",
|
|
38
|
+
"next_model": "step_by_step_bm25_retriever",
|
|
39
|
+
"answer_extractor_regex": ".* answer is:? (.*)\\.?",
|
|
40
|
+
"answer_extractor_remove_last_fullstop": true,
|
|
41
|
+
"terminal_state_next_model": null,
|
|
42
|
+
"terminal_return_type": "pids",
|
|
43
|
+
"global_max_num_paras": 8,
|
|
44
|
+
"end_state": "[EOQ]"
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
"prediction_type": "pids"
|
|
48
|
+
}
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
"""Complete pinned IRCoT-QA graph with explicitly substituted model and index.
|
|
2
|
+
|
|
3
|
+
No source is copied here: the native builder, all six participants, prompt
|
|
4
|
+
reader, best-first loop and prediction extractor execute from the mount.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
import copy
|
|
11
|
+
import hashlib
|
|
12
|
+
import importlib
|
|
13
|
+
import os
|
|
14
|
+
from dataclasses import replace
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from agentinputlab.backends import GenerationRequest
|
|
18
|
+
from agentinputlab.contracts.serialization import content_hash, to_jsonable
|
|
19
|
+
|
|
20
|
+
from .controlled import _NATIVE_LOCK, PinnedIRCoTControlledRuntime, _LiveBindings
|
|
21
|
+
from .driver import (
|
|
22
|
+
DriverError,
|
|
23
|
+
IRCoTDriver,
|
|
24
|
+
_BoundedCapture,
|
|
25
|
+
_load_config,
|
|
26
|
+
_parse_request,
|
|
27
|
+
_project_trajectory,
|
|
28
|
+
_RetrievalCapture,
|
|
29
|
+
_safe_file,
|
|
30
|
+
_TracingController,
|
|
31
|
+
_upstream_import_scope,
|
|
32
|
+
_working_directory,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
QA_PROFILE = "upstream.hotpotqa.codex_qa.v1"
|
|
36
|
+
QA_BINDING_MODE = "agentinputlab.ircot.native_like_qa.v1"
|
|
37
|
+
QA_MANIFEST = Path(__file__).with_name("qa_compatibility_manifest.json")
|
|
38
|
+
QA_ROLES = {
|
|
39
|
+
"retrieve_and_reset_paragraphs",
|
|
40
|
+
"step_by_step_cot_gen",
|
|
41
|
+
"step_by_step_exit_controller",
|
|
42
|
+
"copy_question",
|
|
43
|
+
"llmqa",
|
|
44
|
+
"answer_extractor",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def patch_native_qa_config(config, checkout, controls):
|
|
49
|
+
"""Change bounded limits only; all graph transitions/parsers stay native."""
|
|
50
|
+
value = copy.deepcopy(dict(config))
|
|
51
|
+
models = value.get("models")
|
|
52
|
+
if (
|
|
53
|
+
not isinstance(models, dict)
|
|
54
|
+
or {v.get("name") for v in models.values()} != QA_ROLES
|
|
55
|
+
or len(models) != 6
|
|
56
|
+
):
|
|
57
|
+
raise DriverError(
|
|
58
|
+
"CONFIG_INVALID", "native IRCoT-QA requires all six original participants"
|
|
59
|
+
)
|
|
60
|
+
for model in models.values():
|
|
61
|
+
role = model["name"]
|
|
62
|
+
if role == "retrieve_and_reset_paragraphs":
|
|
63
|
+
model["retrieval_count"] = controls.per_step_top_k
|
|
64
|
+
model["global_max_num_paras"] = controls.max_total_documents
|
|
65
|
+
model["cumulate_titles"] = controls.reset_policy != "replace_each_step"
|
|
66
|
+
elif role in ("step_by_step_cot_gen", "step_by_step_exit_controller"):
|
|
67
|
+
model["max_num_sentences"] = controls.max_steps
|
|
68
|
+
if role == "step_by_step_exit_controller":
|
|
69
|
+
model["global_max_num_paras"] = controls.max_total_documents
|
|
70
|
+
if role in ("step_by_step_cot_gen", "llmqa"):
|
|
71
|
+
if _safe_file(checkout.root, model.get("prompt_file")) != checkout.prompt_path:
|
|
72
|
+
raise DriverError("PROMPT_MISMATCH", "native IRCoT-QA uses an unpinned prompt")
|
|
73
|
+
if model.get("gen_model") != "gpt3":
|
|
74
|
+
raise DriverError("CONFIG_INVALID", "native IRCoT-QA generator class changed")
|
|
75
|
+
if value.get("prediction_type") != "answer":
|
|
76
|
+
raise DriverError("CONFIG_INVALID", "native IRCoT-QA must use answer prediction")
|
|
77
|
+
return value
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class PinnedIRCoTNativeQARuntime(PinnedIRCoTControlledRuntime):
|
|
81
|
+
"""Dedicated-process complete QA runtime; inherited startup checks unchanged."""
|
|
82
|
+
|
|
83
|
+
worker_id = "agentinputlab.ircot.native_like_full_qa.v1"
|
|
84
|
+
|
|
85
|
+
def _checkout(self):
|
|
86
|
+
if Path(os.path.expanduser("~")).resolve() != self.cache_home:
|
|
87
|
+
raise DriverError("CACHE_SCOPE_CHANGED", "dedicated worker HOME changed")
|
|
88
|
+
checkout = IRCoTDriver(
|
|
89
|
+
manifest_path=QA_MANIFEST,
|
|
90
|
+
profile_id=QA_PROFILE,
|
|
91
|
+
environment={"AGENTINPUTLAB_UPSTREAM_ROOT": str(self.source_root)},
|
|
92
|
+
)._checkout()
|
|
93
|
+
return replace(
|
|
94
|
+
checkout,
|
|
95
|
+
profile_id="native_like.hotpotqa.full_qa.v1",
|
|
96
|
+
profile={
|
|
97
|
+
**checkout.profile,
|
|
98
|
+
"identity_kind": "native_like_substituted_qa",
|
|
99
|
+
"corpus_revision": self.corpus_revision,
|
|
100
|
+
"index_revision": self.index_revision,
|
|
101
|
+
},
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
def fingerprint(self):
|
|
105
|
+
value = dict(super().fingerprint())
|
|
106
|
+
value.update(
|
|
107
|
+
binding_mode=QA_BINDING_MODE,
|
|
108
|
+
profile_id="native_like.hotpotqa.full_qa.v1",
|
|
109
|
+
identity_kind="native_like_substituted_qa",
|
|
110
|
+
source_profile=QA_PROFILE,
|
|
111
|
+
runtime_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
|
112
|
+
controlled_binding_helpers_sha256=hashlib.sha256(
|
|
113
|
+
Path(__file__).with_name("controlled.py").read_bytes()
|
|
114
|
+
).hexdigest(),
|
|
115
|
+
full_six_participant_QA_graph=True,
|
|
116
|
+
native_final_prediction_extraction=True,
|
|
117
|
+
shared_final_call=True,
|
|
118
|
+
native_generator_default_output_cap=300,
|
|
119
|
+
caller_backend_sampling_is_explicit=True,
|
|
120
|
+
published_score_parity=False,
|
|
121
|
+
)
|
|
122
|
+
return value
|
|
123
|
+
|
|
124
|
+
def run(self, request):
|
|
125
|
+
from agentinputlab.algorithms.acquire.ircot_native import IRCoTNativeQARequest
|
|
126
|
+
|
|
127
|
+
if not isinstance(request, IRCoTNativeQARequest):
|
|
128
|
+
raise TypeError("native IRCoT-QA requires its typed request")
|
|
129
|
+
if not _NATIVE_LOCK.acquire(blocking=False):
|
|
130
|
+
raise RuntimeError("one native IRCoT call per dedicated worker process")
|
|
131
|
+
self.last_events, self.last_usage = [], {}
|
|
132
|
+
bindings = None
|
|
133
|
+
try:
|
|
134
|
+
identity = self.fingerprint()
|
|
135
|
+
legacy = _parse_request(request.legacy.as_dict())
|
|
136
|
+
if (
|
|
137
|
+
content_hash(identity) != request.expected_runtime_fingerprint_sha256
|
|
138
|
+
or request.seed != self.seed
|
|
139
|
+
or request.max_model_calls > self.max_model_calls
|
|
140
|
+
or request.per_call_output_tokens != self.per_call_output_tokens
|
|
141
|
+
or legacy.controls.max_steps + 1 > request.max_model_calls
|
|
142
|
+
or legacy.controls.max_output_tokens > self.max_output_tokens
|
|
143
|
+
):
|
|
144
|
+
raise DriverError(
|
|
145
|
+
"BINDING_MISMATCH", "native IRCoT-QA source/model/budget identity differs"
|
|
146
|
+
)
|
|
147
|
+
if (
|
|
148
|
+
to_jsonable(legacy.generator_fingerprint) != self.generator_fingerprint
|
|
149
|
+
or to_jsonable(self.backend.fingerprint()) != self.generator_fingerprint
|
|
150
|
+
or to_jsonable(legacy.retriever_fingerprint) != self.retriever_fingerprint
|
|
151
|
+
or to_jsonable(self.retriever.fingerprint()) != self.retriever_fingerprint
|
|
152
|
+
or legacy.corpus_revision != self.corpus_revision
|
|
153
|
+
or legacy.index_revision != self.index_revision
|
|
154
|
+
):
|
|
155
|
+
raise DriverError("BINDING_MISMATCH", "native IRCoT-QA context bindings differ")
|
|
156
|
+
checkout = self._checkout()
|
|
157
|
+
if legacy.prompt_hash != checkout.profile["prompt_sha256"]:
|
|
158
|
+
raise DriverError("PROMPT_MISMATCH", "native IRCoT-QA prompt changed")
|
|
159
|
+
bindings = _QABindings(self, legacy, request)
|
|
160
|
+
config = patch_native_qa_config(
|
|
161
|
+
_load_config(
|
|
162
|
+
checkout,
|
|
163
|
+
{
|
|
164
|
+
"AGENTINPUTLAB_IRCOT_RETRIEVER_HOST": "http://127.0.0.1",
|
|
165
|
+
"AGENTINPUTLAB_IRCOT_RETRIEVER_PORT": "0",
|
|
166
|
+
},
|
|
167
|
+
),
|
|
168
|
+
checkout,
|
|
169
|
+
legacy.controls,
|
|
170
|
+
)
|
|
171
|
+
for model in config["models"].values():
|
|
172
|
+
if model["name"] == "retrieve_and_reset_paragraphs":
|
|
173
|
+
model["source_corpus_name"] = self.corpus_revision
|
|
174
|
+
prompt_arguments = model.get("prompt_reader_args", {})
|
|
175
|
+
demonstration_ids = prompt_arguments.get("filter_by_key_values", {}).get("qid", [])
|
|
176
|
+
if legacy.task_id in demonstration_ids:
|
|
177
|
+
raise DriverError(
|
|
178
|
+
"EVALUATION_ID_IS_DEMONSTRATION",
|
|
179
|
+
"native IRCoT-QA evaluation ID appears among the pinned prompt demonstrations",
|
|
180
|
+
)
|
|
181
|
+
capture = _BoundedCapture()
|
|
182
|
+
with (
|
|
183
|
+
_working_directory(checkout.root),
|
|
184
|
+
_upstream_import_scope(checkout.root, fixture_shims=False),
|
|
185
|
+
contextlib.redirect_stdout(capture),
|
|
186
|
+
):
|
|
187
|
+
configurable = importlib.import_module("commaqa.inference.configurable_inference")
|
|
188
|
+
native = importlib.import_module("commaqa.inference.ircot")
|
|
189
|
+
qa_model = importlib.import_module("commaqa.execution.llm_qa_model")
|
|
190
|
+
prompts = importlib.import_module("commaqa.inference.prompt_reader")
|
|
191
|
+
prompts.get_tokenizer = bindings.get_tokenizer
|
|
192
|
+
bindings.fit_prompt = prompts.fit_prompt_into_given_limit
|
|
193
|
+
native.GPT3Generator = bindings.generator_class("reasoning")
|
|
194
|
+
qa_model.GPT3Generator = bindings.generator_class("final_answer")
|
|
195
|
+
retrieval = _RetrievalCapture(bindings.retrieve)
|
|
196
|
+
native.safe_post_request = retrieval
|
|
197
|
+
decomposer, model_map = configurable.build_decomposer_and_models(
|
|
198
|
+
copy.deepcopy(config)
|
|
199
|
+
)
|
|
200
|
+
tracing = _TracingController(
|
|
201
|
+
decomposer.controller,
|
|
202
|
+
{k: v["name"] for k, v in config["models"].items()},
|
|
203
|
+
retrieval,
|
|
204
|
+
)
|
|
205
|
+
decomposer.controller = tracing
|
|
206
|
+
# Observe find_answer_decomp without substituting its loop or
|
|
207
|
+
# the source return_qid_prediction JSON/string extraction.
|
|
208
|
+
native_find = decomposer.find_answer_decomp
|
|
209
|
+
captured = []
|
|
210
|
+
|
|
211
|
+
def observed_find(example, debug=False):
|
|
212
|
+
result = native_find(example, debug=debug)
|
|
213
|
+
captured.append(result[0])
|
|
214
|
+
return result
|
|
215
|
+
|
|
216
|
+
decomposer.find_answer_decomp = observed_find
|
|
217
|
+
example = dict(
|
|
218
|
+
qid=legacy.task_id or legacy.request_id,
|
|
219
|
+
query=legacy.question,
|
|
220
|
+
question=legacy.question,
|
|
221
|
+
metadata={"pinned_title": None, "pinned_para": None, "pin_position": None},
|
|
222
|
+
titles=[],
|
|
223
|
+
paras=[],
|
|
224
|
+
)
|
|
225
|
+
qid, answer, chain = decomposer.return_qid_prediction(example, silent=True)
|
|
226
|
+
if len(captured) != 1 or captured[0] is None or qid != example["qid"]:
|
|
227
|
+
raise DriverError(
|
|
228
|
+
"NATIVE_QA_NO_FINAL_STATE",
|
|
229
|
+
"native full QA produced no final state",
|
|
230
|
+
pre_execution=False,
|
|
231
|
+
)
|
|
232
|
+
if not isinstance(answer, str) or not answer.strip():
|
|
233
|
+
raise DriverError(
|
|
234
|
+
"NATIVE_QA_EMPTY_OR_NON_TEXT",
|
|
235
|
+
"native full QA returned an empty or non-text prediction",
|
|
236
|
+
pre_execution=False,
|
|
237
|
+
)
|
|
238
|
+
result = _project_trajectory(
|
|
239
|
+
legacy,
|
|
240
|
+
checkout,
|
|
241
|
+
config,
|
|
242
|
+
tracing.events,
|
|
243
|
+
captured[0],
|
|
244
|
+
native,
|
|
245
|
+
model_map,
|
|
246
|
+
bindings,
|
|
247
|
+
)
|
|
248
|
+
result.update(
|
|
249
|
+
identity=identity,
|
|
250
|
+
answer=answer,
|
|
251
|
+
exit_reason="answer_detected",
|
|
252
|
+
usage=bindings.usage(),
|
|
253
|
+
native_chain=chain,
|
|
254
|
+
native_controller_events=to_jsonable(tracing.events),
|
|
255
|
+
native_participant_model_calls=result["usage"]["model_calls"],
|
|
256
|
+
native_model_classes={
|
|
257
|
+
k: type(v).__module__ + "." + type(v).__name__ for k, v in model_map.items()
|
|
258
|
+
},
|
|
259
|
+
controlled_runtime_events=copy.deepcopy(self.last_events),
|
|
260
|
+
native_configuration_sha256=content_hash(config),
|
|
261
|
+
unknown_token_usage_model_calls=bindings.unknown_usage_calls,
|
|
262
|
+
usage_is_complete=bindings.unknown_usage_calls == 0,
|
|
263
|
+
upstream_stdout_sha256=capture.hexdigest,
|
|
264
|
+
)
|
|
265
|
+
if self.fingerprint() != identity:
|
|
266
|
+
raise DriverError(
|
|
267
|
+
"BINDING_MISMATCH",
|
|
268
|
+
"native IRCoT-QA runtime identity changed",
|
|
269
|
+
pre_execution=False,
|
|
270
|
+
)
|
|
271
|
+
return result
|
|
272
|
+
except (Exception, SystemExit) as exc:
|
|
273
|
+
usage = (
|
|
274
|
+
bindings.usage()
|
|
275
|
+
if bindings
|
|
276
|
+
else dict(model_calls=0, retrieval_calls=0, input_tokens=0, output_tokens=0)
|
|
277
|
+
)
|
|
278
|
+
return dict(
|
|
279
|
+
status="error",
|
|
280
|
+
failure_code=getattr(exc, "code", "NATIVE_QA_EXECUTION_FAILED"),
|
|
281
|
+
error_type=type(exc).__name__,
|
|
282
|
+
message=getattr(exc, "public_message", "native IRCoT-QA execution failed"),
|
|
283
|
+
consumed=bool(usage["model_calls"] or usage["retrieval_calls"]),
|
|
284
|
+
usage=usage,
|
|
285
|
+
controlled_runtime_events=copy.deepcopy(self.last_events),
|
|
286
|
+
unknown_token_usage_model_calls=bindings.unknown_usage_calls if bindings else 0,
|
|
287
|
+
usage_is_complete=not bindings or bindings.unknown_usage_calls == 0,
|
|
288
|
+
)
|
|
289
|
+
finally:
|
|
290
|
+
self.last_usage = bindings.usage() if bindings else {}
|
|
291
|
+
_NATIVE_LOCK.release()
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class _QABindings(_LiveBindings):
|
|
295
|
+
def __init__(self, owner, request, full_request):
|
|
296
|
+
super().__init__(owner, request)
|
|
297
|
+
self.full_request = full_request
|
|
298
|
+
|
|
299
|
+
def generator_class(self, stage):
|
|
300
|
+
binding = self
|
|
301
|
+
owner = self.owner
|
|
302
|
+
|
|
303
|
+
class Generator:
|
|
304
|
+
def __init__(self, **kwargs):
|
|
305
|
+
self.options = dict(kwargs)
|
|
306
|
+
|
|
307
|
+
def generate_text_sequence(self, prompt):
|
|
308
|
+
cap = min(owner.per_call_output_tokens, int(self.options.get("max_tokens", 300)))
|
|
309
|
+
# Native default300 remains300 even though the common ceiling
|
|
310
|
+
# is512. Reserve one native final call and its bounded output.
|
|
311
|
+
reserved = min(owner.per_call_output_tokens, 300) if stage == "reasoning" else 0
|
|
312
|
+
remaining = binding.request.controls.max_output_tokens - binding.output_tokens
|
|
313
|
+
if (
|
|
314
|
+
len(binding.generations)
|
|
315
|
+
>= binding.full_request.max_model_calls - (1 if stage == "reasoning" else 0)
|
|
316
|
+
or remaining <= reserved
|
|
317
|
+
):
|
|
318
|
+
raise DriverError(
|
|
319
|
+
"MODEL_BUDGET_EXCEEDED", "full QA shared budget exhausted before dispatch"
|
|
320
|
+
)
|
|
321
|
+
cap = min(cap, remaining - reserved)
|
|
322
|
+
if any(
|
|
323
|
+
self.options.get(k, v) != v
|
|
324
|
+
for k, v in {
|
|
325
|
+
"n": 1,
|
|
326
|
+
"best_of": 1,
|
|
327
|
+
"top_p": 1,
|
|
328
|
+
"frequency_penalty": 0,
|
|
329
|
+
"presence_penalty": 0,
|
|
330
|
+
"logprobs": 0,
|
|
331
|
+
}.items()
|
|
332
|
+
):
|
|
333
|
+
raise DriverError(
|
|
334
|
+
"GENERATOR_OPTIONS_UNSUPPORTED",
|
|
335
|
+
"native full QA received unsupported sampling options",
|
|
336
|
+
)
|
|
337
|
+
fitted = binding.fit_prompt(
|
|
338
|
+
original_prompt=prompt.rstrip(),
|
|
339
|
+
model_length_limit=8000,
|
|
340
|
+
estimated_generation_length=cap,
|
|
341
|
+
demonstration_delimiter="\n\n\n",
|
|
342
|
+
shuffle=False,
|
|
343
|
+
remove_method=self.options.get("remove_method", "first"),
|
|
344
|
+
tokenizer_model_name="gpt2",
|
|
345
|
+
last_is_test_example=True,
|
|
346
|
+
)
|
|
347
|
+
call = GenerationRequest(
|
|
348
|
+
prompt=fitted,
|
|
349
|
+
temperature=self.options.get("temperature", 0),
|
|
350
|
+
max_tokens=cap,
|
|
351
|
+
seed=owner.seed + len(binding.generations),
|
|
352
|
+
stop=tuple(self.options.get("stop", ["\n"])),
|
|
353
|
+
metadata={
|
|
354
|
+
"release_id": "acquire.ircot.native_like_substituted",
|
|
355
|
+
"stage": stage,
|
|
356
|
+
"native_generator_options": self.options,
|
|
357
|
+
"native_default_output_cap": 300,
|
|
358
|
+
},
|
|
359
|
+
)
|
|
360
|
+
event = dict(
|
|
361
|
+
operation="native_generator",
|
|
362
|
+
stage=stage,
|
|
363
|
+
request_hash=call.request_hash,
|
|
364
|
+
prompt_sha256=content_hash(fitted),
|
|
365
|
+
max_tokens=cap,
|
|
366
|
+
seed=call.seed,
|
|
367
|
+
stop=list(call.stop),
|
|
368
|
+
response_observed=False,
|
|
369
|
+
)
|
|
370
|
+
binding.generations.append(event)
|
|
371
|
+
owner.last_events.append(event)
|
|
372
|
+
response = owner.backend.generate(call)
|
|
373
|
+
event.update(
|
|
374
|
+
response_observed=True,
|
|
375
|
+
response_text=response.text,
|
|
376
|
+
finish_reason=response.finish_reason,
|
|
377
|
+
usage=to_jsonable(response.usage),
|
|
378
|
+
)
|
|
379
|
+
binding.input_tokens += response.usage.input_tokens
|
|
380
|
+
binding.output_tokens += response.usage.output_tokens
|
|
381
|
+
if response.usage.output_tokens > cap:
|
|
382
|
+
raise DriverError(
|
|
383
|
+
"OUTPUT_BUDGET_EXCEEDED",
|
|
384
|
+
"provider exceeded native full QA output cap",
|
|
385
|
+
pre_execution=False,
|
|
386
|
+
)
|
|
387
|
+
return [(response.text, 0.0)]
|
|
388
|
+
|
|
389
|
+
return Generator
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
__all__ = [
|
|
393
|
+
"QA_BINDING_MODE",
|
|
394
|
+
"QA_PROFILE",
|
|
395
|
+
"QA_ROLES",
|
|
396
|
+
"PinnedIRCoTNativeQARuntime",
|
|
397
|
+
"patch_native_qa_config",
|
|
398
|
+
]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "agentinputlab.ircot.compatibility.v1",
|
|
3
|
+
"upstream": {
|
|
4
|
+
"repository": "https://github.com/StonyBrookNLP/ircot",
|
|
5
|
+
"commit": "3c1820f698eea5eeddb4fba3c56b64c961e063e4",
|
|
6
|
+
"license_spdx": "Apache-2.0",
|
|
7
|
+
"license_sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
|
8
|
+
"require_clean_checkout": true
|
|
9
|
+
},
|
|
10
|
+
"source_files": {
|
|
11
|
+
"commaqa/inference/configurable_inference.py": "6d19f148ef0637707d1dd481e604944144eb296754c88429706a514b111db54d",
|
|
12
|
+
"commaqa/inference/constants.py": "4482a7eb15e107e72f364b7357a8963662a063a730aa3861a6b51e3018d99570",
|
|
13
|
+
"commaqa/inference/data_instances.py": "018a4addbb5d6fb81c76af049e1ae1e50ac227ab48ee7a69a9cb453b00ff6317",
|
|
14
|
+
"commaqa/inference/ircot.py": "a2233cbdae107e80cd383993edadc43c2491d2360a878b05a97dfa66a0b0eebf",
|
|
15
|
+
"commaqa/inference/model_search.py": "c22ddb376bd165a0872da11fa0310cda8c25e00c62d9efa3e47cd36371889558",
|
|
16
|
+
"commaqa/inference/prompt_reader.py": "910b6e971edd00460a570260d669128163965f2f61fe230db64be055131d311e",
|
|
17
|
+
"commaqa/inference/participant_qa.py": "2e7e1e69509abc48483a1a656bd1e0fa97eff8692ad83cb11b962771a47a6eef",
|
|
18
|
+
"commaqa/execution/llm_qa_model.py": "1cbf61aa5d05948103b612bb7191319f4c895d6442a1fff1efc752548a8d4543",
|
|
19
|
+
"commaqa/models/gpt3generator.py": "93ef7b884859e63a56c1ad4ac28dcb3bdd15da1090cd9552908ff6dd2d0e9c78"
|
|
20
|
+
},
|
|
21
|
+
"allowed_model_names": [
|
|
22
|
+
"retrieve_and_reset_paragraphs",
|
|
23
|
+
"step_by_step_cot_gen",
|
|
24
|
+
"step_by_step_exit_controller",
|
|
25
|
+
"copy_question",
|
|
26
|
+
"llmqa",
|
|
27
|
+
"answer_extractor"
|
|
28
|
+
],
|
|
29
|
+
"profiles": {
|
|
30
|
+
"upstream.hotpotqa.codex_qa.v1": {
|
|
31
|
+
"identity_kind": "upstream_native",
|
|
32
|
+
"config_scope": "upstream",
|
|
33
|
+
"config_path": "base_configs/ircot_qa_codex_hotpotqa.jsonnet",
|
|
34
|
+
"config_sha256": "191607081660bac659b1513718aa74d66250fa5e0f6bc8dd5efb0e459eb95cb6",
|
|
35
|
+
"prompt_scope": "upstream",
|
|
36
|
+
"prompt_path": "prompts/hotpotqa/gold_with_2_distractors_context_cot_qa_codex.txt",
|
|
37
|
+
"prompt_sha256": "af56b51582a5fdbebf5a5d50ab95cefcb95d92986cf820774ccbaa897a71552f",
|
|
38
|
+
"corpus_revision": "hotpotqa",
|
|
39
|
+
"binding_attestation": "startup_exact_model_retriever_tokenizer_dependencies_with_explicit_substitutions",
|
|
40
|
+
"config_ext_vars": {
|
|
41
|
+
"RETRIEVER_HOST": "AGENTINPUTLAB_IRCOT_RETRIEVER_HOST",
|
|
42
|
+
"RETRIEVER_PORT": "AGENTINPUTLAB_IRCOT_RETRIEVER_PORT"
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"patches": [
|
|
47
|
+
{
|
|
48
|
+
"id": "native_qa_bounded_controls_v1",
|
|
49
|
+
"changes": [
|
|
50
|
+
"bound native reasoning sentences, retrieval counts and active documents to explicit controls",
|
|
51
|
+
"reserve the native final QA call inside the same total call/output budget",
|
|
52
|
+
"retain the original six-participant graph, prompt filtering/fitting and answer extractor",
|
|
53
|
+
"bind generation/retrieval/GPT2 tokenizer at startup, without native cache or retry"
|
|
54
|
+
]
|
|
55
|
+
}
|
|
56
|
+
]
|
|
57
|
+
}
|
workers/ircot/runtime.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Core ``WorkerClient`` adapter for the pinned IRCoT driver."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from agentinputlab.contracts import ExternalDependencyError
|
|
11
|
+
from agentinputlab.integrations.external_workers.ircot import IRCoTWorkerRequest
|
|
12
|
+
|
|
13
|
+
from .driver import (
|
|
14
|
+
DEFAULT_MANIFEST,
|
|
15
|
+
DEFAULT_PROFILE,
|
|
16
|
+
PROTOCOL_VERSION,
|
|
17
|
+
UPSTREAM_COMMIT,
|
|
18
|
+
DriverError,
|
|
19
|
+
IRCoTDriver,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class PinnedIRCoTWorker:
|
|
24
|
+
"""Expose the manifest-locked fixture profile through ``WorkerClient``.
|
|
25
|
+
|
|
26
|
+
The direct adapter is intentionally suitable only for the frozen profile.
|
|
27
|
+
Upstream-native profiles fail closed in :class:`IRCoTDriver` unless the
|
|
28
|
+
driver is launched as its isolated JSONL process.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
worker_id = "agentinputlab.ircot.pinned.in_process.v1"
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
*,
|
|
36
|
+
manifest_path: Path = DEFAULT_MANIFEST,
|
|
37
|
+
profile_id: str = DEFAULT_PROFILE,
|
|
38
|
+
environment: Mapping[str, str] | None = None,
|
|
39
|
+
) -> None:
|
|
40
|
+
self.driver = IRCoTDriver(
|
|
41
|
+
manifest_path=manifest_path,
|
|
42
|
+
profile_id=profile_id,
|
|
43
|
+
environment=environment,
|
|
44
|
+
isolated_process=False,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def call(self, payload: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
48
|
+
"""Return the driver's structured success or fail-closed response."""
|
|
49
|
+
|
|
50
|
+
return self.driver.safe_call(payload)
|
|
51
|
+
|
|
52
|
+
def call_controlled(self, request: IRCoTWorkerRequest) -> Mapping[str, Any]:
|
|
53
|
+
"""Call the driver with the bridge's immutable request contract."""
|
|
54
|
+
|
|
55
|
+
if not isinstance(request, IRCoTWorkerRequest):
|
|
56
|
+
raise TypeError("PinnedIRCoTWorker requires IRCoTWorkerRequest")
|
|
57
|
+
return self.call(request.as_dict())
|
|
58
|
+
|
|
59
|
+
def fingerprint(self) -> Mapping[str, Any]:
|
|
60
|
+
"""Return source, manifest, profile, and upstream execution identity."""
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
identity = self.driver.identity()
|
|
64
|
+
driver_sha256 = _sha256_file(Path(__file__).with_name("driver.py"))
|
|
65
|
+
adapter_sha256 = _sha256_file(Path(__file__))
|
|
66
|
+
except DriverError as exc:
|
|
67
|
+
raise ExternalDependencyError(
|
|
68
|
+
f"pinned IRCoT runtime identity failed with {exc.code}"
|
|
69
|
+
) from exc
|
|
70
|
+
return {
|
|
71
|
+
"worker_id": self.worker_id,
|
|
72
|
+
"execution_mode": "in_process_external_environment",
|
|
73
|
+
"protocol_version": PROTOCOL_VERSION,
|
|
74
|
+
"upstream_revision": UPSTREAM_COMMIT,
|
|
75
|
+
"profile_id": identity["profile_id"],
|
|
76
|
+
"identity_kind": identity["identity_kind"],
|
|
77
|
+
"compatibility_manifest_sha256": identity["manifest_sha256"],
|
|
78
|
+
"config_sha256": identity["config_sha256"],
|
|
79
|
+
"prompt_sha256": identity["prompt_sha256"],
|
|
80
|
+
"binding_attestation": identity["binding_attestation"],
|
|
81
|
+
"full_api_status": identity["full_api_status"],
|
|
82
|
+
"driver_sha256": driver_sha256,
|
|
83
|
+
"runtime_adapter_sha256": adapter_sha256,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _sha256_file(path: Path) -> str:
|
|
88
|
+
try:
|
|
89
|
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
90
|
+
except OSError as exc:
|
|
91
|
+
raise DriverError(
|
|
92
|
+
"RUNTIME_SOURCE_MISSING",
|
|
93
|
+
"IRCoT runtime source file is unavailable",
|
|
94
|
+
) from exc
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
__all__ = ["PinnedIRCoTWorker"]
|