agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# Third-party source and asset boundaries
|
|
2
|
+
|
|
3
|
+
AgentInputLab standardizes interfaces and supplies clean-room control logic,
|
|
4
|
+
thin adapters, replay schemas, and external-worker protocols. It does not
|
|
5
|
+
vendor the audited upstream repositories, model weights, or benchmark data.
|
|
6
|
+
Small worker-side compatibility paths may follow a permissively licensed
|
|
7
|
+
upstream control flow; those paths name the source symbol and exact revision in
|
|
8
|
+
their local manifest and preserve the upstream attribution.
|
|
9
|
+
|
|
10
|
+
The authoritative, commit-pinned inventory is
|
|
11
|
+
`manifests/algorithms/registry.yaml`. The following summary is a release
|
|
12
|
+
safeguard, not legal advice.
|
|
13
|
+
|
|
14
|
+
| Method | Source license disposition | Core distribution rule |
|
|
15
|
+
|---|---|---|
|
|
16
|
+
| Query2doc official generations | CC-BY-4.0 asset | Download by manifest; preserve attribution |
|
|
17
|
+
| QueryGym | Apache-2.0 | Optional community adapter; source not vendored |
|
|
18
|
+
| Rewrite-Retrieve-Read | NOASSERTION | External checkout only |
|
|
19
|
+
| ReFormeR | NOASSERTION | External checkout only |
|
|
20
|
+
| InfoCQR | NOASSERTION | External checkout only |
|
|
21
|
+
| Plan-and-Solve | NOASSERTION | Clean-room paper-spec or external checkout |
|
|
22
|
+
| Decomposed Prompting | Apache-2.0 | Pinned external worker; bounded best-first compatibility path follows `BestFirstDecomposer.find_answer_decomp` and preserves attribution |
|
|
23
|
+
| Self-Ask | MIT | Minimal faithful engineering; preserve notice |
|
|
24
|
+
| LLMCompiler | MIT | Optional external/full-system integration |
|
|
25
|
+
| Plan-and-Budget | MIT | Minimal logic port requires preserved notice |
|
|
26
|
+
| HyDE | NOASSERTION | Clean-room paper reimplementation only |
|
|
27
|
+
| IRCoT | Apache-2.0 | Isolated external worker |
|
|
28
|
+
| Adaptive-RAG | Apache-2.0 | Rebuild recipe; assets separately licensed |
|
|
29
|
+
| R²AG | MIT | Native/factorial integration; weights separately gated |
|
|
30
|
+
| Q-RAG | NOASSERTION | User-provided checkout/checkpoints only |
|
|
31
|
+
| Ask or Assume? | MIT root; PolyForm enterprise subtree | Enterprise subtree excluded |
|
|
32
|
+
| UDCS | Apache-2.0 | Prompt/component integration; preserve notice |
|
|
33
|
+
| LongLLMLingua and LLMLingua-2 code | MIT | Lazy official adapter; weights/data separately governed |
|
|
34
|
+
| LLMLingua-2 MeetingBank training data | CC-BY-NC-SA-4.0 | Never bundled by runtime package |
|
|
35
|
+
| RECOMP code | MIT | Weights and training data remain NOASSERTION |
|
|
36
|
+
| ACON | MIT | Runtime/recipe integration; paper/code drift disclosed |
|
|
37
|
+
| SmartSearch | MIT | Optional isolated inference/training environment |
|
|
38
|
+
| RQ-RAG | NOASSERTION | External checkout only |
|
|
39
|
+
| ACE | Apache-2.0 | External worker/replay protocol |
|
|
40
|
+
| MCE | MIT | Sandboxed external recipe only |
|
|
41
|
+
|
|
42
|
+
The Plan-and-Budget allocation/schedule port follows
|
|
43
|
+
`junhongmit/P-and-B@2ca2d90c6d3d630ec2b11cc22e0f56751f7f2b7e`.
|
|
44
|
+
Copyright (c) 2024 Junhong Lin. Its upstream MIT permission notice is reproduced
|
|
45
|
+
in `NOTICE`; cache datasets and their underlying terms are separate and are not
|
|
46
|
+
bundled. Solver references/answers are excluded from the public cache view.
|
|
47
|
+
|
|
48
|
+
The Query2doc cached loader audits only the author's two native TREC split files
|
|
49
|
+
at `intfloat/query2doc_msmarco@3019b760cd849830aec6d0654dbe8a4a03e791d6`.
|
|
50
|
+
Attribution: Liang Wang, Nan Yang and Furu Wei, *Query2doc: Query Expansion with
|
|
51
|
+
Large Language Models* (EMNLP 2023). These are `text-davinci-003` generations;
|
|
52
|
+
the package contains checksums and loading code, not the generations. Use also
|
|
53
|
+
requires review of the underlying MS MARCO data terms.
|
|
54
|
+
|
|
55
|
+
## Optional retrieval model profiles
|
|
56
|
+
|
|
57
|
+
AgentInputLab distributes only model identifiers, immutable repository
|
|
58
|
+
revisions, and preprocessing contracts for the profiles below. It does not
|
|
59
|
+
bundle, mirror, or redistribute model weights. A configured runtime obtains
|
|
60
|
+
weights from the user's local cache or, only when explicitly enabled, from the
|
|
61
|
+
named upstream repository.
|
|
62
|
+
|
|
63
|
+
| Profile asset | License disposition | Core distribution rule |
|
|
64
|
+
|---|---|---|
|
|
65
|
+
| BAAI BGE base English v1.5 | MIT | Pinned downloader/local-cache profile; no bundled weights |
|
|
66
|
+
| intfloat E5 base v2 | MIT | Pinned downloader/local-cache profile; no bundled weights |
|
|
67
|
+
| Qwen3 Embedding 0.6B | Apache-2.0 | Pinned downloader/local-cache profile; no bundled weights |
|
|
68
|
+
| facebook Contriever MS MARCO | CC-BY-NC-4.0 | Included for the current academic research scope; pinned downloader/local-cache profile; no bundled weights |
|
|
69
|
+
|
|
70
|
+
## Benchmark and metric dependencies
|
|
71
|
+
|
|
72
|
+
| Asset or package | License disposition | Distribution rule |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| BEIR ArguAna official archive | CC-BY-SA-4.0 | Checksum-pinned downloader only; data is not included in wheels, sdists, containers, or Git |
|
|
75
|
+
| BRIGHT Earth Science and Economics | CC-BY-4.0 | Official query/qrel and corpus Parquet components are independently checksum-pinned; downloader only |
|
|
76
|
+
| MuSiQue v1.0 | CC-BY-4.0 | Official archive is checksum-pinned; only answerable dev and its evaluator-only overlap audit are normalized locally |
|
|
77
|
+
| 2WikiMultiHopQA `data_ids_april7` | Apache-2.0 | Official corrected archive and alias table are checksum-pinned; only dev is normalized locally |
|
|
78
|
+
| HotpotQA distractor validation | CC-BY-SA-4.0 | Official Hugging Face organization mirror is checksum-pinned; source annotation defects remain auditable |
|
|
79
|
+
| QASPER v0.3 train/dev archive | CC-BY-4.0 | Official archive is checksum-pinned; only the caption-augmented dev setting is normalized locally |
|
|
80
|
+
| tau2 1.0.1 Banking Knowledge source | MIT | Exact-commit archive is checksum-pinned and downloader-only; hidden simulator, evaluator, and database state remain evaluator-side in local prepared caches |
|
|
81
|
+
| AppWorld / AppWorld-UL | Apache-2.0 code repository; benchmark assets separately governed upstream | Exact-commit external runtime bridge; no runtime, simulator state, evaluator gold, or task rows are vendored |
|
|
82
|
+
| Microsoft STATE-Bench 0.8.1 | MIT | Exact-commit external runtime bridge; only a project-authored hash-locked task-ID membership index is packaged |
|
|
83
|
+
| Apple ToolSandbox | Apple source-code license in the repository `LICENSE` | Exact-commit external runtime bridge for academic evaluation; upstream runtime, scenarios, and evaluator implementation are not redistributed |
|
|
84
|
+
| ClariQ official dev TSV | NOASSERTION | Explicit local license-review gate; no rows or derived cache are redistributed |
|
|
85
|
+
| `ir-measures` | MIT | Optional `evaluation` dependency; resolved version is recorded in each evaluation environment |
|
|
86
|
+
| `pyarrow` | Apache-2.0 | Optional `benchmarks` dependency used to read pinned upstream Parquet assets; no benchmark rows are bundled |
|
|
87
|
+
| `rouge-score` | Apache-2.0 | Optional `evaluation` dependency; resolved version is recorded in each evaluation environment |
|
|
88
|
+
|
|
89
|
+
The byte locks and source provenance for audited smoke data are maintained in
|
|
90
|
+
`manifests/benchmarks/smoke_assets.json`. Synthetic unit-test fixtures are
|
|
91
|
+
project-authored and contain no copied benchmark rows.
|
|
92
|
+
|
|
93
|
+
For all assets, source, model, and data licenses are evaluated independently.
|
|
94
|
+
An open-source code license never propagates automatically to weights or data.
|
workers/README.md
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# Pinned upstream workers
|
|
2
|
+
|
|
3
|
+
This directory contains auditable drivers for algorithms whose original runtime
|
|
4
|
+
cannot safely live inside the dependency-free AgentInputLab wheel. A worker is
|
|
5
|
+
not a vendored copy of an upstream repository: users provide the exact checkout
|
|
6
|
+
and optional weights, while the driver verifies the declared commit, license,
|
|
7
|
+
source files, configuration artifacts, and protocol identity before execution.
|
|
8
|
+
|
|
9
|
+
The worker boundary standardizes transport, limits, trace projection, and
|
|
10
|
+
fair-comparison settings. It does not replace the upstream algorithm. Any
|
|
11
|
+
compatibility or security patch is listed in the method's manifest and included
|
|
12
|
+
in its runtime fingerprint.
|
|
13
|
+
|
|
14
|
+
The following claims are deliberately separate:
|
|
15
|
+
|
|
16
|
+
1. **Protocol tested** — malformed inputs and outputs fail closed.
|
|
17
|
+
2. **Pinned plumbing tested** — a small checkout exercises the real upstream
|
|
18
|
+
symbols through the AgentInputLab bridge.
|
|
19
|
+
3. **Upstream boot tested** — the audited checkout imports in its locked
|
|
20
|
+
dependency environment.
|
|
21
|
+
4. **Result parity tested** — official fixtures or benchmark outputs match the
|
|
22
|
+
upstream reference within a declared tolerance.
|
|
23
|
+
|
|
24
|
+
A worker remains `external_only`, `planned`, or `retrain_required` until the
|
|
25
|
+
applicable upstream boot and parity gates pass. Fake fixtures are never evidence
|
|
26
|
+
of paper-result reproduction.
|
|
27
|
+
|
|
28
|
+
Current worker directories are documented independently. Each README lists its
|
|
29
|
+
upstream revision, supported profile, required environment variables, execution
|
|
30
|
+
mode, and remaining parity limitations.
|
|
31
|
+
|
|
32
|
+
The `ask_or_assume/` worker is intentionally narrower than a full OpenHands
|
|
33
|
+
deployment: it runs only the pinned IntentAgent decision and forced-question
|
|
34
|
+
component path and labels all outputs as component-only.
|
|
35
|
+
|
|
36
|
+
The `reformer/` worker executes source only from a user-mounted pinned checkout.
|
|
37
|
+
Its upstream has no license file, so it requires operator and per-request
|
|
38
|
+
`NOASSERTION` acknowledgements and never vendors upstream code.
|
|
39
|
+
|
|
40
|
+
The `recomp/` worker imports the pinned official sentence-scoring API and
|
|
41
|
+
replays the official abstractive prompt/generate path against user-mounted,
|
|
42
|
+
hash-attested checkpoints. It fixes each input block as one candidate sentence,
|
|
43
|
+
requires a retriever fingerprint without performing retrieval, and keeps
|
|
44
|
+
paper-native top-N assembly separate from controlled assembly.
|
|
45
|
+
|
|
46
|
+
The `rrr/` worker verifies the unlicensed, credential-risk pinned tarball
|
|
47
|
+
without extracting/importing it, then runs a hash-bound compatible inference
|
|
48
|
+
chain over user-mounted prompts, frozen search artifacts, and local checkpoints.
|
|
49
|
+
Warm-up/PPO checkpoints are inference-only; controlled retrieval is explicitly
|
|
50
|
+
separate from the frozen Bing-cache profile.
|
|
51
|
+
|
|
52
|
+
The `infocqr/` worker directly calls pinned official teacher prompt functions
|
|
53
|
+
or replays checksum-locked author caches. Because neither source/cache licensing
|
|
54
|
+
nor a released T5 student checkpoint is available, it is dual-gated and never
|
|
55
|
+
vendors upstream material or fabricates student weights.
|
|
56
|
+
|
|
57
|
+
The `smartsearch/` worker runs only frozen inference with the upstream
|
|
58
|
+
10-round/top-5/8192-token loop. It replaces the unsafe code-execution search
|
|
59
|
+
sandbox with a fixed retriever shim and labels the result as a controlled
|
|
60
|
+
adapter; training and paper-result parity are outside this worker.
|
|
61
|
+
|
|
62
|
+
The `qrag/` worker runs a user-mounted, hash-attested frozen checkpoint through
|
|
63
|
+
the two-step E5-Hotpot action path and records every available Q-vector, action,
|
|
64
|
+
and selected candidate. Source and checkpoint licenses are acknowledged
|
|
65
|
+
separately; reward, gold labels, and training are unavailable at evaluation
|
|
66
|
+
time.
|
|
67
|
+
|
|
68
|
+
The `adaptive_rag/` worker verifies a clean pinned checkout and exact router
|
|
69
|
+
snapshot/weights/tokenizer, generator, retriever, corpus, index, and prompt
|
|
70
|
+
closures. A supervisor applies the upstream first-step A/B/C routing rule,
|
|
71
|
+
reserves budget before every call, executes only the selected **controlled
|
|
72
|
+
reimplementation** arm, and emits a hash-linked event log that the host
|
|
73
|
+
independently recomputes. Its hermetic runtime is fixture-only; neither the
|
|
74
|
+
source-checkout smoke nor this controlled arm is evidence of upstream arm,
|
|
75
|
+
batch/postprocess-script, or paper-result parity.
|
|
76
|
+
|
|
77
|
+
The `r2ag/` worker verifies a clean pinned RRAG checkout plus separately
|
|
78
|
+
hash-bound R²-Former, projection, feature schema, and complete reader/tokenizer/
|
|
79
|
+
feature-embedder file closures. It is designed to execute RFormer → projection
|
|
80
|
+
→ retrieval-marker-only embedding replacement → reader generation on an
|
|
81
|
+
allowlisted, fixed candidate DTO and is labeled `controlled_source_informed`.
|
|
82
|
+
CI exercises the complete host/real-driver boundary with a fixture runtime; it
|
|
83
|
+
does not run a production neural runtime, official paper weights, or
|
|
84
|
+
paper-result parity.
|
|
85
|
+
|
|
86
|
+
The `mce/` worker loads the hash-pinned upstream `run_iteration` orchestration
|
|
87
|
+
but applies a hash-bound semantic patch at four unsafe/unimplemented seams:
|
|
88
|
+
meta/base agents run through an isolated manager, interface validation is
|
|
89
|
+
remote/static without local import, and evaluation runs in a separate trusted
|
|
90
|
+
oracle supervisor. Generated code remains in a verified worker-side
|
|
91
|
+
content-addressed store; the host receives only an opaque handle, hashes,
|
|
92
|
+
scores, usage, and a closed regular-file manifest. This is a controlled thin
|
|
93
|
+
wrapper—not `official_native` E2B or paper-result parity—and production
|
|
94
|
+
attestation/credentials remain operator supplied.
|
workers/ace/README.md
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
# ACE generic text-task worker
|
|
2
|
+
|
|
3
|
+
This directory contains the real upstream worker for
|
|
4
|
+
`recipe.ace.offline`. It does not reimplement ACE: every successful request
|
|
5
|
+
constructs the pinned upstream `ace.ace.ACE` object and calls
|
|
6
|
+
`ACE.run(mode="offline")`.
|
|
7
|
+
|
|
8
|
+
## Pinned upstream
|
|
9
|
+
|
|
10
|
+
- Repository: `https://github.com/ace-agent/ace.git`
|
|
11
|
+
- Commit: `bcb7cea0504afad6f55fec4845dd4864c9f9eee7`
|
|
12
|
+
- License: Apache-2.0
|
|
13
|
+
- License SHA-256:
|
|
14
|
+
`c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4`
|
|
15
|
+
|
|
16
|
+
`compatibility_manifest.json` pins the license, package metadata, ACE
|
|
17
|
+
orchestrator, Generator, Reflector, Curator, prompts, LLM helper, logging, and
|
|
18
|
+
playbook helper sources. The worker verifies the Git HEAD and every listed hash
|
|
19
|
+
before importing anything from the mounted checkout. Symlinked pinned files,
|
|
20
|
+
source drift, a different commit, a dirty checkout (including untracked files),
|
|
21
|
+
and AppWorld imports fail closed.
|
|
22
|
+
|
|
23
|
+
The checkout is an operator startup argument. A JSONL request cannot select a
|
|
24
|
+
checkout, path, module, callable, command, endpoint, or environment variable.
|
|
25
|
+
The driver never invokes a shell and only invokes a fixed, argument-list
|
|
26
|
+
`git status --porcelain` subprocess during startup verification; requests
|
|
27
|
+
cannot influence that command. It never adds `ace-appworld` to the import path.
|
|
28
|
+
BulletpointAnalyzer is disabled.
|
|
29
|
+
|
|
30
|
+
## Install upstream dependencies
|
|
31
|
+
|
|
32
|
+
Use an isolated Python 3.10+ worker environment:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install 'agentinputlab[ace]'
|
|
36
|
+
python -m workers.ace.driver --checkout /opt/upstreams/ace --verify-only
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The `ace` extra installs the dependencies used by this generic-text worker:
|
|
40
|
+
NumPy, the OpenAI-compatible client, Python-dotenv and Tiktoken. It does not
|
|
41
|
+
install the upstream checkout or unrelated embedding models, FAISS, AppWorld,
|
|
42
|
+
or the optional BulletpointAnalyzer stack. BulletpointAnalyzer remains disabled
|
|
43
|
+
in this profile. Mounted source and license hashes are still verified before
|
|
44
|
+
imports; installing dependencies is not a substitute for that verification.
|
|
45
|
+
The worker itself adds no hidden model client. Credentials must already be
|
|
46
|
+
exported in the worker environment using the upstream provider variable:
|
|
47
|
+
|
|
48
|
+
- `OPENAI_API_KEY`
|
|
49
|
+
- `SAMBANOVA_API_KEY`
|
|
50
|
+
- `TOGETHER_API_KEY`
|
|
51
|
+
- `COMMONSTACK_API_KEY`
|
|
52
|
+
|
|
53
|
+
Loading credentials from a checkout `.env` is disabled. The default path keeps
|
|
54
|
+
the endpoints implemented by the pinned upstream. An operator may instead bind
|
|
55
|
+
an OpenAI-compatible deployment once at worker startup with `base_url` and
|
|
56
|
+
`api_key_env`; JSONL requests still cannot choose or mutate an endpoint,
|
|
57
|
+
credential name, model, or deployment option.
|
|
58
|
+
|
|
59
|
+
Verify the checkout and import without starting JSONL service:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m workers.ace.driver \
|
|
63
|
+
--checkout /opt/upstreams/ace \
|
|
64
|
+
--verify-only
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Explicit LLM configuration
|
|
68
|
+
|
|
69
|
+
Serve mode requires `--llm-config-json`. There are no implicit model or pricing
|
|
70
|
+
defaults. The three fingerprints must exactly equal the
|
|
71
|
+
`ace_model_fingerprints` used by `ACEOfflineRecipe.fit`.
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{
|
|
75
|
+
"api_provider": "openai",
|
|
76
|
+
"max_tokens_per_call": 4096,
|
|
77
|
+
"json_mode": true,
|
|
78
|
+
"base_url": "http://127.0.0.1:8000/v1",
|
|
79
|
+
"api_key_env": "OPENAI_API_KEY",
|
|
80
|
+
"extra_body": {
|
|
81
|
+
"chat_template_kwargs": {"enable_thinking": false}
|
|
82
|
+
},
|
|
83
|
+
"roles": {
|
|
84
|
+
"target": {
|
|
85
|
+
"model": "generator-model-id",
|
|
86
|
+
"fingerprint": {
|
|
87
|
+
"provider": "openai",
|
|
88
|
+
"model": "generator-model-id",
|
|
89
|
+
"revision": "deployment-revision"
|
|
90
|
+
},
|
|
91
|
+
"input_cost_per_million": 1.0,
|
|
92
|
+
"output_cost_per_million": 2.0
|
|
93
|
+
},
|
|
94
|
+
"reflector": {
|
|
95
|
+
"model": "reflector-model-id",
|
|
96
|
+
"fingerprint": {
|
|
97
|
+
"provider": "openai",
|
|
98
|
+
"model": "reflector-model-id",
|
|
99
|
+
"revision": "deployment-revision"
|
|
100
|
+
},
|
|
101
|
+
"input_cost_per_million": 1.0,
|
|
102
|
+
"output_cost_per_million": 2.0
|
|
103
|
+
},
|
|
104
|
+
"curator": {
|
|
105
|
+
"model": "curator-model-id",
|
|
106
|
+
"fingerprint": {
|
|
107
|
+
"provider": "openai",
|
|
108
|
+
"model": "curator-model-id",
|
|
109
|
+
"revision": "deployment-revision"
|
|
110
|
+
},
|
|
111
|
+
"input_cost_per_million": 1.0,
|
|
112
|
+
"output_cost_per_million": 2.0
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Zero prices are valid for explicitly unpriced local deployments; they must
|
|
119
|
+
still be stated. `base_url`, `api_key_env`, and `extra_body` are optional,
|
|
120
|
+
startup-only deployment controls and are accepted only for `api_provider` set
|
|
121
|
+
to `openai`. For an operator-bound endpoint, the worker sends the fit request's
|
|
122
|
+
`controls.seed` to all three role models as well as seeding local algorithm
|
|
123
|
+
randomness. A different seed in `extra_body` is rejected before a provider call;
|
|
124
|
+
the worker does not silently override either declaration. Model/server seed
|
|
125
|
+
support determines reproducibility. Start the worker with the serialized object
|
|
126
|
+
as one argument:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
python workers/ace/driver.py \
|
|
130
|
+
--checkout /opt/upstreams/ace \
|
|
131
|
+
--llm-config-json "$ACE_LLM_CONFIG"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The process reads one strict JSON object per stdin line and writes exactly one
|
|
135
|
+
JSON object per stdout line. Upstream stdout is discarded into a one-MiB
|
|
136
|
+
bounded hash sink; the response retains only its byte count and SHA-256 digest,
|
|
137
|
+
never raw prompts or responses. Exceeding the bound fails the request closed.
|
|
138
|
+
Duplicate JSON keys, non-finite values, unknown protocol fields, and model
|
|
139
|
+
fingerprint mismatches are rejected.
|
|
140
|
+
|
|
141
|
+
## Generic text-task mapping
|
|
142
|
+
|
|
143
|
+
The existing recipe sends `ACEExample` records. The worker maps each record to
|
|
144
|
+
the upstream task-neutral shape:
|
|
145
|
+
|
|
146
|
+
```text
|
|
147
|
+
ACEExample.input -> question and optional context
|
|
148
|
+
ACEExample.expected_output -> target
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
A string input becomes the question directly. A mapping may contain exactly one
|
|
152
|
+
of `question`, `query`, `prompt`, `instruction`, or `q`, plus optional
|
|
153
|
+
`context`. With no recognized question field, the whole JSON value is rendered
|
|
154
|
+
canonically as text. Non-string targets are also rendered as canonical JSON.
|
|
155
|
+
By default, evaluation is stripped, case-insensitive exact match.
|
|
156
|
+
|
|
157
|
+
### Native retrieval feedback
|
|
158
|
+
|
|
159
|
+
For retrieval fitting, `ACERetrievalFeedback` binds an explicit benchmark
|
|
160
|
+
adapter, retriever, and disjoint training/validation groups. It preserves the
|
|
161
|
+
adapter's native qrels and exclusion rules. Training inputs contain the visible
|
|
162
|
+
information need and evidence; their evaluator target is an opaque case ID,
|
|
163
|
+
not a gold query or an answer. The upstream ACE reflection branch tests whether
|
|
164
|
+
the declared native metric is positive; validation selection uses its mean
|
|
165
|
+
(for example, nDCG@10), not text exact match. Invalid query JSON has zero
|
|
166
|
+
training feedback without extraction or another model call. Infrastructure
|
|
167
|
+
errors remain errors. This controlled task mapping is not a paper-parity claim.
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from agentinputlab.recipes import ACERetrievalFeedback
|
|
171
|
+
|
|
172
|
+
feedback = ACERetrievalFeedback(
|
|
173
|
+
adapter=adapter, retriever=retriever,
|
|
174
|
+
train=train_examples, validation=validation_examples,
|
|
175
|
+
metric="ndcg_at_10", top_k=15,
|
|
176
|
+
)
|
|
177
|
+
train = feedback.fit_examples("train")
|
|
178
|
+
validation = feedback.fit_examples("validation")
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
An in-process service can pass `data_processor=feedback` to `ACEWorkerRuntime`.
|
|
182
|
+
For a subprocess worker, provide an operator-owned factory function that
|
|
183
|
+
constructs this object from verified local dataset and corpus configuration:
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
python -m workers.ace.driver \
|
|
187
|
+
--checkout /opt/upstreams/ace \
|
|
188
|
+
--llm-config-json "$ACE_LLM_CONFIG" \
|
|
189
|
+
--task-processor-factory my_project.ir_feedback:make_feedback \
|
|
190
|
+
--task-processor-config-json '{"dataset_config":"/opt/data/ir.json"}'
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
The factory receives the startup JSON as keyword arguments. It must construct
|
|
194
|
+
the same feedback binding as the fit caller, perform no model calls, and expose
|
|
195
|
+
`answer_is_correct`, `evaluate_accuracy`, and `fingerprint`. Dataset/corpus
|
|
196
|
+
acquisition belongs before fitting. The driver records the factory module and
|
|
197
|
+
configuration hashes and the processor fingerprint; factory stdout is bounded
|
|
198
|
+
and excluded from JSONL. A processor may expose `validate_examples` to reject
|
|
199
|
+
altered or held-out fit inputs before any provider attempt. Requests cannot
|
|
200
|
+
select or change this factory. Put credentials only in the environment, never
|
|
201
|
+
in the factory configuration. Without these options the original worker and
|
|
202
|
+
its text-scoring behavior are unchanged.
|
|
203
|
+
|
|
204
|
+
The worker independently recomputes the split-audit hash and recipe request ID.
|
|
205
|
+
The request ID binds the complete canonical train and validation records,
|
|
206
|
+
including expected outputs and metadata, as well as controls, model
|
|
207
|
+
fingerprints, playbook hash, split-audit hash, and upstream commit. The worker
|
|
208
|
+
requires disjoint train/validation IDs and never accepts test examples in the
|
|
209
|
+
offline request.
|
|
210
|
+
|
|
211
|
+
## Upstream execution and playbook fidelity
|
|
212
|
+
|
|
213
|
+
For every request the worker:
|
|
214
|
+
|
|
215
|
+
1. renders native bullets as upstream lines of the form
|
|
216
|
+
`[id] helpful=X harmful=Y :: content`;
|
|
217
|
+
2. initializes the pinned `ACE` with the configured Generator, Reflector, and
|
|
218
|
+
Curator;
|
|
219
|
+
3. derives `next_global_id` with the pinned upstream helper;
|
|
220
|
+
4. calls `ACE.run(mode="offline")` once, with one epoch, curator frequency one,
|
|
221
|
+
and validation at the final training step;
|
|
222
|
+
5. observes, but does not reorder, the upstream Generator → Reflector → Curator
|
|
223
|
+
calls and their shared mutable playbook state;
|
|
224
|
+
6. parses upstream `best_playbook`, preserving existing IDs, content, counters,
|
|
225
|
+
and metadata;
|
|
226
|
+
7. emits replayable `add`, `edit`, and `delete` operations plus the exact
|
|
227
|
+
AgentInputLab `ACEPlaybook` candidate hash.
|
|
228
|
+
|
|
229
|
+
To remain compatible with the current diff protocol, structured order is
|
|
230
|
+
canonicalized as existing bullet order followed by upstream-created bullet
|
|
231
|
+
order. This does not change ACE's text supplied during execution; it only makes
|
|
232
|
+
the returned diff deterministic and exactly replayable by
|
|
233
|
+
`ACEWorkerResponse.from_mapping`.
|
|
234
|
+
|
|
235
|
+
Run artifacts and detailed upstream logs live in a per-request temporary
|
|
236
|
+
directory and are deleted after the response. Only validated usage totals and
|
|
237
|
+
the selected playbook cross the JSONL boundary.
|
|
238
|
+
|
|
239
|
+
## Current parity limits
|
|
240
|
+
|
|
241
|
+
- The pinned `ACE.run` exposes only its whole-run validation-best playbook, not
|
|
242
|
+
validation scores for every curator step. Therefore one worker `iteration`
|
|
243
|
+
represents one complete real upstream offline run. Internal sample-level
|
|
244
|
+
Generator/Reflector/Curator state transitions still execute unchanged.
|
|
245
|
+
- Request `max_rounds` maps to upstream `max_num_rounds`, i.e. reflection and
|
|
246
|
+
regeneration rounds per incorrect sample. It is not the number of curator
|
|
247
|
+
steps.
|
|
248
|
+
- At the pinned commit, upstream `apply_curator_operations` fully applies only
|
|
249
|
+
`ADD`; counter changes appear as replayable `edit` operations in the worker
|
|
250
|
+
response. The worker does not invent unsupported merge semantics.
|
|
251
|
+
- `max_model_calls` counts actual calls to the pinned components'
|
|
252
|
+
`api_client.chat.completions.create` seam. Retry attempts share the same
|
|
253
|
+
request counter and are rejected before the next provider attempt when the
|
|
254
|
+
ceiling is reached. Successful agent results must still expose token metadata
|
|
255
|
+
or the request fails rather than silently undercounting it.
|
|
256
|
+
- Token and cost ceilings are checked from provider-reported successful-call
|
|
257
|
+
usage and explicit startup pricing. A call that itself crosses a ceiling is
|
|
258
|
+
rejected after that call returns; the upstream API has no preflight token
|
|
259
|
+
estimate. Failed provider attempts count toward `max_model_calls`, but their
|
|
260
|
+
billable tokens or cost cannot be added unless the provider returns usage
|
|
261
|
+
metadata for that failed attempt.
|
workers/ace/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Pinned ACE JSONL worker."""
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "agentinputlab.ace.compatibility.v1",
|
|
3
|
+
"upstream": {
|
|
4
|
+
"repository": "https://github.com/ace-agent/ace.git",
|
|
5
|
+
"commit": "bcb7cea0504afad6f55fec4845dd4864c9f9eee7",
|
|
6
|
+
"license_path": "LICENSE.txt",
|
|
7
|
+
"license_spdx": "Apache-2.0",
|
|
8
|
+
"license_sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
|
9
|
+
"require_clean_checkout": true
|
|
10
|
+
},
|
|
11
|
+
"source_files": {
|
|
12
|
+
"pyproject.toml": "38856347e37fd50a80114e3f129a3767197e602e8d7b88ba87c7967dcd32ba09",
|
|
13
|
+
"ace/__init__.py": "5cffd7669d4b499d6e2de648cdc186297d7d0b663206fe4760d8a0ce64f39766",
|
|
14
|
+
"ace/ace.py": "70f7af3ff9088498afa8d6939c5e142cd8b0046179516ee336c57586b92ec914",
|
|
15
|
+
"ace/ace_batch.py": "6038bd54cc68597ed764e2dc5b011b4cb2523e31dbf942bcc7e1888597b29ff6",
|
|
16
|
+
"ace/core/__init__.py": "6dd2fc846149071e71a0b0f401c6ad703e6b9fde566b9c751bfc25dabc8c221f",
|
|
17
|
+
"ace/core/generator.py": "4b750a59752bd5bdb24b2f27b80938844d8a7b5d89e86d7d6bb726f3685c25f5",
|
|
18
|
+
"ace/core/reflector.py": "fe5ae8fb006fd16fdef7b8e7ad38dc746195a4bf0a366a82e69c473c9982b811",
|
|
19
|
+
"ace/core/curator.py": "863aa928b28538ded4b86ed330ce40c6012f6ca215f94618bef0615fb4de281b",
|
|
20
|
+
"ace/core/bulletpoint_analyzer.py": "c8348995c2e443a65343efb460939837655521d6962022d29aec1710947be4d2",
|
|
21
|
+
"ace/prompts/__init__.py": "803a8d38db890045fd344e36026a781947787e9b7d9d03e2d046faad33411a5b",
|
|
22
|
+
"ace/prompts/generator.py": "b8b12d4a6281ea4c422a2ee0bd919b50580d388027917e8740f82f97eb24eebb",
|
|
23
|
+
"ace/prompts/reflector.py": "72b343ce2781531dac2fd765543410b6abfc4817c39078dd4f83187c4d217e9c",
|
|
24
|
+
"ace/prompts/curator.py": "cca641bf025896b97f0daabf6208b110c081082930802e1f6d7ccf54fc90c9d7",
|
|
25
|
+
"llm.py": "cc615c5b26773fddc99c702bf2521a1c929e6363a905824b1be3e300337d1026",
|
|
26
|
+
"logger.py": "c54eda5fe3cdb8bc2cbef01be25f3f0a4eb6725123425c4a3674eed84d9cac23",
|
|
27
|
+
"utils.py": "7824aa262de2cd030b6914d1f7de698ec620dbec9c9e5f14ed1fe6322079570c",
|
|
28
|
+
"playbook_utils.py": "c67943f06cb0ee747be04d0599216e40d6c00e0cdf96ddc6af38fc26143e3a07"
|
|
29
|
+
},
|
|
30
|
+
"execution": {
|
|
31
|
+
"entrypoint": "ace.ace.ACE.run",
|
|
32
|
+
"mode": "offline",
|
|
33
|
+
"appworld_allowed": false,
|
|
34
|
+
"request_controlled_paths_allowed": false,
|
|
35
|
+
"request_controlled_imports_allowed": false,
|
|
36
|
+
"upstream_subprocess_allowed": false,
|
|
37
|
+
"verification_git_status_subprocess": true
|
|
38
|
+
}
|
|
39
|
+
}
|