agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,763 @@
|
|
|
1
|
+
"""Strict one-command DAG for the complete production experiment package."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import signal
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import Mapping, Sequence
|
|
14
|
+
from contextlib import suppress
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from types import MappingProxyType
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from agentinputlab.contracts.errors import ArtifactError, ValidationError
|
|
21
|
+
from agentinputlab.contracts.serialization import content_hash, freeze_mapping
|
|
22
|
+
from agentinputlab.experiments.secrets import (
|
|
23
|
+
environment_secret_values,
|
|
24
|
+
redact_secret_text,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
PRODUCTION_CAMPAIGN_PROTOCOL = "agentinputlab.production-campaign.v1"
|
|
28
|
+
|
|
29
|
+
_REQUIRED_STAGES = (
|
|
30
|
+
"data",
|
|
31
|
+
"method_parity",
|
|
32
|
+
"discovery",
|
|
33
|
+
"representative_freeze",
|
|
34
|
+
"family_confirmation",
|
|
35
|
+
"common_support",
|
|
36
|
+
"selector",
|
|
37
|
+
"robustness",
|
|
38
|
+
"stateful_replication",
|
|
39
|
+
"paper_export",
|
|
40
|
+
)
|
|
41
|
+
_ROOT_FIELDS = frozenset(
|
|
42
|
+
{
|
|
43
|
+
"schema_version",
|
|
44
|
+
"protocol",
|
|
45
|
+
"campaign_id",
|
|
46
|
+
"workspace",
|
|
47
|
+
"resume",
|
|
48
|
+
"steps",
|
|
49
|
+
"campaign_hash",
|
|
50
|
+
}
|
|
51
|
+
)
|
|
52
|
+
_STEP_FIELDS = frozenset(
|
|
53
|
+
{
|
|
54
|
+
"step_id",
|
|
55
|
+
"stage",
|
|
56
|
+
"requires",
|
|
57
|
+
"command",
|
|
58
|
+
"cwd",
|
|
59
|
+
"environment",
|
|
60
|
+
"pass_environment",
|
|
61
|
+
"inputs",
|
|
62
|
+
"outputs",
|
|
63
|
+
"timeout_seconds",
|
|
64
|
+
}
|
|
65
|
+
)
|
|
66
|
+
_INPUT_FIELDS = frozenset({"path", "sha256"})
|
|
67
|
+
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
68
|
+
_ENVIRONMENT = re.compile(r"^[A-Z_][A-Z0-9_]*$")
|
|
69
|
+
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
70
|
+
_SHA_TEMPLATE = re.compile(r"^\{sha256:(.+)\}$")
|
|
71
|
+
_REPOSITORY_COMMIT_ENV = "AILAB_REPOSITORY_COMMIT"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True, slots=True)
|
|
75
|
+
class ProductionInput:
|
|
76
|
+
path: str
|
|
77
|
+
sha256: str | None
|
|
78
|
+
|
|
79
|
+
def __post_init__(self) -> None:
|
|
80
|
+
if not isinstance(self.path, str) or not self.path.strip():
|
|
81
|
+
raise ValidationError("production campaign input path is invalid")
|
|
82
|
+
if self.sha256 is not None and (
|
|
83
|
+
not isinstance(self.sha256, str) or _SHA256.fullmatch(self.sha256) is None
|
|
84
|
+
):
|
|
85
|
+
raise ValidationError("production campaign input SHA-256 is invalid")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True, slots=True)
|
|
89
|
+
class ProductionStep:
|
|
90
|
+
step_id: str
|
|
91
|
+
stage: str
|
|
92
|
+
requires: tuple[str, ...]
|
|
93
|
+
command: tuple[str, ...]
|
|
94
|
+
cwd: str
|
|
95
|
+
environment: Mapping[str, str]
|
|
96
|
+
pass_environment: tuple[str, ...]
|
|
97
|
+
inputs: tuple[ProductionInput, ...]
|
|
98
|
+
outputs: tuple[str, ...]
|
|
99
|
+
timeout_seconds: float
|
|
100
|
+
|
|
101
|
+
def __post_init__(self) -> None:
|
|
102
|
+
if not isinstance(self.step_id, str) or _IDENTIFIER.fullmatch(self.step_id) is None:
|
|
103
|
+
raise ValidationError("production campaign step_id is invalid")
|
|
104
|
+
if self.stage not in _REQUIRED_STAGES:
|
|
105
|
+
raise ValidationError(f"unsupported production campaign stage: {self.stage}")
|
|
106
|
+
for label, values in (
|
|
107
|
+
("requires", self.requires),
|
|
108
|
+
("command", self.command),
|
|
109
|
+
("pass_environment", self.pass_environment),
|
|
110
|
+
("outputs", self.outputs),
|
|
111
|
+
):
|
|
112
|
+
if (
|
|
113
|
+
isinstance(values, (str, bytes))
|
|
114
|
+
or not isinstance(values, tuple)
|
|
115
|
+
or any(not isinstance(item, str) or not item for item in values)
|
|
116
|
+
):
|
|
117
|
+
raise ValidationError(f"production campaign step {self.step_id} {label} is invalid")
|
|
118
|
+
if not self.command or self.command[0] not in {"@python", "@agentinputlab"}:
|
|
119
|
+
raise ValidationError(
|
|
120
|
+
f"production campaign step {self.step_id} command must start "
|
|
121
|
+
"with @python or @agentinputlab"
|
|
122
|
+
)
|
|
123
|
+
if self.command[0] == "@python" and len(self.command) < 2:
|
|
124
|
+
raise ValidationError(f"production campaign step {self.step_id} has no Python script")
|
|
125
|
+
if not isinstance(self.cwd, str) or not self.cwd:
|
|
126
|
+
raise ValidationError(f"production campaign step {self.step_id} cwd is invalid")
|
|
127
|
+
if not isinstance(self.environment, Mapping) or any(
|
|
128
|
+
not isinstance(name, str)
|
|
129
|
+
or _ENVIRONMENT.fullmatch(name) is None
|
|
130
|
+
or not isinstance(value, str)
|
|
131
|
+
or not value
|
|
132
|
+
for name, value in self.environment.items()
|
|
133
|
+
):
|
|
134
|
+
raise ValidationError(f"production campaign step {self.step_id} environment is invalid")
|
|
135
|
+
if len(set(self.requires)) != len(self.requires):
|
|
136
|
+
raise ValidationError(f"production campaign step {self.step_id} repeats dependencies")
|
|
137
|
+
if len(set(self.pass_environment)) != len(self.pass_environment) or any(
|
|
138
|
+
_ENVIRONMENT.fullmatch(name) is None for name in self.pass_environment
|
|
139
|
+
):
|
|
140
|
+
raise ValidationError(
|
|
141
|
+
f"production campaign step {self.step_id} environment allowlist is invalid"
|
|
142
|
+
)
|
|
143
|
+
if len(set(self.outputs)) != len(self.outputs):
|
|
144
|
+
raise ValidationError(f"production campaign step {self.step_id} repeats outputs")
|
|
145
|
+
if (
|
|
146
|
+
isinstance(self.timeout_seconds, bool)
|
|
147
|
+
or not isinstance(self.timeout_seconds, (int, float))
|
|
148
|
+
or self.timeout_seconds <= 0
|
|
149
|
+
):
|
|
150
|
+
raise ValidationError(f"production campaign step {self.step_id} timeout is invalid")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True, slots=True)
|
|
154
|
+
class ProductionCampaign:
|
|
155
|
+
campaign_id: str
|
|
156
|
+
workspace: Path
|
|
157
|
+
resume: bool
|
|
158
|
+
steps: tuple[ProductionStep, ...]
|
|
159
|
+
campaign_hash: str
|
|
160
|
+
source_path: Path
|
|
161
|
+
source_sha256: str
|
|
162
|
+
|
|
163
|
+
def __post_init__(self) -> None:
|
|
164
|
+
if not isinstance(self.campaign_id, str) or _IDENTIFIER.fullmatch(self.campaign_id) is None:
|
|
165
|
+
raise ValidationError("production campaign_id is invalid")
|
|
166
|
+
if not isinstance(self.workspace, Path) or not self.workspace.is_absolute():
|
|
167
|
+
raise ValidationError("production campaign workspace must be absolute")
|
|
168
|
+
if not isinstance(self.resume, bool):
|
|
169
|
+
raise ValidationError("production campaign resume must be boolean")
|
|
170
|
+
if not self.steps:
|
|
171
|
+
raise ValidationError("production campaign requires steps")
|
|
172
|
+
identifiers = tuple(step.step_id for step in self.steps)
|
|
173
|
+
if len(set(identifiers)) != len(identifiers):
|
|
174
|
+
raise ValidationError("production campaign step IDs must be unique")
|
|
175
|
+
seen: set[str] = set()
|
|
176
|
+
for step in self.steps:
|
|
177
|
+
unknown = set(step.requires).difference(seen)
|
|
178
|
+
if unknown:
|
|
179
|
+
raise ValidationError(
|
|
180
|
+
f"step {step.step_id} depends on unknown or later steps: "
|
|
181
|
+
+ ", ".join(sorted(unknown))
|
|
182
|
+
)
|
|
183
|
+
seen.add(step.step_id)
|
|
184
|
+
observed_stages = tuple(dict.fromkeys(step.stage for step in self.steps))
|
|
185
|
+
if observed_stages != _REQUIRED_STAGES:
|
|
186
|
+
raise ValidationError(
|
|
187
|
+
"production campaign must cover the exact ordered stages: "
|
|
188
|
+
+ ", ".join(_REQUIRED_STAGES)
|
|
189
|
+
)
|
|
190
|
+
outputs: dict[str, str] = {}
|
|
191
|
+
for step in self.steps:
|
|
192
|
+
for path in step.outputs:
|
|
193
|
+
if path in outputs:
|
|
194
|
+
raise ValidationError(f"production output {path!r} has multiple producers")
|
|
195
|
+
outputs[path] = step.step_id
|
|
196
|
+
if _SHA256.fullmatch(self.campaign_hash) is None:
|
|
197
|
+
raise ValidationError("production campaign_hash is invalid")
|
|
198
|
+
if _SHA256.fullmatch(self.source_sha256) is None:
|
|
199
|
+
raise ValidationError("production campaign source_sha256 is invalid")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def production_campaign_hash(value: Mapping[str, Any]) -> str:
|
|
203
|
+
"""Hash a campaign before the self-hash field is populated."""
|
|
204
|
+
|
|
205
|
+
return content_hash({key: item for key, item in value.items() if key != "campaign_hash"})
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def load_production_campaign(path: Path | str) -> ProductionCampaign:
|
|
209
|
+
source = Path(path).expanduser()
|
|
210
|
+
if source.is_symlink():
|
|
211
|
+
raise ArtifactError("production campaign file must not be a symlink")
|
|
212
|
+
source = source.resolve()
|
|
213
|
+
try:
|
|
214
|
+
raw = source.read_bytes()
|
|
215
|
+
value = json.loads(raw.decode("utf-8"))
|
|
216
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
217
|
+
raise ArtifactError("cannot load production campaign JSON") from exc
|
|
218
|
+
if not isinstance(value, Mapping) or set(value) != _ROOT_FIELDS:
|
|
219
|
+
raise ValidationError("production campaign fields are invalid")
|
|
220
|
+
if value["schema_version"] != 1 or value["protocol"] != PRODUCTION_CAMPAIGN_PROTOCOL:
|
|
221
|
+
raise ValidationError("unsupported production campaign protocol")
|
|
222
|
+
if value["campaign_hash"] != production_campaign_hash(value):
|
|
223
|
+
raise ArtifactError("production campaign self-hash differs")
|
|
224
|
+
raw_steps = value["steps"]
|
|
225
|
+
if isinstance(raw_steps, (str, bytes)) or not isinstance(raw_steps, Sequence):
|
|
226
|
+
raise ValidationError("production campaign steps must be an array")
|
|
227
|
+
steps: list[ProductionStep] = []
|
|
228
|
+
for index, raw_step in enumerate(raw_steps):
|
|
229
|
+
if not isinstance(raw_step, Mapping) or set(raw_step) != _STEP_FIELDS:
|
|
230
|
+
raise ValidationError(f"production step {index} fields are invalid")
|
|
231
|
+
inputs: list[ProductionInput] = []
|
|
232
|
+
raw_inputs = raw_step["inputs"]
|
|
233
|
+
if isinstance(raw_inputs, (str, bytes)) or not isinstance(raw_inputs, Sequence):
|
|
234
|
+
raise ValidationError(f"production step {index} inputs are invalid")
|
|
235
|
+
for raw_input in raw_inputs:
|
|
236
|
+
if not isinstance(raw_input, Mapping) or set(raw_input) != _INPUT_FIELDS:
|
|
237
|
+
raise ValidationError(f"production step {index} input fields are invalid")
|
|
238
|
+
inputs.append(
|
|
239
|
+
ProductionInput(
|
|
240
|
+
path=raw_input["path"],
|
|
241
|
+
sha256=raw_input["sha256"],
|
|
242
|
+
)
|
|
243
|
+
)
|
|
244
|
+
steps.append(
|
|
245
|
+
ProductionStep(
|
|
246
|
+
step_id=raw_step["step_id"],
|
|
247
|
+
stage=raw_step["stage"],
|
|
248
|
+
requires=tuple(raw_step["requires"]),
|
|
249
|
+
command=tuple(raw_step["command"]),
|
|
250
|
+
cwd=raw_step["cwd"],
|
|
251
|
+
environment=freeze_mapping(raw_step["environment"]),
|
|
252
|
+
pass_environment=tuple(raw_step["pass_environment"]),
|
|
253
|
+
inputs=tuple(inputs),
|
|
254
|
+
outputs=tuple(raw_step["outputs"]),
|
|
255
|
+
timeout_seconds=raw_step["timeout_seconds"],
|
|
256
|
+
)
|
|
257
|
+
)
|
|
258
|
+
return ProductionCampaign(
|
|
259
|
+
campaign_id=value["campaign_id"],
|
|
260
|
+
workspace=Path(value["workspace"]).expanduser().resolve(),
|
|
261
|
+
resume=value["resume"],
|
|
262
|
+
steps=tuple(steps),
|
|
263
|
+
campaign_hash=value["campaign_hash"],
|
|
264
|
+
source_path=source,
|
|
265
|
+
source_sha256=hashlib.sha256(raw).hexdigest(),
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def run_production_campaign(
|
|
270
|
+
campaign: ProductionCampaign,
|
|
271
|
+
*,
|
|
272
|
+
execute: bool,
|
|
273
|
+
) -> Mapping[str, Any]:
|
|
274
|
+
"""Preflight the full DAG, then optionally execute it in topological order."""
|
|
275
|
+
|
|
276
|
+
repository = Path(__file__).resolve().parents[2]
|
|
277
|
+
producers = {output: step.step_id for step in campaign.steps for output in step.outputs}
|
|
278
|
+
blockers: dict[str, tuple[str, ...]] = {}
|
|
279
|
+
for step in campaign.steps:
|
|
280
|
+
messages = _step_preflight(
|
|
281
|
+
step,
|
|
282
|
+
campaign=campaign,
|
|
283
|
+
repository=repository,
|
|
284
|
+
producers=producers,
|
|
285
|
+
execution_time=False,
|
|
286
|
+
)
|
|
287
|
+
if messages:
|
|
288
|
+
blockers[step.step_id] = messages
|
|
289
|
+
ready = not blockers
|
|
290
|
+
if not execute:
|
|
291
|
+
return freeze_mapping(
|
|
292
|
+
{
|
|
293
|
+
"schema_version": 1,
|
|
294
|
+
"protocol": "agentinputlab.production-campaign-result.v1",
|
|
295
|
+
"campaign_id": campaign.campaign_id,
|
|
296
|
+
"campaign_hash": campaign.campaign_hash,
|
|
297
|
+
"ready": ready,
|
|
298
|
+
"executed": False,
|
|
299
|
+
"required_stages": _REQUIRED_STAGES,
|
|
300
|
+
"step_count": len(campaign.steps),
|
|
301
|
+
"blockers": blockers,
|
|
302
|
+
}
|
|
303
|
+
)
|
|
304
|
+
if not ready:
|
|
305
|
+
return freeze_mapping(
|
|
306
|
+
{
|
|
307
|
+
"schema_version": 1,
|
|
308
|
+
"protocol": "agentinputlab.production-campaign-result.v1",
|
|
309
|
+
"campaign_id": campaign.campaign_id,
|
|
310
|
+
"campaign_hash": campaign.campaign_hash,
|
|
311
|
+
"ready": False,
|
|
312
|
+
"executed": False,
|
|
313
|
+
"execution_blocked": True,
|
|
314
|
+
"required_stages": _REQUIRED_STAGES,
|
|
315
|
+
"step_count": len(campaign.steps),
|
|
316
|
+
"blockers": blockers,
|
|
317
|
+
}
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
campaign.workspace.mkdir(parents=True, exist_ok=True)
|
|
321
|
+
receipt_root = campaign.workspace / ".agentinputlab" / "receipts"
|
|
322
|
+
log_root = campaign.workspace / ".agentinputlab" / "logs"
|
|
323
|
+
receipt_root.mkdir(parents=True, exist_ok=True)
|
|
324
|
+
log_root.mkdir(parents=True, exist_ok=True)
|
|
325
|
+
results: list[Mapping[str, Any]] = []
|
|
326
|
+
for step in campaign.steps:
|
|
327
|
+
receipt_path = receipt_root / f"{step.step_id}.json"
|
|
328
|
+
if campaign.resume and _completed_receipt(
|
|
329
|
+
receipt_path,
|
|
330
|
+
step=step,
|
|
331
|
+
campaign=campaign,
|
|
332
|
+
repository=repository,
|
|
333
|
+
):
|
|
334
|
+
results.append(
|
|
335
|
+
{
|
|
336
|
+
"step_id": step.step_id,
|
|
337
|
+
"stage": step.stage,
|
|
338
|
+
"status": "resumed",
|
|
339
|
+
"receipt": str(receipt_path),
|
|
340
|
+
}
|
|
341
|
+
)
|
|
342
|
+
continue
|
|
343
|
+
runtime_blockers = _step_preflight(
|
|
344
|
+
step,
|
|
345
|
+
campaign=campaign,
|
|
346
|
+
repository=repository,
|
|
347
|
+
producers=producers,
|
|
348
|
+
execution_time=True,
|
|
349
|
+
)
|
|
350
|
+
if runtime_blockers:
|
|
351
|
+
results.append(
|
|
352
|
+
{
|
|
353
|
+
"step_id": step.step_id,
|
|
354
|
+
"stage": step.stage,
|
|
355
|
+
"status": "blocked",
|
|
356
|
+
"blockers": runtime_blockers,
|
|
357
|
+
}
|
|
358
|
+
)
|
|
359
|
+
break
|
|
360
|
+
command = _command(
|
|
361
|
+
step.command,
|
|
362
|
+
campaign=campaign,
|
|
363
|
+
repository=repository,
|
|
364
|
+
)
|
|
365
|
+
cwd = _resolved_path(
|
|
366
|
+
step.cwd,
|
|
367
|
+
campaign=campaign,
|
|
368
|
+
repository=repository,
|
|
369
|
+
)
|
|
370
|
+
environment = {
|
|
371
|
+
"PATH": os.environ.get("PATH", ""),
|
|
372
|
+
"PYTHONPATH": os.pathsep.join((str(repository / "src"), str(repository))),
|
|
373
|
+
"PYTHONUNBUFFERED": "1",
|
|
374
|
+
}
|
|
375
|
+
for name, value in step.environment.items():
|
|
376
|
+
environment[name] = _expand_argument(
|
|
377
|
+
value,
|
|
378
|
+
campaign=campaign,
|
|
379
|
+
repository=repository,
|
|
380
|
+
)
|
|
381
|
+
for name in step.pass_environment:
|
|
382
|
+
environment[name] = os.environ[name]
|
|
383
|
+
secrets = environment_secret_values(
|
|
384
|
+
environment,
|
|
385
|
+
step.pass_environment,
|
|
386
|
+
)
|
|
387
|
+
attempt_id = f"{time.time_ns()}-{os.getpid()}"
|
|
388
|
+
started = time.time()
|
|
389
|
+
return_code, raw_stdout, raw_stderr, timed_out = _run_command(
|
|
390
|
+
command,
|
|
391
|
+
cwd=cwd,
|
|
392
|
+
env=environment,
|
|
393
|
+
timeout_seconds=step.timeout_seconds,
|
|
394
|
+
)
|
|
395
|
+
duration = time.time() - started
|
|
396
|
+
stdout = redact_secret_text(raw_stdout, secret_values=secrets)
|
|
397
|
+
stderr = redact_secret_text(raw_stderr, secret_values=secrets)
|
|
398
|
+
stdout_path = log_root / f"{step.step_id}.{attempt_id}.stdout.log"
|
|
399
|
+
stderr_path = log_root / f"{step.step_id}.{attempt_id}.stderr.log"
|
|
400
|
+
stdout_path.write_text(stdout, encoding="utf-8")
|
|
401
|
+
stderr_path.write_text(stderr, encoding="utf-8")
|
|
402
|
+
output_hashes = _output_hashes(
|
|
403
|
+
step,
|
|
404
|
+
campaign=campaign,
|
|
405
|
+
repository=repository,
|
|
406
|
+
)
|
|
407
|
+
input_hashes = _input_hashes(
|
|
408
|
+
step,
|
|
409
|
+
campaign=campaign,
|
|
410
|
+
repository=repository,
|
|
411
|
+
)
|
|
412
|
+
status = (
|
|
413
|
+
"completed"
|
|
414
|
+
if return_code == 0 and not timed_out and len(output_hashes) == len(step.outputs)
|
|
415
|
+
else ("timed_out" if timed_out else "failed")
|
|
416
|
+
)
|
|
417
|
+
receipt = {
|
|
418
|
+
"schema_version": 1,
|
|
419
|
+
"protocol": "agentinputlab.production-step-receipt.v1",
|
|
420
|
+
"campaign_id": campaign.campaign_id,
|
|
421
|
+
"campaign_hash": campaign.campaign_hash,
|
|
422
|
+
"step_id": step.step_id,
|
|
423
|
+
"stage": step.stage,
|
|
424
|
+
"status": status,
|
|
425
|
+
"command_hash": content_hash(command),
|
|
426
|
+
"return_code": return_code,
|
|
427
|
+
"timed_out": timed_out,
|
|
428
|
+
"timeout_seconds": step.timeout_seconds,
|
|
429
|
+
"duration_seconds": duration,
|
|
430
|
+
"inputs": dict(input_hashes),
|
|
431
|
+
"outputs": dict(output_hashes),
|
|
432
|
+
"stdout_log": str(stdout_path),
|
|
433
|
+
"stderr_log": str(stderr_path),
|
|
434
|
+
}
|
|
435
|
+
_archive_receipt(receipt_path, receipt_root=receipt_root)
|
|
436
|
+
_write_atomic(receipt_path, receipt)
|
|
437
|
+
results.append(
|
|
438
|
+
{
|
|
439
|
+
"step_id": step.step_id,
|
|
440
|
+
"stage": step.stage,
|
|
441
|
+
"status": status,
|
|
442
|
+
"return_code": return_code,
|
|
443
|
+
"receipt": str(receipt_path),
|
|
444
|
+
}
|
|
445
|
+
)
|
|
446
|
+
if status != "completed":
|
|
447
|
+
break
|
|
448
|
+
all_succeeded = len(results) == len(campaign.steps) and all(
|
|
449
|
+
item["status"] in {"completed", "resumed"} for item in results
|
|
450
|
+
)
|
|
451
|
+
return freeze_mapping(
|
|
452
|
+
{
|
|
453
|
+
"schema_version": 1,
|
|
454
|
+
"protocol": "agentinputlab.production-campaign-result.v1",
|
|
455
|
+
"campaign_id": campaign.campaign_id,
|
|
456
|
+
"campaign_hash": campaign.campaign_hash,
|
|
457
|
+
"ready": True,
|
|
458
|
+
"executed": True,
|
|
459
|
+
"all_succeeded": all_succeeded,
|
|
460
|
+
"required_stages": _REQUIRED_STAGES,
|
|
461
|
+
"steps": results,
|
|
462
|
+
}
|
|
463
|
+
)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _step_preflight(
|
|
467
|
+
step: ProductionStep,
|
|
468
|
+
*,
|
|
469
|
+
campaign: ProductionCampaign,
|
|
470
|
+
repository: Path,
|
|
471
|
+
producers: Mapping[str, str],
|
|
472
|
+
execution_time: bool,
|
|
473
|
+
) -> tuple[str, ...]:
|
|
474
|
+
blockers: list[str] = []
|
|
475
|
+
try:
|
|
476
|
+
_command(
|
|
477
|
+
step.command,
|
|
478
|
+
campaign=campaign,
|
|
479
|
+
repository=repository,
|
|
480
|
+
allow_missing_hash=not execution_time,
|
|
481
|
+
)
|
|
482
|
+
cwd = _resolved_path(
|
|
483
|
+
step.cwd,
|
|
484
|
+
campaign=campaign,
|
|
485
|
+
repository=repository,
|
|
486
|
+
)
|
|
487
|
+
if not cwd.is_dir():
|
|
488
|
+
blockers.append(f"cwd is absent: {cwd}")
|
|
489
|
+
except (ArtifactError, ValidationError) as exc:
|
|
490
|
+
blockers.append(str(exc))
|
|
491
|
+
missing_environment = [name for name in step.pass_environment if not os.environ.get(name)]
|
|
492
|
+
if missing_environment:
|
|
493
|
+
blockers.append("missing environment: " + ", ".join(sorted(missing_environment)))
|
|
494
|
+
for name, value in step.environment.items():
|
|
495
|
+
try:
|
|
496
|
+
_expand_argument(
|
|
497
|
+
value,
|
|
498
|
+
campaign=campaign,
|
|
499
|
+
repository=repository,
|
|
500
|
+
allow_missing_hash=not execution_time,
|
|
501
|
+
)
|
|
502
|
+
except (ArtifactError, ValidationError) as exc:
|
|
503
|
+
blockers.append(f"environment {name}: {exc}")
|
|
504
|
+
expected_commit = step.environment.get(_REPOSITORY_COMMIT_ENV)
|
|
505
|
+
if expected_commit is not None:
|
|
506
|
+
try:
|
|
507
|
+
observed_commit = subprocess.run(
|
|
508
|
+
("git", "rev-parse", "HEAD"),
|
|
509
|
+
cwd=repository,
|
|
510
|
+
check=True,
|
|
511
|
+
capture_output=True,
|
|
512
|
+
text=True,
|
|
513
|
+
).stdout.strip()
|
|
514
|
+
except (OSError, subprocess.CalledProcessError):
|
|
515
|
+
blockers.append("cannot resolve the executing repository commit")
|
|
516
|
+
else:
|
|
517
|
+
if observed_commit != expected_commit:
|
|
518
|
+
blockers.append(
|
|
519
|
+
"repository commit differs: "
|
|
520
|
+
f"expected {expected_commit}, observed {observed_commit}"
|
|
521
|
+
)
|
|
522
|
+
for item in step.inputs:
|
|
523
|
+
path = _resolved_path(
|
|
524
|
+
item.path,
|
|
525
|
+
campaign=campaign,
|
|
526
|
+
repository=repository,
|
|
527
|
+
)
|
|
528
|
+
if not path.is_file():
|
|
529
|
+
producer = producers.get(item.path)
|
|
530
|
+
if not execution_time and producer in step.requires:
|
|
531
|
+
continue
|
|
532
|
+
blockers.append(f"input is absent: {path}")
|
|
533
|
+
continue
|
|
534
|
+
if item.sha256 is not None and _sha256(path) != item.sha256:
|
|
535
|
+
blockers.append(f"input checksum differs: {path}")
|
|
536
|
+
return tuple(blockers)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _command(
|
|
540
|
+
raw: Sequence[str],
|
|
541
|
+
*,
|
|
542
|
+
campaign: ProductionCampaign,
|
|
543
|
+
repository: Path,
|
|
544
|
+
allow_missing_hash: bool = False,
|
|
545
|
+
) -> tuple[str, ...]:
|
|
546
|
+
expanded = tuple(
|
|
547
|
+
_expand_argument(
|
|
548
|
+
item,
|
|
549
|
+
campaign=campaign,
|
|
550
|
+
repository=repository,
|
|
551
|
+
allow_missing_hash=allow_missing_hash,
|
|
552
|
+
)
|
|
553
|
+
for item in raw
|
|
554
|
+
)
|
|
555
|
+
if expanded[0] == "@agentinputlab":
|
|
556
|
+
return (sys.executable, "-m", "agentinputlab.cli.main", *expanded[1:])
|
|
557
|
+
script = Path(expanded[1])
|
|
558
|
+
if not script.is_absolute():
|
|
559
|
+
script = (repository / script).resolve()
|
|
560
|
+
allowed_roots = (
|
|
561
|
+
repository / "scripts" / "experiments",
|
|
562
|
+
repository / "doc" / "paper_kdd2027" / "scripts",
|
|
563
|
+
)
|
|
564
|
+
if not script.is_file() or not any(script.is_relative_to(root) for root in allowed_roots):
|
|
565
|
+
raise ArtifactError(
|
|
566
|
+
"production Python commands are restricted to reviewed experiment and paper scripts"
|
|
567
|
+
)
|
|
568
|
+
return (sys.executable, str(script), *expanded[2:])
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _expand_argument(
|
|
572
|
+
value: str,
|
|
573
|
+
*,
|
|
574
|
+
campaign: ProductionCampaign,
|
|
575
|
+
repository: Path,
|
|
576
|
+
allow_missing_hash: bool = False,
|
|
577
|
+
) -> str:
|
|
578
|
+
match = _SHA_TEMPLATE.fullmatch(value)
|
|
579
|
+
if match is not None:
|
|
580
|
+
path = _resolved_path(
|
|
581
|
+
match.group(1),
|
|
582
|
+
campaign=campaign,
|
|
583
|
+
repository=repository,
|
|
584
|
+
)
|
|
585
|
+
if not path.is_file():
|
|
586
|
+
if allow_missing_hash:
|
|
587
|
+
return "0" * 64
|
|
588
|
+
raise ArtifactError(f"cannot hash absent generated artifact: {path}")
|
|
589
|
+
return _sha256(path)
|
|
590
|
+
return value.replace("{repo}", str(repository)).replace(
|
|
591
|
+
"{workspace}",
|
|
592
|
+
str(campaign.workspace),
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _resolved_path(
|
|
597
|
+
value: str,
|
|
598
|
+
*,
|
|
599
|
+
campaign: ProductionCampaign,
|
|
600
|
+
repository: Path,
|
|
601
|
+
) -> Path:
|
|
602
|
+
expanded = value.replace("{repo}", str(repository)).replace(
|
|
603
|
+
"{workspace}",
|
|
604
|
+
str(campaign.workspace),
|
|
605
|
+
)
|
|
606
|
+
path = Path(expanded).expanduser()
|
|
607
|
+
if not path.is_absolute():
|
|
608
|
+
path = repository / path
|
|
609
|
+
return path.resolve()
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def _sha256(path: Path) -> str:
|
|
613
|
+
digest = hashlib.sha256()
|
|
614
|
+
with path.open("rb") as handle:
|
|
615
|
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
616
|
+
digest.update(block)
|
|
617
|
+
return digest.hexdigest()
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _output_hashes(
|
|
621
|
+
step: ProductionStep,
|
|
622
|
+
*,
|
|
623
|
+
campaign: ProductionCampaign,
|
|
624
|
+
repository: Path,
|
|
625
|
+
) -> Mapping[str, str]:
|
|
626
|
+
result: dict[str, str] = {}
|
|
627
|
+
for raw in step.outputs:
|
|
628
|
+
path = _resolved_path(raw, campaign=campaign, repository=repository)
|
|
629
|
+
if path.is_file():
|
|
630
|
+
result[raw] = _sha256(path)
|
|
631
|
+
elif path.is_dir():
|
|
632
|
+
result[raw] = content_hash(
|
|
633
|
+
{
|
|
634
|
+
str(item.relative_to(path)): _sha256(item)
|
|
635
|
+
for item in sorted(path.rglob("*"))
|
|
636
|
+
if item.is_file()
|
|
637
|
+
}
|
|
638
|
+
)
|
|
639
|
+
return MappingProxyType(result)
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def _input_hashes(
|
|
643
|
+
step: ProductionStep,
|
|
644
|
+
*,
|
|
645
|
+
campaign: ProductionCampaign,
|
|
646
|
+
repository: Path,
|
|
647
|
+
) -> Mapping[str, str]:
|
|
648
|
+
"""Return the exact input identities consumed by one production step.
|
|
649
|
+
|
|
650
|
+
Inputs are recorded independently of the optional checksum pinned in the
|
|
651
|
+
campaign document. Generated upstream artifacts normally cannot have a
|
|
652
|
+
checksum before execution, but their observed bytes must still be bound to
|
|
653
|
+
the downstream receipt so that resume cannot reuse stale derived outputs.
|
|
654
|
+
"""
|
|
655
|
+
|
|
656
|
+
result: dict[str, str] = {}
|
|
657
|
+
for item in step.inputs:
|
|
658
|
+
path = _resolved_path(item.path, campaign=campaign, repository=repository)
|
|
659
|
+
if path.is_file():
|
|
660
|
+
result[item.path] = _sha256(path)
|
|
661
|
+
return MappingProxyType(result)
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _completed_receipt(
|
|
665
|
+
path: Path,
|
|
666
|
+
*,
|
|
667
|
+
step: ProductionStep,
|
|
668
|
+
campaign: ProductionCampaign,
|
|
669
|
+
repository: Path,
|
|
670
|
+
) -> bool:
|
|
671
|
+
if not path.is_file():
|
|
672
|
+
return False
|
|
673
|
+
try:
|
|
674
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
675
|
+
except (OSError, UnicodeError, json.JSONDecodeError):
|
|
676
|
+
return False
|
|
677
|
+
if (
|
|
678
|
+
not isinstance(value, Mapping)
|
|
679
|
+
or value.get("campaign_hash") != campaign.campaign_hash
|
|
680
|
+
or value.get("step_id") != step.step_id
|
|
681
|
+
or value.get("status") != "completed"
|
|
682
|
+
):
|
|
683
|
+
return False
|
|
684
|
+
observed = _output_hashes(
|
|
685
|
+
step,
|
|
686
|
+
campaign=campaign,
|
|
687
|
+
repository=repository,
|
|
688
|
+
)
|
|
689
|
+
observed_inputs = _input_hashes(
|
|
690
|
+
step,
|
|
691
|
+
campaign=campaign,
|
|
692
|
+
repository=repository,
|
|
693
|
+
)
|
|
694
|
+
return (
|
|
695
|
+
dict(observed) == value.get("outputs")
|
|
696
|
+
and len(observed) == len(step.outputs)
|
|
697
|
+
and dict(observed_inputs) == value.get("inputs")
|
|
698
|
+
and len(observed_inputs) == len(step.inputs)
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def _write_atomic(path: Path, value: Mapping[str, Any]) -> None:
|
|
703
|
+
encoded = (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode(
|
|
704
|
+
"utf-8"
|
|
705
|
+
)
|
|
706
|
+
temporary = path.with_name(f".{path.name}.tmp")
|
|
707
|
+
temporary.write_bytes(encoded)
|
|
708
|
+
temporary.replace(path)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def _archive_receipt(path: Path, *, receipt_root: Path) -> None:
|
|
712
|
+
if not path.is_file():
|
|
713
|
+
return
|
|
714
|
+
raw = path.read_bytes()
|
|
715
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
716
|
+
history = receipt_root / "history" / path.stem
|
|
717
|
+
history.mkdir(parents=True, exist_ok=True)
|
|
718
|
+
destination = history / f"{time.time_ns()}-{digest}.json"
|
|
719
|
+
if destination.exists():
|
|
720
|
+
raise ArtifactError("production receipt archive collision")
|
|
721
|
+
path.replace(destination)
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def _run_command(
|
|
725
|
+
command: Sequence[str],
|
|
726
|
+
*,
|
|
727
|
+
cwd: Path,
|
|
728
|
+
env: Mapping[str, str],
|
|
729
|
+
timeout_seconds: float,
|
|
730
|
+
) -> tuple[int, str, str, bool]:
|
|
731
|
+
process = subprocess.Popen(
|
|
732
|
+
command,
|
|
733
|
+
cwd=cwd,
|
|
734
|
+
env=dict(env),
|
|
735
|
+
stdout=subprocess.PIPE,
|
|
736
|
+
stderr=subprocess.PIPE,
|
|
737
|
+
text=True,
|
|
738
|
+
start_new_session=True,
|
|
739
|
+
)
|
|
740
|
+
try:
|
|
741
|
+
stdout, stderr = process.communicate(timeout=timeout_seconds)
|
|
742
|
+
return process.returncode, stdout, stderr, False
|
|
743
|
+
except subprocess.TimeoutExpired:
|
|
744
|
+
with suppress(ProcessLookupError):
|
|
745
|
+
os.killpg(process.pid, signal.SIGTERM)
|
|
746
|
+
try:
|
|
747
|
+
stdout, stderr = process.communicate(timeout=5)
|
|
748
|
+
except subprocess.TimeoutExpired:
|
|
749
|
+
with suppress(ProcessLookupError):
|
|
750
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
751
|
+
stdout, stderr = process.communicate()
|
|
752
|
+
return -signal.SIGTERM, stdout, stderr, True
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
__all__ = [
|
|
756
|
+
"PRODUCTION_CAMPAIGN_PROTOCOL",
|
|
757
|
+
"ProductionCampaign",
|
|
758
|
+
"ProductionInput",
|
|
759
|
+
"ProductionStep",
|
|
760
|
+
"load_production_campaign",
|
|
761
|
+
"production_campaign_hash",
|
|
762
|
+
"run_production_campaign",
|
|
763
|
+
]
|