dsh-operator-agent 0.7.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsh_operator_agent-0.7.2.dist-info/METADATA +241 -0
- dsh_operator_agent-0.7.2.dist-info/RECORD +155 -0
- dsh_operator_agent-0.7.2.dist-info/WHEEL +5 -0
- dsh_operator_agent-0.7.2.dist-info/entry_points.txt +3 -0
- dsh_operator_agent-0.7.2.dist-info/licenses/LICENSE +201 -0
- dsh_operator_agent-0.7.2.dist-info/licenses/NOTICE +7 -0
- dsh_operator_agent-0.7.2.dist-info/top_level.txt +2 -0
- operator_agent/__init__.py +43 -0
- operator_agent/__main__.py +12 -0
- operator_agent/app/__init__.py +1 -0
- operator_agent/app/__main__.py +8 -0
- operator_agent/app/_harness_controls.py +21 -0
- operator_agent/app/autonomous.py +345 -0
- operator_agent/app/cli.py +321 -0
- operator_agent/app/conductor.py +4050 -0
- operator_agent/app/hitl_tools.py +1403 -0
- operator_agent/app/host_workflow.py +196 -0
- operator_agent/app/intent_parser.py +298 -0
- operator_agent/app/mcp_server.py +810 -0
- operator_agent/app/tool_calls.py +331 -0
- operator_agent/core/__init__.py +1 -0
- operator_agent/core/achievable.py +186 -0
- operator_agent/core/artifact_store.py +296 -0
- operator_agent/core/backend_caps.py +318 -0
- operator_agent/core/candidates.py +723 -0
- operator_agent/core/confirm.py +215 -0
- operator_agent/core/contract.py +625 -0
- operator_agent/core/corpus_admission.py +225 -0
- operator_agent/core/corpus_journal.py +253 -0
- operator_agent/core/environment.py +306 -0
- operator_agent/core/execution_audit.py +66 -0
- operator_agent/core/hardware_caps.py +641 -0
- operator_agent/core/human_ledger.py +157 -0
- operator_agent/core/interface_contract.py +241 -0
- operator_agent/core/kernel_db.py +992 -0
- operator_agent/core/kernel_templates.py +122 -0
- operator_agent/core/novelty.py +153 -0
- operator_agent/core/report.py +709 -0
- operator_agent/core/rule_templates.py +30 -0
- operator_agent/core/run_contract.py +278 -0
- operator_agent/core/run_layout.py +446 -0
- operator_agent/core/schemas.py +765 -0
- operator_agent/core/source_lang.py +154 -0
- operator_agent/core/tolerance.py +333 -0
- operator_agent/core/workload.py +168 -0
- operator_agent/decide/__init__.py +1 -0
- operator_agent/decide/anticheat.py +569 -0
- operator_agent/decide/bench_cases.py +361 -0
- operator_agent/decide/builtin_layernorm.py +13 -0
- operator_agent/decide/calibration.py +344 -0
- operator_agent/decide/compile_evidence.py +188 -0
- operator_agent/decide/correctness.py +240 -0
- operator_agent/decide/correctness_boundaries.py +344 -0
- operator_agent/decide/correctness_profiles.py +170 -0
- operator_agent/decide/cuda_calibration.py +329 -0
- operator_agent/decide/cuda_ruler_profile.py +151 -0
- operator_agent/decide/eligibility.py +837 -0
- operator_agent/decide/escalation.py +245 -0
- operator_agent/decide/exhaustion.py +134 -0
- operator_agent/decide/failure_route.py +211 -0
- operator_agent/decide/input_fidelity.py +188 -0
- operator_agent/decide/measurement.py +1449 -0
- operator_agent/decide/oracle.py +461 -0
- operator_agent/decide/peak_probe.py +207 -0
- operator_agent/decide/quality_evaluation.py +421 -0
- operator_agent/decide/quality_pairing.py +336 -0
- operator_agent/decide/quality_portfolio.py +363 -0
- operator_agent/decide/quality_preflight.py +328 -0
- operator_agent/decide/quality_suite.py +317 -0
- operator_agent/decide/reference.py +60 -0
- operator_agent/decide/reference_contract.py +239 -0
- operator_agent/decide/remote_input_fidelity.py +281 -0
- operator_agent/decide/resource_check.py +711 -0
- operator_agent/decide/session_inputs.py +219 -0
- operator_agent/decide/tool_evidence.py +164 -0
- operator_agent/decide/verifier.py +3043 -0
- operator_agent/execute/__init__.py +1 -0
- operator_agent/execute/backend_ownership.py +145 -0
- operator_agent/execute/budget.py +258 -0
- operator_agent/execute/exec_guard.py +1294 -0
- operator_agent/execute/fake_backend/__init__.py +8 -0
- operator_agent/execute/fake_backend/backend.py +177 -0
- operator_agent/execute/fake_backend/config.py +60 -0
- operator_agent/execute/fake_backend/model.py +78 -0
- operator_agent/execute/host_task_budget.py +559 -0
- operator_agent/execute/llm_budget.py +527 -0
- operator_agent/execute/llm_client.py +898 -0
- operator_agent/execute/llm_transport.py +55 -0
- operator_agent/execute/mock_gpu.py +646 -0
- operator_agent/execute/placement.py +355 -0
- operator_agent/execute/remote_executor.py +2130 -0
- operator_agent/execute/s5_inner_budget.py +213 -0
- operator_agent/execute/s5_shared_budget.py +320 -0
- operator_agent/execute/s5_verify_client.py +113 -0
- operator_agent/execute/task_budget.py +101 -0
- operator_agent/execute/task_cancellation.py +110 -0
- operator_agent/execute/task_clock.py +12 -0
- operator_agent/generate/__init__.py +1 -0
- operator_agent/generate/agent_logger.py +255 -0
- operator_agent/generate/agent_runtime.py +1361 -0
- operator_agent/generate/agents.py +966 -0
- operator_agent/generate/bandit.py +204 -0
- operator_agent/generate/bounded_repair.py +116 -0
- operator_agent/generate/coder_pipeline.py +540 -0
- operator_agent/generate/fixer.py +850 -0
- operator_agent/generate/generation_context.py +267 -0
- operator_agent/generate/generation_spec.py +88 -0
- operator_agent/generate/host_context.py +241 -0
- operator_agent/generate/optimization_planner.py +717 -0
- operator_agent/generate/planner.py +94 -0
- operator_agent/generate/prompt_manager.py +206 -0
- operator_agent/generate/repair_diagnoser.py +528 -0
- operator_agent/generate/repair_loop.py +240 -0
- operator_agent/generate/repair_scalar_diagnoser.py +441 -0
- operator_agent/generate/repair_stride_diagnoser.py +681 -0
- operator_agent/generate/search_objective.py +215 -0
- operator_agent/generate/selection.py +273 -0
- operator_agent/generate/single_candidate_coder.py +543 -0
- operator_agent/generate/triton_matmul_template.py +220 -0
- operator_agent/generate/triton_opt_rules.py +238 -0
- operator_agent/ops/__init__.py +7 -0
- operator_agent/ops/csa_compress.py +125 -0
- operator_agent/prompts/coder_from_plan.md +45 -0
- operator_agent/prompts/coder_generate.md +50 -0
- operator_agent/prompts/coder_generate_cuda_c.md +86 -0
- operator_agent/prompts/coder_generate_triton.md +65 -0
- operator_agent/prompts/coder_repair.md +28 -0
- operator_agent/prompts/designer_sketch.md +89 -0
- operator_agent/prompts/fixer_reflexion.md +29 -0
- operator_agent/prompts/fixer_repair.md +80 -0
- operator_agent/prompts/intent_parse.md +38 -0
- operator_agent/prompts/planner_plan.md +27 -0
- operator_agent/prompts/search_optimize.md +64 -0
- operator_agent/skills/optimization_recipes.md +1531 -0
- operator_agent/skills/triton_skill.md +235 -0
- remote_scripts/__init__.py +7 -0
- remote_scripts/gpu_claim.py +130 -0
- remote_scripts/matmul4096/ablate.py +135 -0
- remote_scripts/matmul4096/accounting.py +91 -0
- remote_scripts/matmul4096/analyze.py +50 -0
- remote_scripts/matmul4096/baseline.py +92 -0
- remote_scripts/matmul4096/bench_common.py +100 -0
- remote_scripts/matmul4096/deepdive.py +70 -0
- remote_scripts/matmul4096/explain.py +66 -0
- remote_scripts/matmul4096/final_check.py +72 -0
- remote_scripts/matmul4096/matmul_best.py +123 -0
- remote_scripts/matmul4096/probe.py +44 -0
- remote_scripts/matmul4096/robust.py +93 -0
- remote_scripts/matmul4096/stability.py +57 -0
- remote_scripts/matmul4096/sweep1.py +60 -0
- remote_scripts/matmul4096/triton_ref.py +66 -0
- remote_scripts/run_kernel.py +1893 -0
- remote_scripts/run_kernel_xpu.py +660 -0
- remote_scripts/runner_server.py +168 -0
- remote_scripts/session_control.py +103 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""算子开发Agent v0.7
|
|
2
|
+
支持 Triton DSL: 矩阵乘法/Softmax/Attention 等 GPU 算子
|
|
3
|
+
模块结构(2026-10-31 分五层,全量归属见 docs/design/repo-slimming-20261031.md §5):
|
|
4
|
+
core/ 词汇与产物 - schemas / contract / run_contract / candidates / kernel_db /
|
|
5
|
+
artifact_store / run_layout / report / human_ledger
|
|
6
|
+
decide/ 判定 - verifier (L1/L2/L3 + Roofline) / measurement / eligibility /
|
|
7
|
+
correctness* / anticheat / reference* / failure_route / quality_*
|
|
8
|
+
execute/ 执行与预算 - remote_executor / exec_guard / backend_* / mock_gpu /
|
|
9
|
+
task_budget / task_cancellation / llm_*
|
|
10
|
+
generate/ 生成与修复 - planner / coder_pipeline / single_candidate_coder / fixer /
|
|
11
|
+
repair_* / prompt_manager / optimization_planner / triton_*
|
|
12
|
+
app/ 入口与编排 - conductor / cli / mcp_server / hitl_tools / autonomous /
|
|
13
|
+
intent_parser / tool_calls
|
|
14
|
+
ops/ prompts/ skills/ 内嵌子包与资源目录(SKILL 规范 + Prompt 模板)
|
|
15
|
+
外部用法保持不变: `from operator_agent import Conductor` 一类再导出仍走包根 __init__.py,
|
|
16
|
+
`python -m operator_agent` 由包根 __main__.py 委托。"""
|
|
17
|
+
|
|
18
|
+
__version__ = "0.7.2"
|
|
19
|
+
|
|
20
|
+
from .core.schemas import (
|
|
21
|
+
TaskSchema, SourceSpec, TargetSpec, ConstraintSpec,
|
|
22
|
+
TaskType, TaskState, TaskComplexity, RouteType,
|
|
23
|
+
DSLEnum, TritonCompileConfig,
|
|
24
|
+
Precision, VerificationLevel, VerificationResult,
|
|
25
|
+
UnifiedSketch, TilingStrategy, MemoryPlan, PipelineStage,
|
|
26
|
+
KernelArtifact,
|
|
27
|
+
)
|
|
28
|
+
from .core.kernel_db import KernelDatabase, KernelRecord
|
|
29
|
+
from .execute.mock_gpu import MockGPU, ExecutionResult
|
|
30
|
+
from .execute.fake_backend import FakeBackend
|
|
31
|
+
from .execute.remote_executor import RemoteExecutor, RemoteExecutionResult, SSHExecutor
|
|
32
|
+
from .execute.llm_client import LLMClient
|
|
33
|
+
from .generate.prompt_manager import PromptManager
|
|
34
|
+
from .core.artifact_store import ArtifactStore
|
|
35
|
+
from .generate.agent_logger import capture_execution, get_logger, get_latest_logs, read_log
|
|
36
|
+
from .generate.agents import DesignerAgent, CoderAgent, SearchAgent
|
|
37
|
+
from .decide.verifier import Verifier
|
|
38
|
+
from .decide.correctness import CorrectnessCase, CorrectnessSuite
|
|
39
|
+
from .decide.correctness_boundaries import (BoundaryAxis, TensorAxes, BoundaryPolicy,
|
|
40
|
+
generate_correctness_suite, lint_coverage)
|
|
41
|
+
from .decide.correctness_profiles import with_default_correctness
|
|
42
|
+
from .app.conductor import Conductor, Router
|
|
43
|
+
from .app.intent_parser import IntentParser, TestInputGenerator
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""`python -m operator_agent` —— 委托给 app 层的 CLI。
|
|
2
|
+
|
|
3
|
+
2026-10-31: 包分五层后真正的入口在 `operator_agent/app/`,但 `python -m operator_agent`
|
|
4
|
+
是**公开**调用形态(README / user_guide / 插件测试都在用),所以包根留这一个委托文件。
|
|
5
|
+
它不是第二份实现:只 import 并调用 app.cli 的 main。
|
|
6
|
+
"""
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from .app.cli import main
|
|
10
|
+
|
|
11
|
+
if __name__ == "__main__":
|
|
12
|
+
sys.exit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""app 层:见 docs/design/repo-slimming-20261031.md。"""
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Process-local controls reserved for tests and benchmark harnesses.
|
|
2
|
+
|
|
3
|
+
These switches intentionally have no environment-variable or CLI input. A test or
|
|
4
|
+
benchmark may enable them in-process before constructing the agent in order to
|
|
5
|
+
redirect/disable side-effect stores or use an oversized heartbeat interval. A
|
|
6
|
+
normal task, subprocess, or candidate cannot turn these controls on by setting an
|
|
7
|
+
environment variable.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
_ENABLED = False
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def enable_for_harness() -> None:
|
|
14
|
+
"""Enable harness-only test controls for this Python process."""
|
|
15
|
+
global _ENABLED
|
|
16
|
+
_ENABLED = True
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def enabled() -> bool:
|
|
20
|
+
"""Return whether harness-only controls are enabled in this process."""
|
|
21
|
+
return _ENABLED
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""无人值守整段运行(Phase 3 的后端执行体)
|
|
2
|
+
|
|
3
|
+
由 `hitl_tools.run_autonomous()` 以**独立进程**拉起:一拍启动、之后用
|
|
4
|
+
`hitl_tools.autonomous_status()` 查进度。为什么是独立进程而不是在 MCP server 里同步跑:
|
|
5
|
+
|
|
6
|
+
* 一次 `Conductor.execute` 是分钟~小时级,宿主侧的工具调用超时按「一次请求」算;
|
|
7
|
+
* 崩了要能看出「是这一轮失败」还是「进程没了」—— 两件事分开报;
|
|
8
|
+
* 宿主重启不该杀掉已经跑起来的实验。
|
|
9
|
+
|
|
10
|
+
它只做三件事:**读 `declaration.json` → 跑 `Conductor` → 把结论写进 `run-status.json`**。
|
|
11
|
+
判据一行都不在这里(契约冻结、L1/L2/L3、资格、入库门全在既有模块里),这一层只负责
|
|
12
|
+
「把一次无人值守运行装进一个目录」。
|
|
13
|
+
|
|
14
|
+
身份与状态:
|
|
15
|
+
<run>/autonomous.json 启动参数(backend / max_rounds / target_gflops)
|
|
16
|
+
<run>/run-status.json 相位机:running → done | failed | cancelled
|
|
17
|
+
<run>/conductor.json 完成时的摘要(artifact / 停因 / 最好读数 / 报告路径)
|
|
18
|
+
<run>/autonomous.log 子进程的 stdout+stderr
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
import time
|
|
28
|
+
from ..execute.task_clock import monotonic
|
|
29
|
+
import traceback
|
|
30
|
+
import uuid
|
|
31
|
+
import signal
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
STATUS = "run-status.json"
|
|
35
|
+
SUMMARY = "conductor.json"
|
|
36
|
+
CONFIG = "autonomous.json"
|
|
37
|
+
WORKER_RESULT = "worker-result.json"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _atomic_json(path: Path, payload: dict) -> None:
|
|
41
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
42
|
+
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True,
|
|
43
|
+
default=str), encoding="utf-8")
|
|
44
|
+
os.replace(tmp, path)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _backend(name: str, cfg: dict):
|
|
48
|
+
if name == 'fake':
|
|
49
|
+
from ..execute.fake_backend import FakeBackend
|
|
50
|
+
return FakeBackend(cfg.get('fake'), event_path=Path(cfg['run_dir'])/'fake-events.jsonl')
|
|
51
|
+
from ..app.hitl_tools import _backend as make
|
|
52
|
+
return make(name,
|
|
53
|
+
host=cfg.get("host"), port=cfg.get("port"), user=cfg.get("user"),
|
|
54
|
+
key_path=cfg.get("key_path"), work_dir=cfg.get("work_dir"),
|
|
55
|
+
timeout=int(cfg.get("timeout") or 300),
|
|
56
|
+
persistent=bool(cfg.get("persistent")))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _worker_main(run, deadline, started_monotonic, invocation_id=None) -> int:
|
|
60
|
+
from ..execute.task_budget import TaskBudgetConfig, snapshot, stopped_reason
|
|
61
|
+
started = time.time()
|
|
62
|
+
cfg = json.loads((run / CONFIG).read_text(encoding="utf-8"))
|
|
63
|
+
cfg['run_dir'] = str(run)
|
|
64
|
+
context = _execution_context(cfg)
|
|
65
|
+
declaration = json.loads((run / "declaration.json").read_text(encoding="utf-8"))
|
|
66
|
+
policy = TaskBudgetConfig.from_dict(cfg.get('task_budget'))
|
|
67
|
+
|
|
68
|
+
status = {"phase": "running", "pid": os.getpid(), "started_at": started,
|
|
69
|
+
"backend": cfg.get("backend"), "max_rounds": cfg.get("max_rounds"),
|
|
70
|
+
"run_dir": str(run),
|
|
71
|
+
# 语料落点也收在 run 目录:这一版的语料是**运行内**的,不写入仓库那份。
|
|
72
|
+
"corpus": {"kernel_db": "run-local", "failure_db": "run-local"}}
|
|
73
|
+
status.update(task_budget=policy.as_dict(), invocation_id=invocation_id, **context)
|
|
74
|
+
conductor = None
|
|
75
|
+
|
|
76
|
+
# 落点收进 run 目录:无人值守的运行不该把产物撒到仓库或用户 home 的默认位置。
|
|
77
|
+
os.environ["OPERATOR_AGENT_OUTPUT_DIR"] = str(run / "output")
|
|
78
|
+
os.environ["OPERATOR_AGENT_LOG_DIR"] = str(run / "log")
|
|
79
|
+
# ⚠️ 这两个**不能写 "off"**:2026-10-08(§3.5)之后普通环境变量只能**重定向**,
|
|
80
|
+
# 关不掉持久化 —— `off` 会被解析回 `<repo>/output/*.jsonl`,于是子进程会读写仓库
|
|
81
|
+
# 里那份语料。2026-10-18 实测就撞上了(日志里 `FailureDB 重放 95` 来自仓库)。
|
|
82
|
+
os.environ["OPERATOR_AGENT_KERNEL_DB"] = str(run / "kernel_db.jsonl")
|
|
83
|
+
os.environ["OPERATOR_AGENT_FAILURE_DB"] = str(run / "failure_db.jsonl")
|
|
84
|
+
try:
|
|
85
|
+
from ..app.conductor import Conductor
|
|
86
|
+
from ..core.contract import freeze_from_task
|
|
87
|
+
from ..app.hitl_tools import build_task
|
|
88
|
+
from ..execute.llm_client import LLMClient
|
|
89
|
+
from ..execute.s5_inner_budget import production_budget
|
|
90
|
+
from ..execute.budget import BudgetPlan
|
|
91
|
+
|
|
92
|
+
task = build_task(declaration)
|
|
93
|
+
contract = freeze_from_task(task)
|
|
94
|
+
inputs = contract.cases[0].materialize()
|
|
95
|
+
call_timeout = min(float(cfg.get('llm_timeout') or 120), policy.wall_clock_s)
|
|
96
|
+
ledger = production_budget(policy.llm_limits(call_timeout), run/'model-journal',
|
|
97
|
+
deadline=deadline)
|
|
98
|
+
if cfg.get('backend') == 'fake' and cfg.get('fake', {}).get('llm', 'fake') == 'fake':
|
|
99
|
+
from ..execute.fake_backend import FakeLLM
|
|
100
|
+
llm = FakeLLM(task.source.kernel_type, timeout=call_timeout, call_budget=ledger)
|
|
101
|
+
else:
|
|
102
|
+
llm = LLMClient(max_tokens=4096, timeout=call_timeout, call_budget=ledger)
|
|
103
|
+
backend = _backend(str(cfg.get("backend") or "mock"), cfg)
|
|
104
|
+
|
|
105
|
+
conductor = Conductor(
|
|
106
|
+
backend=backend, llm=llm,
|
|
107
|
+
max_optimize_rounds=int(cfg.get("max_rounds") or 1),
|
|
108
|
+
# 一条候选路径:这一层是「入口」,不是搜索策略。换 pipeline 会改候选选择口径,
|
|
109
|
+
# 那件事应当单独评估,不该由「有没有这个工具」顺手改掉。
|
|
110
|
+
use_coder_pipeline=False,
|
|
111
|
+
run_root=str(run),
|
|
112
|
+
budget=BudgetPlan(oracle_total=policy.oracle_total, wall_clock_s=policy.wall_clock_s),
|
|
113
|
+
)
|
|
114
|
+
artifact = conductor.execute(task, inputs,
|
|
115
|
+
target_gflops=float(cfg.get("target_gflops") or 0.0))
|
|
116
|
+
|
|
117
|
+
metrics = getattr(artifact, "benchmark_data", None) or {}
|
|
118
|
+
generation = getattr(artifact, 'metadata', None) or {}
|
|
119
|
+
summary = {
|
|
120
|
+
"phase": "done",
|
|
121
|
+
"artifact_id": getattr(artifact, "kernel_id", ""),
|
|
122
|
+
"all_passed": bool(getattr(artifact, "all_passed", False)),
|
|
123
|
+
"verification_all_passed": bool(getattr(artifact, "all_passed", False)),
|
|
124
|
+
"stopped_reason": (generation.get('stop') or {}).get('reason') or metrics.get("stop_reason"),
|
|
125
|
+
"levels": {r.level.value: bool(r.passed)
|
|
126
|
+
for r in (getattr(artifact, "verification_results", None) or [])},
|
|
127
|
+
"speedup": metrics.get("speedup_vs_baseline"),
|
|
128
|
+
"gflops": metrics.get("gflops"),
|
|
129
|
+
"validity": metrics.get("measurement_validity"),
|
|
130
|
+
"path": getattr(artifact, "saved_dir", ""),
|
|
131
|
+
"rounds": len(getattr(artifact, "iteration_history", None) or []),
|
|
132
|
+
"candidate_record_count": len(generation.get('candidate_records', [])),
|
|
133
|
+
"selected_source_hash": generation.get('selected_source_hash'),
|
|
134
|
+
"generation_evidence": (str(run / 'generation-records.json')
|
|
135
|
+
if (run / 'generation-records.json').is_file() else None),
|
|
136
|
+
"finished_at": time.time(),
|
|
137
|
+
"duration_s": round(time.time() - started, 1),
|
|
138
|
+
**context,
|
|
139
|
+
}
|
|
140
|
+
budget = snapshot(run, policy, started_monotonic)
|
|
141
|
+
reason = stopped_reason(budget)
|
|
142
|
+
if summary['stopped_reason'] in ('budget_exhausted', 'wall_clock_exhausted'):
|
|
143
|
+
summary['all_passed'] = False
|
|
144
|
+
if reason:
|
|
145
|
+
summary.update(stopped_reason=reason, all_passed=False)
|
|
146
|
+
summary.update(task_budget=policy.as_dict(), budget=budget)
|
|
147
|
+
_atomic_json(run / SUMMARY, summary)
|
|
148
|
+
status.update(phase="done", finished_at=summary["finished_at"],
|
|
149
|
+
duration_s=summary["duration_s"],
|
|
150
|
+
summary_path=str(run / SUMMARY), stopped_reason=summary['stopped_reason'],
|
|
151
|
+
budget=budget)
|
|
152
|
+
status['finished_monotonic'] = monotonic()
|
|
153
|
+
_atomic_json(run / WORKER_RESULT, status)
|
|
154
|
+
print(f"[autonomous] done in {summary['duration_s']}s: "
|
|
155
|
+
f"all_passed={summary['all_passed']} speedup={summary['speedup']}",
|
|
156
|
+
file=sys.stderr)
|
|
157
|
+
return 0
|
|
158
|
+
except Exception as exc: # noqa: BLE001
|
|
159
|
+
failed_monotonic = monotonic()
|
|
160
|
+
tail = "".join(traceback.format_exc().splitlines(keepends=True)[-12:])
|
|
161
|
+
status.update(phase="failed", error=type(exc).__name__, detail=str(exc)[:800],
|
|
162
|
+
traceback=tail, finished_at=time.time(),
|
|
163
|
+
duration_s=round(time.time() - started, 1))
|
|
164
|
+
if (run / 'generation-records.json').is_file():
|
|
165
|
+
status['generation_evidence'] = str(run / 'generation-records.json')
|
|
166
|
+
budget = snapshot(run, policy, started_monotonic, now=failed_monotonic)
|
|
167
|
+
reason = stopped_reason(budget)
|
|
168
|
+
status.update(budget=budget, stopped_reason=reason)
|
|
169
|
+
if reason:
|
|
170
|
+
status.update(phase='done', error=None, detail=None)
|
|
171
|
+
_atomic_json(run / SUMMARY, dict(phase='done', all_passed=False,
|
|
172
|
+
stopped_reason=reason, task_budget=policy.as_dict(), budget=budget,
|
|
173
|
+
finished_at=status['finished_at'], duration_s=status['duration_s'], **context))
|
|
174
|
+
status['finished_monotonic'] = failed_monotonic
|
|
175
|
+
_atomic_json(run / WORKER_RESULT, status)
|
|
176
|
+
print(f"[autonomous] failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
177
|
+
return 0 if reason else 1
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _execution_context(cfg):
|
|
181
|
+
if cfg.get('backend') != 'fake':
|
|
182
|
+
return {}
|
|
183
|
+
from ..execute.fake_backend import execution_context
|
|
184
|
+
return dict(execution_context=execution_context(cfg.get('fake')))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _owned_groups(worker_pid):
|
|
188
|
+
from ..execute.exec_guard import task_process_groups
|
|
189
|
+
return task_process_groups(worker_pid)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _stop_worker(process, observed_groups=(), graceful_s=0):
|
|
193
|
+
from ..execute.exec_guard import stop_task_worker
|
|
194
|
+
return stop_task_worker(process, observed_groups, discover=_owned_groups,
|
|
195
|
+
graceful_s=graceful_s)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _start_worker(run, deadline, started_monotonic, invocation_id):
|
|
199
|
+
from ..execute.exec_guard import start_task_worker
|
|
200
|
+
return start_task_worker([sys.executable, '-m', 'operator_agent.app.autonomous',
|
|
201
|
+
'--worker', str(run), str(deadline), str(started_monotonic), invocation_id],
|
|
202
|
+
cwd=str(Path(__file__).resolve().parents[2]))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _main(argv=None) -> int:
|
|
206
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
207
|
+
if len(argv) == 5 and argv[0] == '--worker':
|
|
208
|
+
run = Path(argv[1]).resolve()
|
|
209
|
+
from ..execute.backend_ownership import owned_persistent_ssh, owned_worker_scope
|
|
210
|
+
graceful = owned_persistent_ssh(json.loads((run/CONFIG).read_text()))
|
|
211
|
+
if not graceful:
|
|
212
|
+
return _worker_main(run, float(argv[2]), float(argv[3]), argv[4])
|
|
213
|
+
with owned_worker_scope(run, argv[4]):
|
|
214
|
+
return _worker_main(run, float(argv[2]), float(argv[3]), argv[4])
|
|
215
|
+
if len(argv) != 1:
|
|
216
|
+
print('usage: python -m operator_agent.autonomous <run_dir>', file=sys.stderr)
|
|
217
|
+
return 2
|
|
218
|
+
from ..execute.task_budget import TaskBudgetConfig, snapshot, stopped_reason
|
|
219
|
+
from ..execute.task_cancellation import (launch_identity, read_json, stop_event,
|
|
220
|
+
partial_artifacts, valid_invocation, cancellation,
|
|
221
|
+
request_cancel)
|
|
222
|
+
from ..execute.exec_guard import process_identity
|
|
223
|
+
run = Path(argv[0]).resolve()
|
|
224
|
+
started_monotonic, started = monotonic(), time.time()
|
|
225
|
+
try:
|
|
226
|
+
cfg = json.loads((run / CONFIG).read_text(encoding='utf-8'))
|
|
227
|
+
policy = TaskBudgetConfig.from_dict(cfg.get('task_budget'))
|
|
228
|
+
except (OSError, ValueError, TypeError) as exc:
|
|
229
|
+
_atomic_json(run / STATUS, dict(phase='failed', pid=os.getpid(),
|
|
230
|
+
error=type(exc).__name__, detail=str(exc), stopped_reason='invalid_task_budget',
|
|
231
|
+
finished_at=time.time(), started_at=started))
|
|
232
|
+
return 1
|
|
233
|
+
deadline = started_monotonic + policy.wall_clock_s
|
|
234
|
+
invocation_id = cfg.get('invocation_id')
|
|
235
|
+
if invocation_id is None: # legacy/manual supervisor entry gets its own identity
|
|
236
|
+
invocation_id = uuid.uuid4().hex
|
|
237
|
+
launch_identity(run, invocation_id, kind='autonomous')
|
|
238
|
+
launch = read_json(run/'launch.json') or {}
|
|
239
|
+
if not valid_invocation(invocation_id) or launch.get('invocation_id') != invocation_id:
|
|
240
|
+
return 1 # never overwrite another invocation's status or signal its PID
|
|
241
|
+
status = dict(phase='running', stage='executing', pid=os.getpid(), started_at=started,
|
|
242
|
+
backend=cfg.get('backend'), max_rounds=cfg.get('max_rounds'), run_dir=str(run),
|
|
243
|
+
task_budget=policy.as_dict(), invocation_id=invocation_id,
|
|
244
|
+
corpus={'kernel_db':'run-local', 'failure_db':'run-local'})
|
|
245
|
+
context = _execution_context(cfg)
|
|
246
|
+
status.update(context)
|
|
247
|
+
status['pid_identity'] = process_identity(os.getpid())
|
|
248
|
+
status['budget'] = snapshot(run, policy, started_monotonic)
|
|
249
|
+
_atomic_json(run / STATUS, status)
|
|
250
|
+
reason, event_at = stop_event(run, invocation_id, deadline)
|
|
251
|
+
if reason == 'user_cancelled':
|
|
252
|
+
status.update(phase='cancelled', stopped_reason=reason, finished_at=time.time(),
|
|
253
|
+
cancellation=cancellation(run, invocation_id), partial_artifacts=partial_artifacts(run),
|
|
254
|
+
cleanup=dict(status='no_worker_started', worker_reaped=True,
|
|
255
|
+
remote_execution='unknown', provider_call='unknown'))
|
|
256
|
+
_atomic_json(run / SUMMARY, dict(phase='cancelled', all_passed=False,
|
|
257
|
+
stopped_reason=reason, task_budget=policy.as_dict(), budget=status['budget'], **context))
|
|
258
|
+
_atomic_json(run / STATUS, status)
|
|
259
|
+
return 1
|
|
260
|
+
def interrupted(signum, frame):
|
|
261
|
+
request_cancel(run, invocation_id)
|
|
262
|
+
previous = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGINT, signal.SIGTERM)}
|
|
263
|
+
try:
|
|
264
|
+
process = _start_worker(run, deadline, started_monotonic, invocation_id)
|
|
265
|
+
except OSError as exc:
|
|
266
|
+
status.update(phase='failed', error=type(exc).__name__, detail=str(exc),
|
|
267
|
+
finished_at=time.time(), stopped_reason='worker_start_failed')
|
|
268
|
+
_atomic_json(run / STATUS, status)
|
|
269
|
+
for sig, handler in previous.items():
|
|
270
|
+
signal.signal(sig, handler)
|
|
271
|
+
return 1
|
|
272
|
+
status['worker_pid'] = process.pid
|
|
273
|
+
last_update = 0.
|
|
274
|
+
from ..execute.exec_guard import OwnedGroups
|
|
275
|
+
observed_groups = OwnedGroups([process.pid])
|
|
276
|
+
reason = event_at = None
|
|
277
|
+
from ..execute.backend_ownership import owned_persistent_ssh
|
|
278
|
+
graceful_remote = owned_persistent_ssh(cfg)
|
|
279
|
+
try:
|
|
280
|
+
while process.poll() is None:
|
|
281
|
+
remaining = deadline-monotonic()
|
|
282
|
+
reason, event_at = stop_event(run, invocation_id, deadline)
|
|
283
|
+
if reason:
|
|
284
|
+
break
|
|
285
|
+
groups, discovered = _owned_groups(process.pid)
|
|
286
|
+
observed_groups.update(groups)
|
|
287
|
+
observed_groups.discovery_complete &= discovered
|
|
288
|
+
if monotonic()-last_update >= 1.:
|
|
289
|
+
status['budget'] = snapshot(run, policy, started_monotonic)
|
|
290
|
+
_atomic_json(run / STATUS, status)
|
|
291
|
+
last_update = monotonic()
|
|
292
|
+
try:
|
|
293
|
+
process.wait(timeout=min(.25, max(.001, remaining)))
|
|
294
|
+
except subprocess.TimeoutExpired:
|
|
295
|
+
pass
|
|
296
|
+
finally:
|
|
297
|
+
cleanup = (_stop_worker(process, observed_groups, 12)
|
|
298
|
+
if reason and graceful_remote else _stop_worker(process, observed_groups))
|
|
299
|
+
try:
|
|
300
|
+
result = json.loads((run / WORKER_RESULT).read_text())
|
|
301
|
+
except (OSError, ValueError):
|
|
302
|
+
result = {}
|
|
303
|
+
same_invocation = result.get('invocation_id') == invocation_id
|
|
304
|
+
finished = result.get('finished_monotonic') if same_invocation else None
|
|
305
|
+
valid_finish = type(finished) in (int, float) and started_monotonic <= finished <= monotonic()
|
|
306
|
+
budget = snapshot(run, policy, started_monotonic,
|
|
307
|
+
now=finished if valid_finish else None)
|
|
308
|
+
reason, event_at = stop_event(run, invocation_id, deadline)
|
|
309
|
+
result_won = valid_finish and (event_at is None or finished <= event_at)
|
|
310
|
+
if reason and not result_won:
|
|
311
|
+
phase = 'cancelled' if reason == 'user_cancelled' else 'done'
|
|
312
|
+
status.update(phase=phase, stopped_reason=reason,
|
|
313
|
+
error=None, detail=None, budget=budget)
|
|
314
|
+
_atomic_json(run / SUMMARY, dict(phase=phase, all_passed=False,
|
|
315
|
+
stopped_reason=reason, task_budget=policy.as_dict(), budget=budget,
|
|
316
|
+
finished_at=time.time(), duration_s=round(time.time()-started, 3), **context))
|
|
317
|
+
elif valid_finish and result.get('phase') in ('done', 'failed'):
|
|
318
|
+
status.update(result, pid=os.getpid(), started_at=started, budget=budget)
|
|
319
|
+
else:
|
|
320
|
+
status.update(phase='failed', error='WorkerExited',
|
|
321
|
+
detail='worker exited without a terminal result', stopped_reason='worker_failed', budget=budget)
|
|
322
|
+
from ..execute.backend_ownership import cleanup_state
|
|
323
|
+
cleanup.update(remote_execution=(cleanup_state(run, invocation_id)
|
|
324
|
+
if graceful_remote else 'unknown'), provider_call='unknown')
|
|
325
|
+
status.update(finished_at=time.time(), duration_s=round(time.time()-started, 3), cleanup=cleanup,
|
|
326
|
+
cancellation=cancellation(run, invocation_id), partial_artifacts=partial_artifacts(run))
|
|
327
|
+
if (run/'generation-records.json').is_file():
|
|
328
|
+
status['generation_evidence'] = str(run/'generation-records.json')
|
|
329
|
+
_atomic_json(run / STATUS, status)
|
|
330
|
+
for sig, handler in previous.items():
|
|
331
|
+
signal.signal(sig, handler)
|
|
332
|
+
return 0 if status['phase'] == 'done' else 1
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def main(argv=None) -> int:
|
|
336
|
+
previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)}
|
|
337
|
+
try:
|
|
338
|
+
return _main(argv)
|
|
339
|
+
finally:
|
|
340
|
+
for sig, handler in previous.items():
|
|
341
|
+
signal.signal(sig, handler)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
if __name__ == "__main__":
|
|
345
|
+
sys.exit(main())
|