agentic-dev-loop 0.12.0__py3-none-any.whl → 0.12.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentic_dev_loop/core/failure.py +21 -0
- agentic_dev_loop/core/verdict.py +87 -0
- agentic_dev_loop/engine/actions.py +6 -5
- agentic_dev_loop/engine/agents.py +94 -2
- agentic_dev_loop/engine/cli.py +3 -0
- agentic_dev_loop/engine/completion.py +48 -0
- agentic_dev_loop/engine/dryrun.py +8 -0
- agentic_dev_loop/engine/graph.py +11 -0
- agentic_dev_loop/engine/llm.py +59 -1
- agentic_dev_loop/engine/onboarding/templates/qa.agent.md +25 -18
- agentic_dev_loop/engine/onboarding/templates/review.agent.md +11 -14
- agentic_dev_loop/engine/provider_wire.py +153 -1
- agentic_dev_loop/engine/run_status.py +360 -0
- agentic_dev_loop/engine/run_status_command.md +23 -0
- agentic_dev_loop/engine/stages.py +174 -45
- {agentic_dev_loop-0.12.0.dist-info → agentic_dev_loop-0.12.2.dist-info}/METADATA +73 -10
- {agentic_dev_loop-0.12.0.dist-info → agentic_dev_loop-0.12.2.dist-info}/RECORD +21 -17
- {agentic_dev_loop-0.12.0.dist-info → agentic_dev_loop-0.12.2.dist-info}/WHEEL +0 -0
- {agentic_dev_loop-0.12.0.dist-info → agentic_dev_loop-0.12.2.dist-info}/entry_points.txt +0 -0
- {agentic_dev_loop-0.12.0.dist-info → agentic_dev_loop-0.12.2.dist-info}/licenses/LICENSE +0 -0
- {agentic_dev_loop-0.12.0.dist-info → agentic_dev_loop-0.12.2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Structural failure vocabulary for agent steps (unit-11).
|
|
2
|
+
|
|
3
|
+
These classes are value objects: they name *why* an agent step failed
|
|
4
|
+
structurally, never how the run should prose its way around the failure. A
|
|
5
|
+
structural failure halts the step and the run - it never enters the retry
|
|
6
|
+
path, because retrying re-runs the same refusal (FD-11, rule E-1).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import enum
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FailureClass(str, enum.Enum):
|
|
15
|
+
"""The named structural failure classes an agent step can raise."""
|
|
16
|
+
|
|
17
|
+
NO_ACTIONS = "no_actions"
|
|
18
|
+
NO_VERDICT_TOOL = "no_verdict_tool"
|
|
19
|
+
INVALID_VERDICT_VALUE = "invalid_verdict_value"
|
|
20
|
+
CONFLICTING_VERDICT = "conflicting_verdict"
|
|
21
|
+
WIRE_SHAPE = "wire_shape"
|
agentic_dev_loop/core/verdict.py
CHANGED
|
@@ -10,10 +10,17 @@ totality for arbitrary input (no crash, no silent routing).
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import enum
|
|
13
|
+
import json
|
|
13
14
|
from dataclasses import dataclass
|
|
15
|
+
from typing import Iterable
|
|
14
16
|
|
|
17
|
+
from .failure import FailureClass
|
|
15
18
|
from .results import Result
|
|
16
19
|
|
|
20
|
+
# The structured verdict channel's tool name. The tool is registered by the
|
|
21
|
+
# engine (engine/agents.py); the entity owns the argument contract.
|
|
22
|
+
RECORD_VERDICT_TOOL = "record_verdict"
|
|
23
|
+
|
|
17
24
|
# Leading markdown/list/quote decoration tolerated on a verdict line.
|
|
18
25
|
_LEAD_DECOR = "#*_`>- "
|
|
19
26
|
# Decoration tolerated between VERDICT and ':' and after the keyword.
|
|
@@ -33,6 +40,15 @@ class VerdictError(enum.Enum):
|
|
|
33
40
|
AMBIGUOUS = "ambiguous"
|
|
34
41
|
|
|
35
42
|
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class VerdictChannelError:
|
|
45
|
+
"""A structured verdict channel failure: the class plus the evidence a
|
|
46
|
+
report must name (the offending value, the configured set, the conflict)."""
|
|
47
|
+
|
|
48
|
+
code: FailureClass
|
|
49
|
+
detail: str
|
|
50
|
+
|
|
51
|
+
|
|
36
52
|
@dataclass(frozen=True)
|
|
37
53
|
class Verdict:
|
|
38
54
|
"""A parsed decision: `value` is the canonical outcome (e.g. 'pass').
|
|
@@ -64,6 +80,77 @@ class Verdict:
|
|
|
64
80
|
return Result.err(VerdictError.NO_VERDICT)
|
|
65
81
|
|
|
66
82
|
|
|
83
|
+
@staticmethod
|
|
84
|
+
def from_events(
|
|
85
|
+
events: Iterable[object], verdicts: dict[str, str]
|
|
86
|
+
) -> "Result[Verdict, VerdictChannelError]":
|
|
87
|
+
"""Decode the verdict from record_verdict tool calls (FR-11.2).
|
|
88
|
+
|
|
89
|
+
The event stream is the only source: prose is never parsed here. Zero
|
|
90
|
+
calls is NO_VERDICT_TOOL; a value outside the configured set is
|
|
91
|
+
INVALID_VERDICT_VALUE; two different decisions are CONFLICTING_VERDICT
|
|
92
|
+
(fail fast, FD-Q1). Matching is case-insensitive because models vary
|
|
93
|
+
the casing of a configured keyword."""
|
|
94
|
+
values: list[str] = []
|
|
95
|
+
calls = 0
|
|
96
|
+
for event in events:
|
|
97
|
+
if getattr(event, "tool_name", None) != RECORD_VERDICT_TOOL:
|
|
98
|
+
continue
|
|
99
|
+
calls += 1
|
|
100
|
+
value = _verdict_argument(event)
|
|
101
|
+
if value is not None:
|
|
102
|
+
values.append(value)
|
|
103
|
+
if calls == 0:
|
|
104
|
+
return Result.err(VerdictChannelError(
|
|
105
|
+
FailureClass.NO_VERDICT_TOOL,
|
|
106
|
+
f"no {RECORD_VERDICT_TOOL} tool call in the event stream"))
|
|
107
|
+
lookup = {keyword.casefold(): value
|
|
108
|
+
for keyword, value in verdicts.items()}
|
|
109
|
+
decided: dict[str, str] = {}
|
|
110
|
+
for value in values:
|
|
111
|
+
canonical = lookup.get(value.casefold())
|
|
112
|
+
if canonical is None:
|
|
113
|
+
configured = ", ".join(sorted(verdicts))
|
|
114
|
+
return Result.err(VerdictChannelError(
|
|
115
|
+
FailureClass.INVALID_VERDICT_VALUE,
|
|
116
|
+
f"{RECORD_VERDICT_TOOL} value {value!r} is not in the "
|
|
117
|
+
f"configured set [{configured}]"))
|
|
118
|
+
decided[canonical] = value
|
|
119
|
+
if not decided:
|
|
120
|
+
return Result.err(VerdictChannelError(
|
|
121
|
+
FailureClass.INVALID_VERDICT_VALUE,
|
|
122
|
+
f"{RECORD_VERDICT_TOOL} call carried no readable verdict "
|
|
123
|
+
"argument"))
|
|
124
|
+
if len(decided) > 1:
|
|
125
|
+
return Result.err(VerdictChannelError(
|
|
126
|
+
FailureClass.CONFLICTING_VERDICT,
|
|
127
|
+
"conflicting verdicts: "
|
|
128
|
+
+ ", ".join(sorted(repr(v) for v in decided.values()))))
|
|
129
|
+
return Result.ok(Verdict(value=next(iter(decided))))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _verdict_argument(event: object) -> str | None:
|
|
133
|
+
"""The verdict argument carried by one RecordVerdictAction event.
|
|
134
|
+
|
|
135
|
+
Prefers the parsed action (the SDK's normal path); falls back to the raw
|
|
136
|
+
tool-call arguments JSON when the action was not parsed."""
|
|
137
|
+
action = getattr(event, "action", None)
|
|
138
|
+
value = getattr(action, "verdict", None)
|
|
139
|
+
if isinstance(value, str):
|
|
140
|
+
return value
|
|
141
|
+
tool_call = getattr(event, "tool_call", None)
|
|
142
|
+
arguments = getattr(tool_call, "arguments", None)
|
|
143
|
+
if not isinstance(arguments, str):
|
|
144
|
+
return None
|
|
145
|
+
try:
|
|
146
|
+
payload = json.loads(arguments)
|
|
147
|
+
except json.JSONDecodeError:
|
|
148
|
+
return None
|
|
149
|
+
if isinstance(payload, dict) and isinstance(payload.get("verdict"), str):
|
|
150
|
+
return payload["verdict"]
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
|
|
67
154
|
def parse_verdict(text: str, verdicts: dict[str, str]) -> str:
|
|
68
155
|
"""Route for a verdict artifact: the canonical decision value on a single
|
|
69
156
|
clear decision, else ``retry``. Never raises - unclear verdicts retry."""
|
|
@@ -215,7 +215,8 @@ def commit(state: dict, params: dict) -> dict:
|
|
|
215
215
|
if not state.get("commit_allowed", False):
|
|
216
216
|
print(f"[{subject}] Skipping auto-commit - worktree was dirty at run "
|
|
217
217
|
f"start; commit the constructed work manually.", flush=True)
|
|
218
|
-
return {
|
|
218
|
+
return {"verified_state": {
|
|
219
|
+
"commit": "skipped: worktree dirty at run start"}}
|
|
219
220
|
if "message" not in params:
|
|
220
221
|
raise ValueError("commit action requires params.message "
|
|
221
222
|
"(a template rendered with {subject})")
|
|
@@ -235,10 +236,10 @@ def commit(state: dict, params: dict) -> dict:
|
|
|
235
236
|
# permission problem) - its error is the diagnostic that matters.
|
|
236
237
|
print(f"WARNING: [{subject}] auto-commit failed - git add: "
|
|
237
238
|
f"{added.stderr.strip()}", file=sys.stderr, flush=True)
|
|
238
|
-
return {}
|
|
239
|
+
return {"verified_state": {"commit": "failed: add"}}
|
|
239
240
|
if not _git(["status", "--porcelain"], cwd=cwd).stdout.strip():
|
|
240
241
|
print(f"[{subject}] Nothing to commit.", flush=True)
|
|
241
|
-
return {}
|
|
242
|
+
return {"verified_state": {"commit": "nothing to commit"}}
|
|
242
243
|
message = params["message"].format(subject=subject)
|
|
243
244
|
result = _git(["-c", f"core.hooksPath={hooks}", "commit", "-m", message],
|
|
244
245
|
cwd=cwd)
|
|
@@ -247,7 +248,7 @@ def commit(state: dict, params: dict) -> dict:
|
|
|
247
248
|
# not a pipeline failure.
|
|
248
249
|
print(f"WARNING: [{subject}] auto-commit failed - commit manually.\n"
|
|
249
250
|
f"{result.stderr.strip()}", file=sys.stderr, flush=True)
|
|
250
|
-
return {}
|
|
251
|
+
return {"verified_state": {"commit": "failed: commit"}}
|
|
251
252
|
short = _git(["rev-parse", "--short", "HEAD"], cwd=cwd).stdout.strip()
|
|
252
253
|
print(f"[{subject}] Committed constructed work as {short}.", flush=True)
|
|
253
|
-
return {}
|
|
254
|
+
return {"verified_state": {"commit": f"committed {short}"}}
|
|
@@ -10,13 +10,17 @@ from pathlib import Path
|
|
|
10
10
|
from typing import TYPE_CHECKING
|
|
11
11
|
|
|
12
12
|
from openhands.sdk import LLM
|
|
13
|
-
from openhands.sdk.tool import
|
|
13
|
+
from openhands.sdk.tool import (Action, Observation, ToolAnnotations,
|
|
14
|
+
ToolDefinition, ToolExecutor, register_tool)
|
|
15
|
+
from pydantic import Field
|
|
14
16
|
from openhands.tools.file_editor import FileEditorTool # noqa: F401 - registers "file_editor"
|
|
15
17
|
from openhands.tools.task_tracker import TaskTrackerTool # noqa: F401 - registers "task_tracker"
|
|
16
18
|
from openhands.tools.terminal import TerminalTool # noqa: F401 - registers "terminal"
|
|
17
19
|
|
|
18
20
|
from .forbidden import ForbiddenRule, match_forbidden_command
|
|
19
21
|
from .brief_prompt import render_brief_prompt
|
|
22
|
+
from agentic_dev_loop.core.verdict import RECORD_VERDICT_TOOL
|
|
23
|
+
|
|
20
24
|
from .isolation import (SandboxError, SandboxExecutionRequest, command_argv,
|
|
21
25
|
resolve_sandbox_env)
|
|
22
26
|
|
|
@@ -32,7 +36,39 @@ if TYPE_CHECKING:
|
|
|
32
36
|
BRIEF_AGENT_NAME = "4f1c9e2b-7a3d-4e5f-8b6c-0d1a2f3c4d5e"
|
|
33
37
|
|
|
34
38
|
|
|
35
|
-
def
|
|
39
|
+
def check_required_tools(tools: list[str], required: tuple[str, ...],
|
|
40
|
+
*, agent_name: str) -> None:
|
|
41
|
+
"""Fail loudly when a step's contract needs a tool the agent file does not
|
|
42
|
+
grant (BR-1: the frontmatter list is the exact allowlist)."""
|
|
43
|
+
missing = [name for name in required if name not in tools]
|
|
44
|
+
if missing:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
f"agent {agent_name!r} does not grant required tools "
|
|
47
|
+
f"{missing}; add them to its frontmatter 'tools:' allowlist")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def validate_agent_contracts(cfg, agents_dir: Path | str) -> None:
|
|
51
|
+
"""Load-time contract check (unit-11, B-2): a step configured with
|
|
52
|
+
``verdicts:`` requires ``record_verdict`` in its agent's tool allowlist.
|
|
53
|
+
|
|
54
|
+
A missing agent file is left to ``load_agent`` (which fails fast naming
|
|
55
|
+
it); this check only inspects files that exist so graph construction stays
|
|
56
|
+
a pure config pass for fake-agent test harnesses."""
|
|
57
|
+
from openhands.sdk.subagent.schema import AgentDefinition
|
|
58
|
+
|
|
59
|
+
for step in cfg.steps:
|
|
60
|
+
if step.agent is None or step.verdicts is None:
|
|
61
|
+
continue
|
|
62
|
+
path = Path(agents_dir) / f"{step.agent}.agent.md"
|
|
63
|
+
if not path.is_file():
|
|
64
|
+
continue
|
|
65
|
+
agent_def = AgentDefinition.load(path)
|
|
66
|
+
check_required_tools(agent_def.tools, (RECORD_VERDICT_TOOL,),
|
|
67
|
+
agent_name=step.agent)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def load_agent(agents_dir: Path | str, name: str, llm: LLM,
|
|
71
|
+
required_tools: tuple[str, ...] = ()) -> "Agent":
|
|
36
72
|
"""Load `<agents_dir>/<name>.agent.md` and build the Agent via the SDK
|
|
37
73
|
factory (frontmatter tool names resolved against the registry, unknown →
|
|
38
74
|
ValueError; the markdown body is delivered as the system prompt suffix).
|
|
@@ -63,6 +99,7 @@ def load_agent(agents_dir: Path | str, name: str, llm: LLM) -> "Agent":
|
|
|
63
99
|
raise ValueError(
|
|
64
100
|
f"Agent file {path} name {agent_def.name!r} does not match "
|
|
65
101
|
f"filename {stem!r}")
|
|
102
|
+
check_required_tools(agent_def.tools, required_tools, agent_name=name)
|
|
66
103
|
return agent_definition_to_factory(agent_def)(llm)
|
|
67
104
|
|
|
68
105
|
|
|
@@ -206,3 +243,58 @@ def register_sandbox_terminal(sandbox: "Sandbox", *, env: dict[str, str | None]
|
|
|
206
243
|
allowlist=allowlist))
|
|
207
244
|
|
|
208
245
|
register_tool("terminal", factory)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
class RecordVerdictAction(Action):
|
|
249
|
+
"""The structured verdict channel's input (unit-11, FR-11.2)."""
|
|
250
|
+
|
|
251
|
+
verdict: str = Field(
|
|
252
|
+
description=(
|
|
253
|
+
"The verdict keyword for this step, exactly as configured in the "
|
|
254
|
+
"pipeline's `verdicts:` map (for example APPROVED or NEEDS "
|
|
255
|
+
"CHANGES)."))
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class RecordVerdictObservation(Observation):
|
|
259
|
+
"""A visible acknowledgement: the wire invariant (D-3) requires the tool
|
|
260
|
+
result to reach the model, so this is never empty."""
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class RecordVerdictExecutor(ToolExecutor):
|
|
264
|
+
def __call__(self, action: RecordVerdictAction,
|
|
265
|
+
conversation=None) -> RecordVerdictObservation:
|
|
266
|
+
return RecordVerdictObservation.from_text(
|
|
267
|
+
f"recorded verdict: {action.verdict}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class RecordVerdictTool(
|
|
271
|
+
ToolDefinition[RecordVerdictAction, RecordVerdictObservation]):
|
|
272
|
+
"""Registers `record_verdict`; validation against the step's configured
|
|
273
|
+
verdict set happens in `Verdict.from_events`, never in the tool schema."""
|
|
274
|
+
|
|
275
|
+
@classmethod
|
|
276
|
+
def create(cls, conv_state=None, **params):
|
|
277
|
+
if params:
|
|
278
|
+
raise ValueError("RecordVerdictTool does not accept parameters")
|
|
279
|
+
return [
|
|
280
|
+
cls(
|
|
281
|
+
action_type=RecordVerdictAction,
|
|
282
|
+
observation_type=RecordVerdictObservation,
|
|
283
|
+
description=(
|
|
284
|
+
"Record this step's verdict. Call exactly once when your "
|
|
285
|
+
"work is complete, passing the configured verdict keyword. "
|
|
286
|
+
"Never state the verdict only in prose: the pipeline reads "
|
|
287
|
+
"this tool call, not your final message."),
|
|
288
|
+
executor=RecordVerdictExecutor(),
|
|
289
|
+
annotations=ToolAnnotations(
|
|
290
|
+
title=RECORD_VERDICT_TOOL,
|
|
291
|
+
readOnlyHint=True,
|
|
292
|
+
destructiveHint=False,
|
|
293
|
+
idempotentHint=True,
|
|
294
|
+
openWorldHint=False,
|
|
295
|
+
),
|
|
296
|
+
)
|
|
297
|
+
]
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
register_tool(RECORD_VERDICT_TOOL, RecordVerdictTool)
|
agentic_dev_loop/engine/cli.py
CHANGED
|
@@ -226,6 +226,9 @@ def main(argv: list[str] | None = None, *, keep_alive: bool = True) -> int:
|
|
|
226
226
|
return _run_copilot_login(argv[1:])
|
|
227
227
|
if argv and argv[0] == "copilot-models":
|
|
228
228
|
return _run_copilot_models(argv[1:])
|
|
229
|
+
if argv and argv[0] == "run-status":
|
|
230
|
+
from .run_status import _run_status
|
|
231
|
+
return _run_status(argv[1:])
|
|
229
232
|
if argv and argv[0] == "init":
|
|
230
233
|
from .onboarding.init import run_init
|
|
231
234
|
return run_init(argv[1:])
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Completion classification from the conversation event stream (unit-11).
|
|
2
|
+
|
|
3
|
+
Completion is a structural fact: an agent step completes only when it took at
|
|
4
|
+
least one verifiable action. The OpenHands SDK marks a content-bearing turn
|
|
5
|
+
with zero tool calls as FINISHED, which is how tool-less narration used to
|
|
6
|
+
read as a clean pass (planning-docs/059). This module reclassifies that shape
|
|
7
|
+
as ``FailureClass.NO_ACTIONS``; truncation semantics stay with ``stages.py``.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Iterable
|
|
13
|
+
|
|
14
|
+
from openhands.sdk.event.llm_convertible.action import ActionEvent
|
|
15
|
+
|
|
16
|
+
from agentic_dev_loop.core.failure import FailureClass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def count_actions(events: Iterable[object]) -> int:
|
|
20
|
+
"""The number of tool calls in the event stream.
|
|
21
|
+
|
|
22
|
+
Each tool call emits exactly one ``ActionEvent`` (a ``FinishAction``
|
|
23
|
+
included: the SDK's done signal is itself a tool call)."""
|
|
24
|
+
return sum(1 for event in events if isinstance(event, ActionEvent))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def classify_completion(events: Iterable[object], *,
|
|
28
|
+
finished: bool) -> FailureClass | None:
|
|
29
|
+
"""Classify a completed conversation, or return None when it took actions.
|
|
30
|
+
|
|
31
|
+
A FINISHED conversation with zero ActionEvents is NO_ACTIONS. A truncated
|
|
32
|
+
conversation keeps the existing truncation path: ``finished`` is False and
|
|
33
|
+
classification is left to the caller."""
|
|
34
|
+
events = list(events)
|
|
35
|
+
if count_actions(events) > 0:
|
|
36
|
+
return None
|
|
37
|
+
if not finished:
|
|
38
|
+
return None
|
|
39
|
+
return FailureClass.NO_ACTIONS
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def completion_evidence(events: Iterable[object]) -> str:
|
|
43
|
+
"""The classifier evidence recorded in a structural failure report."""
|
|
44
|
+
events = list(events)
|
|
45
|
+
actions = [e for e in events if isinstance(e, ActionEvent)]
|
|
46
|
+
tools = ", ".join(sorted({getattr(e, "tool_name", "?") for e in actions}))
|
|
47
|
+
return (f"event_count={len(events)} action_events={len(actions)} "
|
|
48
|
+
f"tool_calls=[{tools}]")
|
|
@@ -13,6 +13,8 @@ from __future__ import annotations
|
|
|
13
13
|
from pathlib import Path
|
|
14
14
|
|
|
15
15
|
from agentic_dev_loop.core.config import PipelineConfig, StepConfig, load_config
|
|
16
|
+
from agentic_dev_loop.core.verdict import RECORD_VERDICT_TOOL
|
|
17
|
+
from .agents import check_required_tools
|
|
16
18
|
from .hostboundary import check_orchestrator_install, check_tooling_integrity
|
|
17
19
|
from .loaders import resolve_docs
|
|
18
20
|
from .isolation import (isolation_state_dir, resolve_effective_profile,
|
|
@@ -30,12 +32,18 @@ def _validate_agents(cfg: PipelineConfig, agents_dir: Path) -> None:
|
|
|
30
32
|
The brief prompt is engine-embedded (046), so a native ``brief_inputs:``
|
|
31
33
|
step (kind 'brief') has `agent is None` and is exempt from the agents_dir
|
|
32
34
|
lookup - it never reads a vendored file."""
|
|
35
|
+
from openhands.sdk.subagent.schema import AgentDefinition
|
|
36
|
+
|
|
33
37
|
for step in cfg.steps:
|
|
34
38
|
if step.agent is None:
|
|
35
39
|
continue
|
|
36
40
|
path = agents_dir / f"{step.agent}.agent.md"
|
|
37
41
|
if not path.is_file():
|
|
38
42
|
raise FileNotFoundError(f"Agent file not found: {path}")
|
|
43
|
+
if step.verdicts is not None:
|
|
44
|
+
agent_def = AgentDefinition.load(path)
|
|
45
|
+
check_required_tools(agent_def.tools, (RECORD_VERDICT_TOOL,),
|
|
46
|
+
agent_name=step.agent)
|
|
39
47
|
|
|
40
48
|
|
|
41
49
|
def _validate_actions(cfg: PipelineConfig, registry: Registry) -> None:
|
agentic_dev_loop/engine/graph.py
CHANGED
|
@@ -19,6 +19,7 @@ from langgraph.graph import END, START, StateGraph
|
|
|
19
19
|
|
|
20
20
|
from agentic_dev_loop import viz
|
|
21
21
|
from .actions import git_worktree_clean, kill_listeners_on_port
|
|
22
|
+
from .agents import validate_agent_contracts
|
|
22
23
|
from .allowlist import resolve_engine_allowlist
|
|
23
24
|
from agentic_dev_loop.core.config import load_config
|
|
24
25
|
from .hostboundary import (check_orchestrator_install,
|
|
@@ -68,6 +69,9 @@ class LoopState(TypedDict):
|
|
|
68
69
|
state_dir: str # resolved pipeline state dir (for the commit action)
|
|
69
70
|
tooling_paths: list[str] # workspace-relative paths the commit never sweeps
|
|
70
71
|
step_verdicts: dict[str, str | None] # per-step pass/fail/None(unclear)
|
|
72
|
+
failure_class: str | None # structural failure class (unit-11)
|
|
73
|
+
mechanical_notes: Annotated[list[str], operator.add] # command/action output
|
|
74
|
+
verified_state: Annotated[dict[str, str], _merge_artifacts] # engine-verified facts
|
|
71
75
|
retry_target: str | None # where the bump node loops back to
|
|
72
76
|
thread_id: str
|
|
73
77
|
brief_hashes: Annotated[dict[str, str], _merge_artifacts] # step name -> input hash (033)
|
|
@@ -78,6 +82,7 @@ def build_graph(runtime: Runtime, checkpointer=None):
|
|
|
78
82
|
"""Compile the pipeline's state graph: one node per step (agent, command,
|
|
79
83
|
or action), plus the bump retry node and the success terminal. Unknown
|
|
80
84
|
action names fail here, at build time, not mid-run."""
|
|
85
|
+
validate_agent_contracts(runtime.cfg, runtime.agents_dir)
|
|
81
86
|
known_actions = (set(runtime.registry.actions)
|
|
82
87
|
| set(runtime.registry.sandbox_actions))
|
|
83
88
|
for step in runtime.cfg.steps:
|
|
@@ -235,6 +240,9 @@ def _run_pipeline(cfg, base, registry, state_dir, max_attempts, args, viz_bus,
|
|
|
235
240
|
"state_dir": str(state_dir),
|
|
236
241
|
"tooling_paths": tooling_paths,
|
|
237
242
|
"step_verdicts": {},
|
|
243
|
+
"failure_class": None,
|
|
244
|
+
"mechanical_notes": [],
|
|
245
|
+
"verified_state": {},
|
|
238
246
|
"retry_target": None,
|
|
239
247
|
"thread_id": thread_id,
|
|
240
248
|
"brief_hashes": {},
|
|
@@ -362,6 +370,9 @@ def _run_pipeline(cfg, base, registry, state_dir, max_attempts, args, viz_bus,
|
|
|
362
370
|
f"{final.get('failed_step', 'unknown')!r}; see "
|
|
363
371
|
f"{state_dir}/last-error.md",
|
|
364
372
|
flush=True)
|
|
373
|
+
if final.get("failure_class"):
|
|
374
|
+
print(f"[{final['subject']}] structural failure class: "
|
|
375
|
+
f"{final['failure_class']}", flush=True)
|
|
365
376
|
else:
|
|
366
377
|
print(f"[{final['subject']}] FAILED after {final['attempt']} "
|
|
367
378
|
f"attempt(s).", flush=True)
|
agentic_dev_loop/engine/llm.py
CHANGED
|
@@ -6,7 +6,11 @@ import os
|
|
|
6
6
|
from openhands.sdk import LLM
|
|
7
7
|
|
|
8
8
|
from agentic_dev_loop.core.config import LlmConfig
|
|
9
|
-
from agentic_dev_loop.engine.provider_wire import adapter_for
|
|
9
|
+
from agentic_dev_loop.engine.provider_wire import (adapter_for,
|
|
10
|
+
merge_choices,
|
|
11
|
+
validate_response_shape,
|
|
12
|
+
validate_string_tool_results,
|
|
13
|
+
validate_tool_call_pairing)
|
|
10
14
|
from agentic_dev_loop.engine.secrets import (
|
|
11
15
|
SecretResolutionError, resolve_secret_spec)
|
|
12
16
|
|
|
@@ -62,6 +66,8 @@ if not getattr(_ORIGINAL_FORMAT_MESSAGES, "_agentic_dev_loop_reasoning_repair",
|
|
|
62
66
|
d.pop("reasoning_content", None)
|
|
63
67
|
if any(getattr(m, "reasoning_content", None) for m in messages):
|
|
64
68
|
raise _reasoning_stripped_error(self)
|
|
69
|
+
validate_string_tool_results(
|
|
70
|
+
messages, out, forced=bool(self.force_string_serializer))
|
|
65
71
|
return out
|
|
66
72
|
|
|
67
73
|
last = None
|
|
@@ -79,12 +85,62 @@ if not getattr(_ORIGINAL_FORMAT_MESSAGES, "_agentic_dev_loop_reasoning_repair",
|
|
|
79
85
|
d["reasoning_content"] = last
|
|
80
86
|
elif rc:
|
|
81
87
|
d["reasoning_content"] = rc
|
|
88
|
+
validate_string_tool_results(
|
|
89
|
+
messages, out, forced=bool(self.force_string_serializer))
|
|
82
90
|
return out
|
|
83
91
|
|
|
84
92
|
_config_gated_reasoning_serializer._agentic_dev_loop_reasoning_repair = True
|
|
85
93
|
LLM.format_messages_for_llm = _config_gated_reasoning_serializer
|
|
86
94
|
|
|
87
95
|
|
|
96
|
+
_ORIGINAL_COMPLETION = LLM.completion
|
|
97
|
+
_ORIGINAL_RESPONSES = LLM.responses
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _wire_checked_completion(self, messages, *args, **kwargs):
|
|
101
|
+
"""D-2 before the request, D-1 on the surfaced response (chat path)."""
|
|
102
|
+
validate_tool_call_pairing(messages)
|
|
103
|
+
response = _ORIGINAL_COMPLETION(self, messages, *args, **kwargs)
|
|
104
|
+
validate_response_shape(response)
|
|
105
|
+
return response
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _wire_checked_responses(self, messages, *args, **kwargs):
|
|
109
|
+
"""D-2 before the request; D-1 where the Responses shape exposes it."""
|
|
110
|
+
validate_tool_call_pairing(messages)
|
|
111
|
+
response = _ORIGINAL_RESPONSES(self, messages, *args, **kwargs)
|
|
112
|
+
validate_response_shape(response)
|
|
113
|
+
return response
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if not getattr(LLM.completion, "_agentic_dev_loop_wire_checked", False):
|
|
117
|
+
_wire_checked_completion._agentic_dev_loop_wire_checked = True
|
|
118
|
+
LLM.completion = _wire_checked_completion
|
|
119
|
+
|
|
120
|
+
if not getattr(LLM.responses, "_agentic_dev_loop_wire_checked", False):
|
|
121
|
+
_wire_checked_responses._agentic_dev_loop_wire_checked = True
|
|
122
|
+
LLM.responses = _wire_checked_responses
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# The Copilot gateway splits Anthropic-model turns across multiple choices
|
|
126
|
+
# (narration-only choice + one choice per tool call); the SDK reads only
|
|
127
|
+
# choices[0] inside LLM.completion, so the merge must happen BEFORE that read,
|
|
128
|
+
# at the _transport_call boundary where the litellm ModelResponse is built.
|
|
129
|
+
_ORIGINAL_TRANSPORT_CALL = LLM._transport_call
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _merge_checked_transport_call(self, **kwargs):
|
|
133
|
+
ret = _ORIGINAL_TRANSPORT_CALL(self, **kwargs)
|
|
134
|
+
if getattr(self, "_agentic_dev_loop_merge_choices", False):
|
|
135
|
+
merge_choices(ret)
|
|
136
|
+
return ret
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
if not getattr(LLM._transport_call, "_agentic_dev_loop_merge_checked", False):
|
|
140
|
+
_merge_checked_transport_call._agentic_dev_loop_merge_checked = True
|
|
141
|
+
LLM._transport_call = _merge_checked_transport_call
|
|
142
|
+
|
|
143
|
+
|
|
88
144
|
def resolve_api_key(cfg: LlmConfig, environ=None) -> str:
|
|
89
145
|
"""Resolve the configured host-side key: the existing env mechanism, or
|
|
90
146
|
the file-backed secret spec. Missing values fail fast naming the fix."""
|
|
@@ -120,4 +176,6 @@ def build_llm(cfg: LlmConfig, reasoning_effort: str | None = None) -> LLM:
|
|
|
120
176
|
force_string_serializer=adapter.force_string_serializer if adapter else None,
|
|
121
177
|
)
|
|
122
178
|
llm._agentic_dev_loop_reasoning_effort = effort
|
|
179
|
+
llm._agentic_dev_loop_merge_choices = (
|
|
180
|
+
adapter.merge_choices if adapter else False)
|
|
123
181
|
return llm
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: qa
|
|
3
|
-
tools: [terminal]
|
|
3
|
+
tools: [terminal, record_verdict]
|
|
4
4
|
---
|
|
5
5
|
# QA Agent
|
|
6
6
|
|
|
@@ -52,10 +52,11 @@ QA section marks "QA executes first-hand".
|
|
|
52
52
|
4. Probe only the uncovered surface per the rules below, then stop.
|
|
53
53
|
5. State your report in your **final response** - the pipeline captures it
|
|
54
54
|
as your qa report. Give every acceptance check a status of `PASS`,
|
|
55
|
-
`FAIL`, `UNVERIFIABLE`, or `ENV`, then
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
55
|
+
`FAIL`, `UNVERIFIABLE`, or `ENV`, then call `record_verdict` with your verdict
|
|
56
|
+
(PASS, FAIL, UNVERIFIABLE, or ENV). If any check is `UNVERIFIABLE`, add
|
|
57
|
+
an `## UNVERIFIABLE` section listing each skipped check and the blocking
|
|
58
|
+
reason. If the verification environment itself is
|
|
59
|
+
broken, call `record_verdict(verdict="ENV")` and add an `## ENV` section (see ENV below).
|
|
59
60
|
|
|
60
61
|
## ENV
|
|
61
62
|
|
|
@@ -76,7 +77,7 @@ than the product code, the verdict is `ENV`.
|
|
|
76
77
|
each environmental blocker with its evidence (command, output, expected).
|
|
77
78
|
|
|
78
79
|
**Anti-livelock rule.** If the same environmental blocker persists across
|
|
79
|
-
2 attempts,
|
|
80
|
+
2 attempts, call `record_verdict(verdict="ENV")` instead of further workarounds. Do not burn
|
|
80
81
|
turns building synthetic fixtures or debugging the harness.
|
|
81
82
|
|
|
82
83
|
## PROBE-PLAN RULES
|
|
@@ -167,15 +168,21 @@ turns building synthetic fixtures or debugging the harness.
|
|
|
167
168
|
|
|
168
169
|
## OUTPUT
|
|
169
170
|
|
|
170
|
-
|
|
171
|
-
check table and findings
|
|
172
|
-
citation (`covered by <test>` or `gate
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
171
|
+
Call `record_verdict` exactly once when your report is ready, then write the
|
|
172
|
+
check table and findings in your final response for humans. Each check row
|
|
173
|
+
carries either a probe result or a citation (`covered by <test>` or `gate
|
|
174
|
+
record`). The verdict values are:
|
|
175
|
+
|
|
176
|
+
- `PASS` when every reachable acceptance check passes. Checks that could
|
|
177
|
+
not be verified are recorded as `UNVERIFIABLE` in the report, not treated
|
|
178
|
+
as failures.
|
|
179
|
+
- `UNVERIFIABLE` when the acceptance checks cannot be touched at all, none
|
|
180
|
+
of them can be verified in this sandbox, or the plan's headline feature
|
|
181
|
+
cannot be verified here. Name the headline feature and its blocking reason
|
|
182
|
+
in the `## UNVERIFIABLE` section.
|
|
183
|
+
- `ENV` when the verification environment itself is broken mid-run and
|
|
184
|
+
PASS/FAIL/UNVERIFIABLE cannot be determined (see ENV above).
|
|
185
|
+
- `FAIL` on any genuine defect, as always.
|
|
186
|
+
|
|
187
|
+
The tool call is the routing channel; the prose no longer carries a
|
|
188
|
+
`VERDICT:` line.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: review
|
|
3
|
-
tools: [terminal]
|
|
3
|
+
tools: [terminal, record_verdict]
|
|
4
4
|
---
|
|
5
5
|
# Review Agent
|
|
6
6
|
|
|
@@ -17,16 +17,12 @@ repo's README.
|
|
|
17
17
|
artifact hygiene, single gated commit point, fail-fast config)?
|
|
18
18
|
- Do the tests pin the new behavior without touching the network
|
|
19
19
|
(FakeAgents in tests/conftest.py is the only permitted stub)?
|
|
20
|
-
4.
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
VERDICT: NEEDS CHANGES
|
|
28
|
-
|
|
29
|
-
List specific issues with file paths and suggestions.
|
|
20
|
+
4. Call the `record_verdict` tool exactly once with `APPROVED` or
|
|
21
|
+
`NEEDS CHANGES` when your review is complete. The tool call is the routing
|
|
22
|
+
channel; prose is not.
|
|
23
|
+
5. State your findings in your **final response** - the pipeline captures it
|
|
24
|
+
as your review for humans. List specific issues with file paths and
|
|
25
|
+
suggestions. Do not put a `VERDICT:` line in the prose.
|
|
30
26
|
|
|
31
27
|
## RULES
|
|
32
28
|
|
|
@@ -48,6 +44,7 @@ List specific issues with file paths and suggestions.
|
|
|
48
44
|
|
|
49
45
|
## OUTPUT
|
|
50
46
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
47
|
+
Call `record_verdict` with `verdict="APPROVED"` or `verdict="NEEDS CHANGES"`
|
|
48
|
+
as soon as your review is complete. The pipeline reads that tool call and
|
|
49
|
+
nothing else. Your final response carries the findings for humans; there is
|
|
50
|
+
no `VERDICT:` line contract.
|