agentic-dev-loop 1.0.0a14__tar.gz → 1.0.0a15__tar.gz

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentic-dev-loop
-Version: 1.0.0a14
+Version: 1.0.0a15
 Summary: Turn an LLM into a gated, sandboxed, resumable construction loop over your codebase.
 Author: Luke Wilson
 License-Expression: Apache-2.0

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "agentic-dev-loop"
-version = "1.0.0a14"
+version = "1.0.0a15"
 description = "Turn an LLM into a gated, sandboxed, resumable construction loop over your codebase."
 readme = "README.md"
 requires-python = ">=3.12"

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/src/agentic_dev_loop/core/config.py

@@ -19,6 +19,7 @@ class VerdictSpec(BaseModel):
 
     pass_: str = Field(alias="pass")
     fail: str = Field(alias="fail")
+    unverifiable: str | None = None
 
 
 class Route(BaseModel):
@@ -131,6 +132,7 @@ class StepConfig(BaseModel):
     secrets: list[str] = []  # declared ref names; action steps reject
     on_pass: RouteTarget | None = None
     on_fail: RouteTarget | None = None
+    on_unverifiable: RouteTarget | None = None
 
     @property
     def kind(self) -> str:
@@ -286,7 +288,7 @@ class PipelineConfig(BaseModel):
                 raise ValueError(f"step name {n!r} collides with a reserved terminal")
         valid = set(names) | TERMINALS
         for step in self.steps:
-            for route in (step.on_pass, step.on_fail):
+            for route in (step.on_pass, step.on_fail, step.on_unverifiable):
                 target = route_target(route)
                 if target is not None and target not in valid:
                     raise ValueError(

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/src/agentic_dev_loop/engine/stages.py

@@ -254,8 +254,13 @@ def _unclear_note(runtime: Runtime, step: StepConfig, attempt: int,
     captured response when it exists, so real findings are never swallowed."""
     expected = ""
     if step.verdicts is not None:
-        expected = (f" (expected `VERDICT: {step.verdicts.pass_}` or "
-                    f"`VERDICT: {step.verdicts.fail}` on its own line)")
+        if step.verdicts.unverifiable is not None:
+            expected = (f" (expected `VERDICT: {step.verdicts.pass_}`, "
+                        f"`VERDICT: {step.verdicts.fail}`, or "
+                        f"`VERDICT: {step.verdicts.unverifiable}` on its own line)")
+        else:
+            expected = (f" (expected `VERDICT: {step.verdicts.pass_}` or "
+                        f"`VERDICT: {step.verdicts.fail}` on its own line)")
     if truncated:
         lead = (f"{step.name} agent was cut short on attempt {attempt} "
                 f"(hit the {max_iterations}-turn limit or was stopped "
@@ -282,6 +287,8 @@ def make_agent_node(runtime: Runtime, step: StepConfig):
     verdicts_map = None
     if step.verdicts is not None:
         verdicts_map = {step.verdicts.pass_: "pass", step.verdicts.fail: "fail"}
+        if step.verdicts.unverifiable is not None:
+            verdicts_map[step.verdicts.unverifiable] = "unverifiable"
     captures = step.artifact is not None or bool(step.produces)
 
     def node(state: dict) -> dict:
@@ -334,6 +341,10 @@ def make_agent_node(runtime: Runtime, step: StepConfig):
                 print(f"[{subject}] {step.name} VERDICT: PASS on attempt {attempt}.",
                       flush=True)
                 return {"step_verdicts": merged, "failed": False}
+            if verdict == "unverifiable":
+                print(f"[{subject}] {step.name} qa verdict: UNVERIFIABLE "
+                      f"(moves on).", flush=True)
+                return {"step_verdicts": merged, "failed": False}
             print(f"[{subject}] {step.name} verdict: fail on attempt {attempt}.",
                   flush=True)
             update = {"step_verdicts": merged,
@@ -519,6 +530,10 @@ def make_router(runtime: Runtime, step: StepConfig):
         verdict = state["step_verdicts"].get(step.name)
         if verdict == "pass":
             return resolve(step.on_pass)
+        if verdict == "unverifiable":
+            if step.on_unverifiable is not None:
+                return resolve(step.on_unverifiable)
+            return resolve(step.on_pass)
         if verdict == "fail":
             if step.on_fail is None:
                 return END

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/src/agentic_dev_loop/viz/server.py

@@ -472,6 +472,7 @@ PAGE = """<!doctype html>
            background: #f0f0f0; color: #333; }
   .badge-pass { background: #e8f5e9; color: #2e7d32; }
   .badge-fail { background: #ffebee; color: #c62828; }
+  .badge-unverifiable { background: #ede7f6; color: #4527a0; }
   .badge-unclear { background: #fff8e1; color: #f57f17; }
   #stages { flex: 1 1 auto; min-height: 0; overflow-y: auto; display: flex;
             flex-direction: column; gap: 8px; }
@@ -487,6 +488,7 @@ PAGE = """<!doctype html>
   .status-pass { color: #2e7d32; }
   .status-fail { color: #b26a00; }
   .status-fatal { color: #c62828; }
+  .status-unverifiable { color: #7e57c2; }
   .console-wrap { max-height: 240px; overflow-y: auto; background: #1e1e1e; }
   .stage-console { margin: 0; padding: 8px 10px; background: #1e1e1e;
                    color: #e0e0e0; font: 12px/1.5 ui-monospace, SFMono-Regular,
@@ -543,6 +545,7 @@ PAGE = """<!doctype html>
   var currentVerdicts = {};
   var verdictFail = new Set();
   var verdictPass = new Set();
+  var verdictUnverifiable = new Set();
   var fatalNodes = new Set();
   var maxAttempts = null;
   var attempt = null;
@@ -610,6 +613,7 @@ PAGE = """<!doctype html>
     if (nodeStatus[n] === "fatal") { return "fatal"; }
     if (verdictFail.has(n)) { return "fail"; }
     if (verdictPass.has(n)) { return "pass"; }
+    if (verdictUnverifiable.has(n)) { return "unverifiable"; }
     return nodeStatus[n] || "pending";
   }
 
@@ -622,6 +626,7 @@ PAGE = """<!doctype html>
       b.className = "badge";
       if (v === "pass") { b.className += " badge-pass"; }
       else if (v === "fail") { b.className += " badge-fail"; }
+      else if (v === "unverifiable") { b.className += " badge-unverifiable"; }
       else { b.className += " badge-unclear"; }
       b.textContent = step + ": " + (v === null || v === undefined ? "no verdict" : v);
       box.appendChild(b);
@@ -632,6 +637,7 @@ PAGE = """<!doctype html>
     verdictSteps.clear();
     verdictPass.clear();
     verdictFail.clear();
+    verdictUnverifiable.clear();
     currentVerdicts = {};
     nodeStatus = {};
     stageWord = {};
@@ -653,6 +659,7 @@ PAGE = """<!doctype html>
       "classDef pass fill:#2e7d32,stroke:#2e7d32,color:#fff",
       "classDef fail fill:#ffb300,stroke:#ffb300,color:#fff",
       "classDef fatal fill:#c62828,stroke:#c62828,color:#fff",
+      "classDef unverifiable fill:#7e57c2,stroke:#7e57c2,color:#fff",
       "classDef pending fill:#f2f0ff,stroke:#555,color:#000"
     ];
     var classLines = Object.keys(nodeStatus).filter(function (n) {
@@ -748,6 +755,7 @@ PAGE = """<!doctype html>
       var v = currentVerdicts[node];
       if (v === "pass") { suffix = " (verdict: PASS)"; }
       else if (v === "fail") { suffix = " (verdict: FAIL)"; }
+      else if (v === "unverifiable") { suffix = " (verdict: UNVERIFIABLE)"; }
     }
     status.textContent = (word ? " " + word : "") + suffix;
     header.className = "stage-header status-" + statusOf(node);
@@ -1064,9 +1072,10 @@ PAGE = """<!doctype html>
           Object.keys(ev.step_verdicts).forEach(function (k) {
             verdictSteps.add(k);
             var v = ev.step_verdicts[k];
-            if (v === "pass") { verdictPass.add(k); verdictFail.delete(k); }
-            else if (v === "fail") { verdictFail.add(k); verdictPass.delete(k); }
-            else { verdictPass.delete(k); verdictFail.delete(k); }
+            if (v === "pass") { verdictPass.add(k); verdictFail.delete(k); verdictUnverifiable.delete(k); }
+            else if (v === "fail") { verdictFail.add(k); verdictPass.delete(k); verdictUnverifiable.delete(k); }
+            else if (v === "unverifiable") { verdictUnverifiable.add(k); verdictPass.delete(k); verdictFail.delete(k); }
+            else { verdictPass.delete(k); verdictFail.delete(k); verdictUnverifiable.delete(k); }
             currentVerdicts[k] = v;
           });
           updateBadges();

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/src/agentic_dev_loop.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentic-dev-loop
-Version: 1.0.0a14
+Version: 1.0.0a15
 Summary: Turn an LLM into a gated, sandboxed, resumable construction loop over your codebase.
 Author: Luke Wilson
 License-Expression: Apache-2.0

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/src/agentic_dev_loop.egg-info/SOURCES.txt

@@ -57,6 +57,7 @@ tests/test_outputs.py
 tests/test_outputs_pbt.py
 tests/test_proc.py
 tests/test_publish.py
+tests/test_qa_prompt.py
 tests/test_registry.py
 tests/test_repo_ascii.py
 tests/test_results.py

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/tests/test_config.py

@@ -195,6 +195,28 @@ def test_unknown_verdicts_key_rejected(tmp_path):
         _load(tmp_path, data)
 
 
+def test_three_keyword_verdict_and_on_unverifiable_load(tmp_path):
+    data = _base()
+    data["steps"][0] = {
+        "name": "qa", "agent": "qa", "artifact": "qa-report.md",
+        "verdicts": {"pass": "PASS", "fail": "FAIL",
+                     "unverifiable": "UNVERIFIABLE"},
+        "on_pass": "success", "on_unverifiable": "success",
+    }
+    step = _load(tmp_path, data).steps[0]
+    assert step.verdicts.pass_ == "PASS"
+    assert step.verdicts.fail == "FAIL"
+    assert step.verdicts.unverifiable == "UNVERIFIABLE"
+    assert step.on_unverifiable == "success"
+
+
+def test_dangling_on_unverifiable_rejected(tmp_path):
+    data = _base()
+    data["steps"][0]["on_unverifiable"] = "nope"
+    with pytest.raises(ValidationError, match="unknown route target 'nope'"):
+        _load(tmp_path, data)
+
+
 def test_dangling_on_pass_rejected(tmp_path):
     data = _base()
     data["steps"][0]["on_pass"] = "nope"

agentic_dev_loop-1.0.0a15/tests/test_qa_prompt.py

@@ -0,0 +1,29 @@
+"""QA prompt contract (024): the no-substitution rule and the three-way
+verdict rules live in the dogfood copy of sdk_agents/qa.agent.md."""
+
+from pathlib import Path
+
+QA_PROMPT = Path(__file__).parent.parent / "agentic-dev-loop" / "sdk_agents" / "qa.agent.md"
+
+
+def _text() -> str:
+    return QA_PROMPT.read_text(encoding="utf-8")
+
+
+def test_no_substitution_rule_present():
+    text = _text()
+    assert "Never substitute unrelated generic probes" in text
+    assert "UNVERIFIABLE" in text
+    assert "blocking reason" in text
+
+
+def test_unverifiable_verdict_rules_present():
+    text = _text()
+    assert "VERDICT: PASS" in text
+    assert "VERDICT: UNVERIFIABLE" in text
+    assert "VERDICT: FAIL" in text
+    assert "## UNVERIFIABLE" in text
+
+
+def test_inconclusive_budget_rule_retained():
+    assert "INCONCLUSIVE" in _text()

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/tests/test_stages.py

@@ -251,6 +251,36 @@ class TestAgentNode:
         make_agent_node(runtime, step)(make_state())
         assert fake_agents.calls[0]["max_iterations"] == 200
 
+    def test_unverifiable_verdict_records_and_moves_on(self, runtime,
+                                                       fake_agents, capsys):
+        step = make_cfg([
+            {"name": "dev", "agent": "dev", "on_pass": "qa"},
+            {"name": "qa", "agent": "qa", "artifact": "qa-report.md",
+             "verdicts": {"pass": "PASS", "fail": "FAIL",
+                          "unverifiable": "UNVERIFIABLE"},
+             "on_pass": "commit", "on_fail": {"goto": "dev", "retry": True},
+             "on_unverifiable": "commit"},
+            {"name": "commit", "action": "commit", "on_pass": "success"}]).steps[1]
+        fake_agents.set("qa", [("write", "qa-report.md", "VERDICT: UNVERIFIABLE")])
+        update = make_agent_node(runtime, step)(make_state())
+        assert update["step_verdicts"] == {"qa": "unverifiable"}
+        assert update["failed"] is False
+        assert "notes" not in update
+        assert "UNVERIFIABLE (moves on)" in capsys.readouterr().out
+
+    def test_unclear_note_mentions_third_keyword(self, runtime, fake_agents):
+        step = make_cfg([
+            {"name": "qa", "agent": "qa", "artifact": "qa-report.md",
+             "verdicts": {"pass": "PASS", "fail": "FAIL",
+                          "unverifiable": "UNVERIFIABLE"},
+             "on_pass": "success"}]).steps[0]
+        fake_agents.set("qa", [("write", "qa-report.md", "undecided")])
+        update = make_agent_node(runtime, step)(make_state())
+        (note,) = update["notes"]
+        assert "VERDICT: PASS" in note
+        assert "VERDICT: FAIL" in note
+        assert "VERDICT: UNVERIFIABLE" in note
+
 
 class TestCommandNode:
     def _node(self, runtime, steps, idx=0):
@@ -464,6 +494,35 @@ class TestRouter:
         router = make_router(runtime, cfg.steps[1])
         assert router(make_state(step_verdicts={"c": "fail"})) == "a"
 
+    def test_unverifiable_routes_on_unverifiable(self, runtime):
+        cfg = make_cfg([
+            {"name": "dev", "agent": "dev", "on_pass": "qa"},
+            {"name": "qa", "agent": "qa", "artifact": "qa-report.md",
+             "verdicts": {"pass": "PASS", "fail": "FAIL",
+                          "unverifiable": "UNVERIFIABLE"},
+             "on_pass": "commit", "on_fail": {"goto": "dev", "retry": True},
+             "on_unverifiable": "commit"},
+            {"name": "commit", "action": "commit", "on_pass": "success"},
+        ])
+        runtime = dataclasses.replace(runtime, cfg=cfg)
+        router = make_router(runtime, cfg.steps[1])
+        assert router(make_state(step_verdicts={"qa": "unverifiable"})) == "commit"
+
+    def test_unverifiable_falls_back_to_on_pass_and_never_retries(self, runtime):
+        cfg = make_cfg([
+            {"name": "dev", "agent": "dev", "on_pass": "qa"},
+            {"name": "qa", "agent": "qa", "artifact": "qa-report.md",
+             "verdicts": {"pass": "PASS", "fail": "FAIL",
+                          "unverifiable": "UNVERIFIABLE"},
+             "on_pass": "commit", "on_fail": {"goto": "dev", "retry": True}},
+            {"name": "commit", "action": "commit", "on_pass": "success"},
+        ])
+        runtime = dataclasses.replace(runtime, cfg=cfg)
+        router = make_router(runtime, cfg.steps[1])
+        # unset on_unverifiable -> on_pass; never on_fail's bump retry
+        assert router(make_state(attempt=1,
+                                 step_verdicts={"qa": "unverifiable"})) == "commit"
+
 
 class TestBumpNode:
     def test_consumes_attempt_resets_verdicts_rebuilds_cleans(self, runtime):

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/tests/test_verdict.py

@@ -3,10 +3,11 @@ Result API (BR-2): decisions route as `Result.ok(Verdict)`; verdict lines
 without a decision, or conflicting decisions, are typed errors that feed
 the attempt-bounded retry path - never exceptions, never silent routing."""
 
-from agentic_dev_loop.core.verdict import Verdict, VerdictError
+from agentic_dev_loop.core.verdict import Verdict, VerdictError, parse_verdict
 
 REVIEW_VERDICTS = {"APPROVED": "approved", "NEEDS CHANGES": "needs_changes"}
 TEST_VERDICTS = {"PASS": "pass", "FAIL": "fail"}
+QA_VERDICTS = {"PASS": "pass", "FAIL": "fail", "UNVERIFIABLE": "unverifiable"}
 
 
 def _parse(text, verdicts):
@@ -169,6 +170,24 @@ class TestVerdictParsing:
     def test_not_unknown_keyword_rejected(self):
         _err("VERDICT: NOT MAYBE", REVIEW_VERDICTS, VerdictError.AMBIGUOUS)
 
+    def test_parse_unverifiable(self):
+        _ok("VERDICT: UNVERIFIABLE", QA_VERDICTS, "unverifiable")
+        _ok("VERDICT: UNVERIFIABLE - no docker in the sandbox",
+            QA_VERDICTS, "unverifiable")
+        _ok("verdict: unverifiable", QA_VERDICTS, "unverifiable")
+
+    def test_parse_verdict_unverifiable_returns_canonical_value(self):
+        assert parse_verdict("VERDICT: UNVERIFIABLE", QA_VERDICTS) == "unverifiable"
+
+    def test_negated_unverifiable_stays_ambiguous_on_three_key_sets(self):
+        _err("VERDICT: NOT UNVERIFIABLE", QA_VERDICTS, VerdictError.AMBIGUOUS)
+
+    def test_conflicting_pass_and_unverifiable_is_ambiguous(self):
+        _err("VERDICT: PASS\nVERDICT: UNVERIFIABLE", QA_VERDICTS,
+             VerdictError.AMBIGUOUS)
+        assert parse_verdict("VERDICT: PASS\nVERDICT: UNVERIFIABLE",
+                             QA_VERDICTS) == "retry"
+
 
 class TestConflictingDecisions:
     """BR-2.4: conflicting *decisions* are never silently resolved."""

{agentic_dev_loop-1.0.0a14 → agentic_dev_loop-1.0.0a15}/tests/test_viz.py

@@ -734,6 +734,15 @@ class TestServer:
         assert "verdictFail" in PAGE
         assert "verdictPass" in PAGE
 
+    def test_page_constant_has_unverifiable_verdict_support(self):
+        assert "classDef unverifiable" in PAGE
+        assert ".badge-unverifiable" in PAGE
+        assert ".status-unverifiable" in PAGE
+        assert "verdictUnverifiable" in PAGE
+        # the new value maps to badge-unverifiable, not the unclear fallthrough
+        assert 'else if (v === "unverifiable")' in PAGE
+        assert " (verdict: UNVERIFIABLE)" in PAGE
+
     def test_page_constant_color_palette(self):
         assert 'classDef running fill:#1976d2' in PAGE
         assert 'classDef fail fill:#ffb300' in PAGE
@@ -1097,6 +1106,19 @@ main();
         assert snaps[1]["classes"]["test"] == "pass"
         assert snaps[1]["classes"]["review"] == "fail"
 
+    def test_unverifiable_verdict_badge_and_class(self, node, tmp_path):
+        events = [
+            {"type": "run_started", "subject": "s", "max_attempts": 5,
+             "attempt": 1, "thread_id": "t"},
+            {"type": "node_finished", "node": "review", "ok": True},
+            {"type": "state", "attempt": 1,
+             "step_verdicts": {"review": "unverifiable"}, "failed": False},
+        ]
+        snaps = self._run_page(node, events, tmp_path)
+
+        assert snaps[0]["badges"] == ["review: unverifiable"]
+        assert snaps[0]["classes"]["review"] == "unverifiable"
+
     def test_copy_button_copies_console_text(self, node, tmp_path):
         events = [
             {"type": "run_started", "subject": "s", "max_attempts": 5,