drydock-cli 0.7.2__tar.gz → 0.7.3__tar.gz

{drydock_cli-0.7.2 → drydock_cli-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: drydock-cli
-Version: 0.7.2
+Version: 0.7.3
 Summary: Drydock — nautical CLI coding agent
 Project-URL: Homepage, https://github.com/fbobe321/drydock
 Project-URL: Repository, https://github.com/fbobe321/drydock

{drydock_cli-0.7.2 → drydock_cli-0.7.3}/PRD.md

@@ -178,3 +178,17 @@ Show the user what the agent has planned in its execution pipeline.
 
 ### P4: Support more LLM backends
 Test with Claude, GPT-4, Gemini to understand which improvements are model-specific.
+
+### Phase 7: Real Test-Driven Fixes (Mar 26)
+
+Shifted to test-driven development with real vLLM backend — no more mocks for critical bugs.
+
+| Issue | Test Result (before fix) | Fix | Test Result (after fix) |
+|-------|------------------------|-----|----------------------|
+| Circuit breaker fires but model keeps calling | FAILED: 20 bash calls, 17 breaker fires ignored | Force-stop conversation after 3 consecutive breaker fires + break tool loop | PASSED: stops within 3 calls |
+
+**Testing methodology changed:**
+- All critical tests run against real vLLM at localhost:8000
+- Tests must FAIL first (proving the bug exists)
+- Then fix code, re-run until PASS
+- 166 total tests (155 mock + 11 real backend)

{drydock_cli-0.7.2 → drydock_cli-0.7.3}/drydock/core/agent_loop.py

@@ -191,6 +191,7 @@ class AgentLoop:
         # Circuit breaker: track tool call signatures to prevent exact repeats
         # Key: hash(tool_name + args), Value: (count, last_result_snippet)
         self._tool_call_history: dict[str, tuple[int, str]] = {}
+        self._consecutive_circuit_breaker_fires: int = 0
 
         system_prompt = get_universal_system_prompt(
             self.tool_manager, self.config, self.skill_manager, self.agent_manager
@@ -512,11 +513,16 @@ class AgentLoop:
                 self.stats.steps += 1
                 user_cancelled = False
                 try:
+                    force_stopped = False
                     async for event in self._perform_llm_turn():
                         if is_user_cancellation_event(event):
                             user_cancelled = True
+                        if isinstance(event, AssistantEvent) and event.stopped_by_middleware:
+                            force_stopped = True
                         yield event
                         await self._save_messages()
+                    if force_stopped:
+                        return
                     # Reset API error count on successful turn
                     api_error_count = 0
                 except (RuntimeError, AgentLoopLLMResponseError) as e:
@@ -896,6 +902,29 @@ class AgentLoop:
     ) -> AsyncGenerator[ToolResultEvent | ToolStreamEvent]:
         # Circuit breaker: block exact duplicate calls after 2 attempts
         if blocked := self._circuit_breaker_check(tool_call):
+            self._consecutive_circuit_breaker_fires += 1
+
+            if self._consecutive_circuit_breaker_fires >= 3:
+                # Model is ignoring the circuit breaker — force stop
+                force_msg = (
+                    f"FORCED STOP: You ignored the circuit breaker {self._consecutive_circuit_breaker_fires} times. "
+                    f"The session is being terminated because you keep running the same command. "
+                    f"You MUST try a completely different approach."
+                )
+                yield ToolResultEvent(
+                    tool_name=tool_call.tool_name,
+                    tool_class=tool_call.tool_class,
+                    error=force_msg,
+                    tool_call_id=tool_call.call_id,
+                )
+                self._handle_tool_response(tool_call, force_msg, "failure")
+                # Inject stop signal into messages
+                yield AssistantEvent(
+                    content=f"\n\n[{force_msg}]\n",
+                    stopped_by_middleware=True,
+                )
+                return
+
             yield ToolResultEvent(
                 tool_name=tool_call.tool_name,
                 tool_class=tool_call.tool_class,
@@ -904,6 +933,9 @@ class AgentLoop:
             )
             self._handle_tool_response(tool_call, blocked, "failure")
             return
+        else:
+            # Reset consecutive fires when a non-blocked call happens
+            self._consecutive_circuit_breaker_fires = 0
 
         try:
             tool_instance = self.tool_manager.get(tool_call.tool_name)
@@ -1060,10 +1092,14 @@ class AgentLoop:
 
     async def _handle_tool_calls(
         self, resolved: ResolvedMessage
-    ) -> AsyncGenerator[ToolCallEvent | ToolResultEvent | ToolStreamEvent]:
+    ) -> AsyncGenerator[ToolCallEvent | ToolResultEvent | ToolStreamEvent | AssistantEvent]:
         async for event in self._emit_failed_tool_events(resolved.failed_calls):
             yield event
         for tool_call in resolved.tool_calls:
+            # Stop processing more tool calls if circuit breaker force-stopped
+            if self._consecutive_circuit_breaker_fires >= 3:
+                break
+
             yield ToolCallEvent(
                 tool_name=tool_call.tool_name,
                 tool_class=tool_call.tool_class,

{drydock_cli-0.7.2 → drydock_cli-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "drydock-cli"
-version = "0.7.2"
+version = "0.7.3"
 description = "Drydock — nautical CLI coding agent"
 readme = "README.md"
 requires-python = ">=3.12"

{drydock_cli-0.7.2 → drydock_cli-0.7.3}/scripts/deploy_to_github.sh

@@ -37,7 +37,7 @@ AUTH_URL="https://${GITHUB_TOKEN}@github.com/fbobe321/drydock.git"
 
 log "Running regression tests..."
 cd "$DRYDOCK_SRC"
-if ! python3 -m pytest tests/test_drydock_regression.py tests/test_drydock_tasks.py tests/test_loop_detection.py tests/test_agent_tasks.py tests/test_integration.py tests/test_user_issues.py \
+if ! python3 -m pytest tests/test_drydock_regression.py tests/test_drydock_tasks.py tests/test_loop_detection.py tests/test_agent_tasks.py tests/test_integration.py tests/test_user_issues.py tests/test_real_issues.py \
     -p no:xdist -p no:cov --override-ini="addopts=" -q 2>&1; then
     log "TESTS FAILED — deploy aborted. Fix the tests before deploying."
     exit 1

{drydock_cli-0.7.2 → drydock_cli-0.7.3}/scripts/publish_to_pypi.sh

@@ -25,7 +25,7 @@ PYPI_TOKEN=$(cat "$TOKEN_FILE")
 
 # --- Run tests ---
 log "Running regression tests..."
-if ! $PYTHON -m pytest tests/test_drydock_regression.py tests/test_drydock_tasks.py tests/test_loop_detection.py tests/test_agent_tasks.py tests/test_integration.py tests/test_user_issues.py \
+if ! $PYTHON -m pytest tests/test_drydock_regression.py tests/test_drydock_tasks.py tests/test_loop_detection.py tests/test_agent_tasks.py tests/test_integration.py tests/test_user_issues.py tests/test_real_issues.py \
     -p no:xdist -p no:cov --override-ini="addopts=" -q 2>&1; then
     log "TESTS FAILED — publish aborted."
     exit 1

drydock_cli-0.7.3/tests/test_real_failures.py

@@ -0,0 +1,95 @@
+"""Tests that FAIL — proving real bugs exist.
+
+These tests reproduce actual user-reported failures against the real backend.
+They should FAIL until the underlying code is fixed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from pathlib import Path
+
+import httpx
+import pytest
+
+from drydock.core.config.harness_files import init_harness_files_manager
+
+# Initialize harness before importing agent
+try:
+    init_harness_files_manager("user", "project")
+except RuntimeError:
+    pass
+
+from drydock.core.agent_loop import AgentLoop
+from drydock.core.agents.models import BuiltinAgentName
+from drydock.core.config import Backend, ModelConfig, ProviderConfig, VibeConfig
+from drydock.core.types import (
+    AssistantEvent,
+    BaseEvent,
+    Role,
+    ToolCallEvent,
+    ToolResultEvent,
+)
+
+
+def _vllm_ok() -> bool:
+    try:
+        return httpx.get("http://localhost:8000/v1/models", timeout=3).status_code == 200
+    except Exception:
+        return False
+
+pytestmark = pytest.mark.skipif(not _vllm_ok(), reason="vLLM not running")
+
+
+def _config(tmp_path: Path) -> VibeConfig:
+    return VibeConfig(
+        active_model="devstral", auto_approve=True, enable_telemetry=False,
+        include_project_context=False, system_prompt_id="tests",
+        providers=[ProviderConfig(name="local", api_base="http://localhost:8000/v1", api_key_env_var="", backend=Backend.GENERIC)],
+        models=[ModelConfig(name="devstral", provider="local", input_price=0, output_price=0)],
+        session_logging={"enabled": False, "save_dir": str(tmp_path / "logs")},
+    )
+
+
+def _agent(tmp_path: Path, max_turns: int = 15) -> AgentLoop:
+    return AgentLoop(config=_config(tmp_path), agent_name=BuiltinAgentName.AUTO_APPROVE, max_turns=max_turns)
+
+
+async def _run(agent, prompt, max_events=100):
+    events = []
+    async for ev in agent.act(prompt):
+        events.append(ev)
+        if len(events) >= max_events:
+            break
+    return events
+
+
+# ============================================================================
+# ISSUE: Circuit breaker fires but model KEEPS calling the same tool
+# The breaker returns an error, but the model ignores it and calls again.
+# Proven: 20 bash calls despite 17 circuit breaker fires.
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_circuit_breaker_actually_stops_execution(tmp_path):
+    """After circuit breaker fires, model should NOT make more than 3 additional
+    identical calls. Currently it makes 17+."""
+    agent = _agent(tmp_path, max_turns=20)
+    events = await _run(agent, "Run the command 'echo hello' ten separate times", max_events=120)
+
+    tool_calls = [e for e in events if isinstance(e, ToolCallEvent)]
+    cb_fires = [e for e in events if isinstance(e, ToolResultEvent)
+                and e.error and "CIRCUIT" in str(e.error)]
+
+    # Circuit breaker should fire
+    if not cb_fires:
+        pytest.skip("Model didn't repeat — can't test circuit breaker")
+
+    # After first circuit breaker fire, model should stop within 3 more calls
+    first_cb_idx = next(i for i, e in enumerate(events) if isinstance(e, ToolResultEvent) and e.error and "CIRCUIT" in str(e.error))
+    calls_after_cb = [e for e in events[first_cb_idx:] if isinstance(e, ToolCallEvent)]
+
+    assert len(calls_after_cb) <= 3, \
+        f"Model made {len(calls_after_cb)} tool calls AFTER circuit breaker fired. " \
+        f"It should stop, not keep going. Total: {len(tool_calls)} calls, {len(cb_fires)} breaker fires."

drydock_cli-0.7.3/tests/test_real_issues.py

@@ -0,0 +1,252 @@
+"""Real backend tests for user-reported issues.
+
+Every test hits the ACTUAL vLLM backend at localhost:8000.
+NO mocks, NO fakes — real DryDock execution.
+
+Skip if vLLM is not running.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from pathlib import Path
+
+import httpx
+import pytest
+
+from drydock.core.agent_loop import AgentLoop
+from drydock.core.agents.models import BuiltinAgentName
+from drydock.core.config import Backend, ModelConfig, ProviderConfig, VibeConfig
+from drydock.core.types import (
+    AssistantEvent,
+    BaseEvent,
+    Role,
+    ToolCallEvent,
+    ToolResultEvent,
+)
+
+
+def _vllm_available() -> bool:
+    try:
+        return httpx.get("http://localhost:8000/v1/models", timeout=3).status_code == 200
+    except Exception:
+        return False
+
+
+pytestmark = pytest.mark.skipif(not _vllm_available(), reason="vLLM not running")
+
+
+def _config(tmp_path: Path) -> VibeConfig:
+    return VibeConfig(
+        active_model="devstral",
+        auto_approve=True,
+        enable_telemetry=False,
+        include_project_context=False,
+        system_prompt_id="cli",
+        providers=[ProviderConfig(
+            name="local", api_base="http://localhost:8000/v1",
+            api_key_env_var="", backend=Backend.GENERIC,
+        )],
+        models=[ModelConfig(
+            name="devstral", provider="local", alias="devstral",
+            input_price=0.0, output_price=0.0,
+        )],
+        session_logging={"enabled": False, "save_dir": str(tmp_path / "logs")},
+    )
+
+
+def _agent(tmp_path: Path, max_turns: int = 5) -> AgentLoop:
+    return AgentLoop(
+        config=_config(tmp_path),
+        agent_name=BuiltinAgentName.AUTO_APPROVE,
+        max_turns=max_turns,
+    )
+
+
+async def _run(agent: AgentLoop, prompt: str, max_events: int = 100) -> list[BaseEvent]:
+    events = []
+    async for ev in agent.act(prompt):
+        events.append(ev)
+        if len(events) >= max_events:
+            break
+    return events
+
+
+# ============================================================================
+# Test 1: "Understood" after error — agent should NOT stop with "Understood."
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_no_understood_stops_agent(tmp_path):
+    """Agent should never inject 'Understood.' into conversation."""
+    agent = _agent(tmp_path)
+    events = await _run(agent, "Read the file /nonexistent/path/fake.py")
+
+    # Check messages — none should contain "Understood."
+    for msg in agent.messages:
+        if msg.role == Role.assistant and msg.content:
+            assert "Understood." not in msg.content, \
+                f"Found 'Understood.' in assistant message — causes premature stops"
+
+
+# ============================================================================
+# Test 2: Circuit breaker actually blocks (not just warns)
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_circuit_breaker_blocks_real(tmp_path):
+    """After 2 identical tool calls, the 3rd should be blocked with error."""
+    agent = _agent(tmp_path, max_turns=10)
+    # Ask something that will likely trigger repeated tool calls
+    events = await _run(agent, "Run 'echo hello' five times in a row, each as a separate bash command")
+
+    tool_results = [e for e in events if isinstance(e, ToolResultEvent)]
+    # Should have some blocked results if circuit breaker works
+    errors = [e for e in tool_results if e.error and "CIRCUIT BREAKER" in str(e.error)]
+
+    # The model might not repeat exactly, but circuit breaker state should exist
+    assert hasattr(agent, "_tool_call_history")
+    assert isinstance(agent._tool_call_history, dict)
+    # At least verify no AttributeError crash
+    for e in tool_results:
+        if e.error:
+            assert "AttributeError" not in str(e.error), f"Crash: {e.error}"
+
+
+# ============================================================================
+# Test 3: Tool calls don't crash with AttributeError
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_no_attribute_error_on_tools(tmp_path):
+    """No AttributeError should occur during tool execution."""
+    agent = _agent(tmp_path)
+    events = await _run(agent, "List the files in the current directory")
+
+    for e in events:
+        if isinstance(e, ToolResultEvent) and e.error:
+            assert "AttributeError" not in str(e.error), \
+                f"AttributeError in tool result: {e.error}"
+            assert "raw_arguments" not in str(e.error), \
+                f"Old raw_arguments bug: {e.error}"
+
+
+# ============================================================================
+# Test 4: Agent doesn't loop on ambiguous prompt
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_ambiguous_prompt_limited(tmp_path):
+    """Typing 'test' should not cause 20+ tool calls."""
+    agent = _agent(tmp_path, max_turns=8)
+    events = await _run(agent, "test", max_events=60)
+
+    tool_calls = [e for e in events if isinstance(e, ToolCallEvent)]
+    assert len(tool_calls) < 25, \
+        f"Ambiguous prompt 'test' caused {len(tool_calls)} tool calls — too many"
+
+
+# ============================================================================
+# Test 5: Message ordering clean after real tool calls
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_no_user_after_tool(tmp_path):
+    """No user message should follow a tool message in the history."""
+    agent = _agent(tmp_path)
+    events = await _run(agent, "What Python version is installed? Run python3 --version")
+
+    for i in range(1, len(agent.messages)):
+        if agent.messages[i].role == Role.user and agent.messages[i-1].role == Role.tool:
+            pytest.fail(f"user after tool at position {i}: "
+                       f"'{agent.messages[i].content[:50]}' after tool '{agent.messages[i-1].content[:50]}'")
+
+
+# ============================================================================
+# Test 6: Agent actually uses tools (not just text responses)
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_agent_uses_tools(tmp_path):
+    """Agent should call bash/grep/read_file when asked to do something concrete."""
+    agent = _agent(tmp_path)
+    events = await _run(agent, "Run: echo 'drydock integration test'")
+
+    tool_calls = [e for e in events if isinstance(e, ToolCallEvent)]
+    # Should have at least one tool call for a concrete command request
+    assert len(tool_calls) >= 1, \
+        "Agent gave text-only response when asked to run a command"
+
+
+# ============================================================================
+# Test 7: DryDock branding in system prompt (real check)
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_system_prompt_branding(tmp_path):
+    """System prompt should say 'DryDock' not 'Drydock' or 'Vibe'."""
+    agent = _agent(tmp_path)
+    # Don't even need to run — just check the system prompt
+    system_msg = agent.messages[0]
+    assert system_msg.role == Role.system
+    assert "DryDock" in system_msg.content, "System prompt missing 'DryDock'"
+    assert "Vibe" not in system_msg.content or "mistral-vibe" in system_msg.content, \
+        "System prompt still references 'Vibe'"
+
+
+# ============================================================================
+# Test 8: Write file doesn't hang (has timeout)
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_write_file_timeout_exists(tmp_path):
+    """write_file should complete within reasonable time, not hang."""
+    agent = _agent(tmp_path)
+    test_file = tmp_path / "test_write.txt"
+
+    events = await _run(agent,
+        f"Write the text 'hello world' to the file {test_file}")
+
+    # Should complete (not hang) — if it hangs, the test runner will timeout
+    assert len(events) >= 1
+    # Check if file was created (model might or might not do it)
+    # The important thing is it didn't hang
+
+
+# ============================================================================
+# Test 9: Agent stops cleanly (not infinite loop)
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_agent_stops_cleanly(tmp_path):
+    """Agent should stop after completing a task, not loop forever."""
+    agent = _agent(tmp_path, max_turns=5)
+    events = await _run(agent, "What is 2+2? Just say the number.", max_events=30)
+
+    # Should stop within reasonable events
+    assert len(events) < 30, f"Agent produced {len(events)} events — possible loop"
+
+
+# ============================================================================
+# Test 10: Consultant config field exists and works
+# ============================================================================
+
+@pytest.mark.asyncio
+async def test_consultant_config_real(tmp_path):
+    """VibeConfig should accept consultant_model field without error."""
+    config = VibeConfig(
+        active_model="devstral",
+        consultant_model="gemini-2.5-pro",
+        auto_approve=True,
+        enable_telemetry=False,
+        providers=[ProviderConfig(
+            name="local", api_base="http://localhost:8000/v1",
+            api_key_env_var="", backend=Backend.GENERIC,
+        )],
+        models=[ModelConfig(
+            name="devstral", provider="local",
+            input_price=0.0, output_price=0.0,
+        )],
+    )
+    assert config.consultant_model == "gemini-2.5-pro"