ripperdoc 0.2.9__py3-none-any.whl → 0.3.0__py3-none-any.whl

ripperdoc/utils/mcp.py

@@ -17,12 +17,13 @@ from ripperdoc.utils.token_estimation import estimate_tokens
 
 logger = get_logger()
 
+
 try:
     import mcp.types as mcp_types  # type: ignore[import-not-found]
     from mcp.client.session import ClientSession  # type: ignore[import-not-found]
     from mcp.client.sse import sse_client  # type: ignore[import-not-found]
     from mcp.client.stdio import StdioServerParameters, stdio_client  # type: ignore[import-not-found]
-    from mcp.client.streamable_http import streamablehttp_client  # type: ignore[import-not-found]
+    from mcp.client.streamable_http import streamable_http_client  # type: ignore[import-not-found]
 
     MCP_AVAILABLE = True
 except (ImportError, ModuleNotFoundError):  # pragma: no cover - handled gracefully at runtime
@@ -217,6 +218,14 @@ class McpRuntime:
         self.sessions: Dict[str, ClientSession] = {}
         self.servers: List[McpServerInfo] = []
         self._closed = False
+        # Track MCP streams for proper cleanup ordering
+        # We need to close write streams BEFORE exiting the stdio_client context
+        # to allow the internal tasks to exit cleanly
+        self._mcp_write_streams: List[Any] = []
+        # Track the underlying async generators from @asynccontextmanager wrappers
+        # These need to be explicitly closed after exit stack cleanup to prevent
+        # shutdown_asyncgens() from trying to close them in a different task
+        self._raw_async_generators: List[Any] = []
 
     async def connect(self, configs: Dict[str, McpServerInfo]) -> List[McpServerInfo]:
         logger.info(
@@ -281,19 +290,24 @@ class McpRuntime:
             if config.type in ("sse", "sse-ide"):
                 if not config.url:
                     raise ValueError("SSE MCP server requires a 'url'.")
-                read_stream, write_stream = await self._exit_stack.enter_async_context(
-                    sse_client(config.url, headers=config.headers or None)
-                )
+                cm = sse_client(config.url, headers=config.headers or None)
+                # Track the underlying async generator for explicit cleanup
+                if hasattr(cm, "gen"):
+                    self._raw_async_generators.append(cm.gen)
+                read_stream, write_stream = await self._exit_stack.enter_async_context(cm)
+                self._mcp_write_streams.append(write_stream)
             elif config.type in ("http", "streamable-http"):
                 if not config.url:
                     raise ValueError("HTTP MCP server requires a 'url'.")
-                read_stream, write_stream, _ = await self._exit_stack.enter_async_context(
-                    streamablehttp_client(
-                        url=config.url,
-                        headers=config.headers or None,
-                        terminate_on_close=True,
-                    )
+                cm = streamable_http_client(  # type: ignore[call-arg]
+                    url=config.url,
+                    terminate_on_close=True,
                 )
+                # Track the underlying async generator for explicit cleanup
+                if hasattr(cm, "gen"):
+                    self._raw_async_generators.append(cm.gen)
+                read_stream, write_stream, _ = await self._exit_stack.enter_async_context(cm)
+                self._mcp_write_streams.append(write_stream)
             else:
                 if not config.command:
                     raise ValueError("Stdio MCP server requires a 'command'.")
@@ -303,9 +317,12 @@ class McpRuntime:
                     env=config.env or None,
                     cwd=self.project_path,
                 )
-                read_stream, write_stream = await self._exit_stack.enter_async_context(
-                    stdio_client(stdio_params)
-                )
+                cm = stdio_client(stdio_params)
+                # Track the underlying async generator for explicit cleanup
+                if hasattr(cm, "gen"):
+                    self._raw_async_generators.append(cm.gen)
+                read_stream, write_stream = await self._exit_stack.enter_async_context(cm)
+                self._mcp_write_streams.append(write_stream)
 
             if read_stream is None or write_stream is None:
                 raise ValueError("Failed to create read/write streams for MCP server")
@@ -392,17 +409,39 @@ class McpRuntime:
             "[mcp] Shutting down MCP runtime",
             extra={"project_path": str(self.project_path), "session_count": len(self.sessions)},
         )
+
+        # CRITICAL: Close all MCP write streams FIRST to signal internal tasks to stop.
+        for write_stream in self._mcp_write_streams:
+            try:
+                await write_stream.aclose()
+            except BaseException:  # pragma: no cover
+                pass
+        self._mcp_write_streams.clear()
+
+        # Small delay to allow internal tasks to notice stream closure and exit
+        await asyncio.sleep(0.1)
+
+        # CRITICAL: Close the raw async generators BEFORE the exit stack cleanup.
+        # This prevents asyncio's shutdown_asyncgens() from trying to close them
+        # later, which would cause the "cancel scope in different task" error.
+        for gen in self._raw_async_generators:
+            try:
+                await gen.aclose()
+            except BaseException:  # pragma: no cover
+                pass
+        self._raw_async_generators.clear()
+
+        # Now close the exit stack
         try:
             await self._exit_stack.aclose()
         except BaseException as exc:  # pragma: no cover - defensive shutdown
-            # Swallow noisy ExceptionGroups from stdio_client cancel scopes during exit.
             logger.debug(
-                "[mcp] Suppressed MCP shutdown error",
+                "[mcp] Suppressed MCP shutdown error during exit_stack.aclose()",
                 extra={"error": str(exc), "project_path": str(self.project_path)},
             )
-        finally:
-            self.sessions.clear()
-            self.servers.clear()
+
+        self.sessions.clear()
+        self.servers.clear()
 
 
 _runtime_var: contextvars.ContextVar[Optional[McpRuntime]] = contextvars.ContextVar(
@@ -453,6 +492,29 @@ async def ensure_mcp_runtime(project_path: Optional[Path] = None) -> McpRuntime:
     # Keep a module-level reference so sync callers that hop event loops can reuse it.
     global _global_runtime
     _global_runtime = runtime
+
+    # Install custom exception handler to suppress MCP asyncgen cleanup errors.
+    # These errors occur due to anyio cancel scope issues when stdio_client async
+    # generators are finalized by Python's asyncgen hooks. The errors are harmless
+    # but noisy, so we suppress them here.
+    loop = asyncio.get_running_loop()
+    original_handler = loop.get_exception_handler()
+
+    def mcp_exception_handler(loop: asyncio.AbstractEventLoop, context: Dict[str, Any]) -> None:
+        asyncgen = context.get("asyncgen")
+        # Suppress MCP stdio_client asyncgen cleanup errors
+        if asyncgen and "stdio_client" in str(asyncgen):
+            logger.debug("[mcp] Suppressed asyncgen cleanup error for stdio_client")
+            return
+        # Call original handler for other errors
+        if original_handler:
+            original_handler(loop, context)
+        else:
+            loop.default_exception_handler(context)
+
+    loop.set_exception_handler(mcp_exception_handler)
+    logger.debug("[mcp] Installed custom exception handler for asyncgen cleanup")
+
     return runtime
 
 

ripperdoc/utils/message_formatting.py

@@ -151,15 +151,18 @@ def format_reasoning_preview(reasoning: Any, show_full_thinking: bool = False) -
         text = "\n".join(p for p in parts if p)
     else:
         text = str(reasoning)
-    
+
     if show_full_thinking:
         return text
-    
+
     lines = text.strip().splitlines()
     if not lines:
         return ""
-    preview = lines[0][:250]
-    if len(lines) > 1 or len(lines[0]) > 250:
+    first_line = lines[0]
+    if not first_line:
+        return "..." if len(lines) > 1 else ""
+    preview = first_line[:250]
+    if len(lines) > 1 or len(first_line) > 250:
         preview += "..."
     return preview
 

ripperdoc/utils/messages.py

@@ -5,7 +5,7 @@ for communication with AI models.
 """
 
 from typing import Any, Dict, List, Optional, Union
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 from uuid import uuid4
 from enum import Enum
 from ripperdoc.utils.log import get_logger
@@ -35,6 +35,23 @@ class MessageContent(BaseModel):
     name: Optional[str] = None
     input: Optional[Dict[str, object]] = None
     is_error: Optional[bool] = None
+    # Image/vision content fields
+    source_type: Optional[str] = None  # "base64", "url", "file"
+    media_type: Optional[str] = None  # "image/jpeg", "image/png", etc.
+    image_data: Optional[str] = None  # base64-encoded image data or URL
+
+    @field_validator("input", mode="before")
+    @classmethod
+    def validate_input(cls, v):
+        """Ensure input is always a dict, never a Pydantic model."""
+        if v is not None and not isinstance(v, dict):
+            if hasattr(v, "model_dump"):
+                v = v.model_dump()
+            elif hasattr(v, "dict"):
+                v = v.dict()
+            else:
+                v = {"value": str(v)}
+        return v
 
 
 def _content_block_to_api(block: MessageContent) -> Dict[str, Any]:
@@ -53,11 +70,19 @@ def _content_block_to_api(block: MessageContent) -> Dict[str, Any]:
             "signature": getattr(block, "signature", None),
         }
     if block_type == "tool_use":
+        input_value = getattr(block, "input", None) or {}
+        # Ensure input is a dict, not a Pydantic model
+        if hasattr(input_value, "model_dump"):
+            input_value = input_value.model_dump()
+        elif hasattr(input_value, "dict"):
+            input_value = input_value.dict()
+        elif not isinstance(input_value, dict):
+            input_value = {"value": str(input_value)}
         return {
             "type": "tool_use",
             "id": getattr(block, "id", None) or getattr(block, "tool_use_id", "") or "",
             "name": getattr(block, "name", None) or "",
-            "input": getattr(block, "input", None) or {},
+            "input": input_value,
         }
     if block_type == "tool_result":
         result: Dict[str, Any] = {
@@ -73,6 +98,15 @@ def _content_block_to_api(block: MessageContent) -> Dict[str, Any]:
         if getattr(block, "is_error", None) is not None:
             result["is_error"] = block.is_error
         return result
+    if block_type == "image":
+        return {
+            "type": "image",
+            "source": {
+                "type": getattr(block, "source_type", None) or "base64",
+                "media_type": getattr(block, "media_type", None) or "image/jpeg",
+                "data": getattr(block, "image_data", None) or "",
+            },
+        }
     # Default to text block
     return {
         "type": "text",
@@ -124,6 +158,15 @@ def _content_block_to_openai(block: MessageContent) -> Dict[str, Any]:
             "tool_call_id": tool_call_id,
             "content": getattr(block, "text", None) or getattr(block, "content", None) or "",
         }
+    if block_type == "image":
+        # OpenAI uses data URL format for images
+        media_type = getattr(block, "media_type", None) or "image/jpeg"
+        image_data = getattr(block, "image_data", None) or ""
+        data_url = f"data:{media_type};base64,{image_data}"
+        return {
+            "type": "image_url",
+            "image_url": {"url": data_url},
+        }
     # Fallback text message
     return {
         "role": "assistant",
@@ -152,6 +195,7 @@ class UserMessage(BaseModel):
     type: str = "user"
     message: Message
     uuid: str = ""
+    parent_tool_use_id: Optional[str] = None
     tool_use_result: Optional[object] = None
 
     def __init__(self, **data: object) -> None:
@@ -166,9 +210,17 @@ class AssistantMessage(BaseModel):
     type: str = "assistant"
     message: Message
     uuid: str = ""
+    parent_tool_use_id: Optional[str] = None
     cost_usd: float = 0.0
     duration_ms: float = 0.0
     is_api_error_message: bool = False
+    # Model and token usage information
+    model: Optional[str] = None
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_tokens: int = 0
+    cache_creation_tokens: int = 0
+    error: Optional[str] = None
 
     def __init__(self, **data: object) -> None:
         if "uuid" not in data or not data["uuid"]:
@@ -185,6 +237,7 @@ class ProgressMessage(BaseModel):
     content: Any
     normalized_messages: List[Message] = []
     sibling_tool_use_ids: set[str] = set()
+    is_subagent_message: bool = False  # Flag to indicate if content is a subagent message
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     def __init__(self, **data: object) -> None:
@@ -194,7 +247,9 @@ class ProgressMessage(BaseModel):
 
 
 def create_user_message(
-    content: Union[str, List[Dict[str, Any]]], tool_use_result: Optional[object] = None
+    content: Union[str, List[Dict[str, Any]]],
+    tool_use_result: Optional[object] = None,
+    parent_tool_use_id: Optional[str] = None,
 ) -> UserMessage:
     """Create a user message."""
     if isinstance(content, str):
@@ -228,7 +283,39 @@ def create_user_message(
                 f"ids={[getattr(b, 'tool_use_id', None) for b in tool_result_blocks]}"
             )
 
-    return UserMessage(message=message, tool_use_result=tool_use_result)
+    return UserMessage(
+        message=message,
+        tool_use_result=tool_use_result,
+        parent_tool_use_id=parent_tool_use_id,
+    )
+
+
+def _normalize_content_item(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Normalize a content item to ensure all fields are JSON-serializable.
+
+    This is needed because some API providers may return Pydantic models
+    for tool input fields, which need to be converted to dicts for proper
+    serialization and later processing.
+
+    Args:
+        item: The content item dict from API response
+
+    Returns:
+        Normalized content item with all fields JSON-serializable
+    """
+    normalized = dict(item)
+
+    # If input is a Pydantic model, convert to dict
+    if 'input' in normalized and normalized['input'] is not None:
+        input_value = normalized['input']
+        if hasattr(input_value, 'model_dump'):
+            normalized['input'] = input_value.model_dump()
+        elif hasattr(input_value, 'dict'):
+            normalized['input'] = input_value.dict()
+        elif not isinstance(input_value, dict):
+            normalized['input'] = {'value': str(input_value)}
+
+    return normalized
 
 
 def create_assistant_message(
@@ -237,12 +324,20 @@ def create_assistant_message(
     duration_ms: float = 0.0,
     reasoning: Optional[Any] = None,
     metadata: Optional[Dict[str, Any]] = None,
+    model: Optional[str] = None,
+    input_tokens: int = 0,
+    output_tokens: int = 0,
+    cache_read_tokens: int = 0,
+    cache_creation_tokens: int = 0,
+    parent_tool_use_id: Optional[str] = None,
+    error: Optional[str] = None,
 ) -> AssistantMessage:
     """Create an assistant message."""
     if isinstance(content, str):
         message_content: Union[str, List[MessageContent]] = content
     else:
-        message_content = [MessageContent(**item) for item in content]
+        # Normalize content items to ensure tool input is always a dict
+        message_content = [MessageContent(**_normalize_content_item(item)) for item in content]
 
     message = Message(
         role=MessageRole.ASSISTANT,
@@ -251,7 +346,18 @@ def create_assistant_message(
         metadata=metadata or {},
     )
 
-    return AssistantMessage(message=message, cost_usd=cost_usd, duration_ms=duration_ms)
+    return AssistantMessage(
+        message=message,
+        cost_usd=cost_usd,
+        duration_ms=duration_ms,
+        model=model,
+        input_tokens=input_tokens,
+        output_tokens=output_tokens,
+        cache_read_tokens=cache_read_tokens,
+        cache_creation_tokens=cache_creation_tokens,
+        parent_tool_use_id=parent_tool_use_id,
+        error=error,
+    )
 
 
 def create_progress_message(
@@ -259,6 +365,7 @@ def create_progress_message(
     sibling_tool_use_ids: set[str],
     content: Any,
     normalized_messages: Optional[List[Message]] = None,
+    is_subagent_message: bool = False,
 ) -> ProgressMessage:
     """Create a progress message."""
     return ProgressMessage(
@@ -266,6 +373,7 @@ def create_progress_message(
         sibling_tool_use_ids=sibling_tool_use_ids,
         content=content,
         normalized_messages=normalized_messages or [],
+        is_subagent_message=is_subagent_message,
     )
 
 
@@ -434,33 +542,90 @@ def normalize_messages_for_api(
             meta = _msg_metadata(msg)
             if isinstance(user_content, list):
                 if protocol == "openai":
-                    # Map each block to an OpenAI-style message
-                    openai_msgs: List[Dict[str, Any]] = []
-                    for block in user_content:
-                        block_type = getattr(block, "type", None)
-                        if block_type == "tool_result":
-                            tool_results_seen += 1
-                            # Skip tool_result blocks that lack a preceding tool_use
-                            tool_id = getattr(block, "tool_use_id", None) or getattr(
-                                block, "id", None
-                            )
-                            if not tool_id:
-                                skipped_tool_results_no_call += 1
-                                continue
-                            call_pos = tool_use_positions.get(tool_id)
-                            if call_pos is None or call_pos >= msg_index:
-                                skipped_tool_results_no_call += 1
-                                continue
-                        mapped = _content_block_to_openai(block)
-                        if mapped:
-                            openai_msgs.append(mapped)
-                    if meta and openai_msgs:
-                        for candidate in openai_msgs:
-                            for key in ("reasoning_content", "reasoning_details", "reasoning"):
-                                if key in meta and meta[key] is not None:
-                                    candidate[key] = meta[key]
-                    normalized.extend(openai_msgs)
-                    continue
+                    # Check if this message contains images
+                    has_images = any(
+                        getattr(block, "type", None) == "image" for block in user_content
+                    )
+                    has_text_only = all(
+                        getattr(block, "type", None) in ("text", "image", "tool_result")
+                        for block in user_content
+                    )
+
+                    # If message has images or only text/images (no tool_result), use content array format
+                    if has_images or (
+                        has_text_only
+                        and not any(
+                            getattr(block, "type", None) == "tool_result" for block in user_content
+                        )
+                    ):
+                        content_array: List[Dict[str, Any]] = []
+                        for block in user_content:
+                            block_type = getattr(block, "type", None)
+                            if block_type == "image":
+                                content_array.append(_content_block_to_openai(block))
+                            elif block_type == "text":
+                                content_array.append(
+                                    {
+                                        "type": "text",
+                                        "text": getattr(block, "text", "") or "",
+                                    }
+                                )
+                            elif block_type == "tool_result":
+                                # Handle tool_result separately
+                                tool_results_seen += 1
+                                tool_id = getattr(block, "tool_use_id", None) or getattr(
+                                    block, "id", None
+                                )
+                                if not tool_id:
+                                    skipped_tool_results_no_call += 1
+                                    continue
+                                call_pos = tool_use_positions.get(tool_id)
+                                if call_pos is None or call_pos >= msg_index:
+                                    skipped_tool_results_no_call += 1
+                                    continue
+                                mapped = _content_block_to_openai(block)
+                                if mapped:
+                                    normalized.append(mapped)
+
+                        if content_array:
+                            user_msg: Dict[str, Any] = {
+                                "role": "user",
+                                "content": content_array,
+                            }
+                            if meta:
+                                for key in ("reasoning_content", "reasoning_details", "reasoning"):
+                                    if key in meta and meta[key] is not None:
+                                        user_msg[key] = meta[key]
+                            normalized.append(user_msg)
+                        continue
+                    else:
+                        # Original behavior for tool_result messages
+                        openai_msgs: List[Dict[str, Any]] = []
+                        for block in user_content:
+                            block_type = getattr(block, "type", None)
+                            if block_type == "tool_result":
+                                tool_results_seen += 1
+                                # Skip tool_result blocks that lack a preceding tool_use
+                                tool_id = getattr(block, "tool_use_id", None) or getattr(
+                                    block, "id", None
+                                )
+                                if not tool_id:
+                                    skipped_tool_results_no_call += 1
+                                    continue
+                                call_pos = tool_use_positions.get(tool_id)
+                                if call_pos is None or call_pos >= msg_index:
+                                    skipped_tool_results_no_call += 1
+                                    continue
+                            mapped = _content_block_to_openai(block)
+                            if mapped:
+                                openai_msgs.append(mapped)
+                        if meta and openai_msgs:
+                            for candidate in openai_msgs:
+                                for key in ("reasoning_content", "reasoning_details", "reasoning"):
+                                    if key in meta and meta[key] is not None:
+                                        candidate[key] = meta[key]
+                        normalized.extend(openai_msgs)
+                        continue
                 api_blocks = []
                 for block in user_content:
                     if getattr(block, "type", None) == "tool_result":

ripperdoc/utils/pending_messages.py

@@ -0,0 +1,50 @@
+"""Thread-safe queue for pending conversation messages.
+
+Allows background tasks or external events to enqueue user messages that
+should be injected into the conversation once the current iteration
+finishes. Messages are drained in FIFO order.
+"""
+
+from collections import deque
+import threading
+from typing import Any, Deque, Dict, List, Optional
+
+from ripperdoc.utils.messages import UserMessage, create_user_message
+
+
+class PendingMessageQueue:
+    """Thread-safe queue for pending user messages."""
+
+    def __init__(self) -> None:
+        self._queue: Deque[UserMessage] = deque()
+        self._lock = threading.Lock()
+
+    def enqueue(self, message: UserMessage) -> None:
+        """Add a pre-built UserMessage to the queue."""
+        with self._lock:
+            self._queue.append(message)
+
+    def enqueue_text(self, text: str, metadata: Optional[Dict[str, Any]] = None) -> None:
+        """Create and enqueue a UserMessage with optional metadata."""
+        message = create_user_message(text)
+        if metadata:
+            try:
+                message.message.metadata.update(metadata)
+            except Exception:
+                # Best-effort metadata attachment; ignore failures.
+                pass
+        self.enqueue(message)
+
+    def drain(self) -> List[UserMessage]:
+        """Drain all pending messages in FIFO order."""
+        with self._lock:
+            if not self._queue:
+                return []
+            messages = list(self._queue)
+            self._queue.clear()
+            return messages
+
+    def has_messages(self) -> bool:
+        """Check if there are pending messages."""
+        with self._lock:
+            return bool(self._queue)

ripperdoc/utils/permissions/shell_command_validation.py

@@ -662,7 +662,7 @@ def validate_shell_command(shell_command: str) -> ValidationResult:
         lex = shlex.shlex(cmd, posix=True)
         lex.whitespace_split = True  # Split on whitespace, better for argument parsing
         lex.commenters = ""  # Don't treat # as comment for security analysis
-        
+
         tokens = []
         try:
             # Get all tokens
@@ -691,7 +691,7 @@ def validate_shell_command(shell_command: str) -> ValidationResult:
                 # Single ; & | are dangerous
                 return True
             i += 1
-        
+
         # Also check for find -exec escaped semicolon pattern
         # shlex will have already parsed \; as separate token ';' (since escaped)
         # We need to check if this ; is part of find -exec pattern
@@ -716,7 +716,7 @@ def validate_shell_command(shell_command: str) -> ValidationResult:
                     continue
                 # Not part of find -exec, so it's dangerous
                 return True
-        
+
         return False
 
     if has_metachars_outside_quotes(sanitized_for_metachar_check):