code-puppy 0.0.336__py3-none-any.whl → 0.0.348__py3-none-any.whl

code_puppy/http_utils.py

@@ -5,10 +5,10 @@ This module provides functions for creating properly configured HTTP clients.
 """
 
 import asyncio
-import logging
 import os
 import socket
 import time
+from dataclasses import dataclass
 from typing import Any, Dict, Optional, Union
 
 import httpx
@@ -16,7 +16,69 @@ import requests
 
 from code_puppy.config import get_http2
 
-logger = logging.getLogger(__name__)
+
+@dataclass
+class ProxyConfig:
+    """Configuration for proxy and SSL settings."""
+
+    verify: Union[bool, str, None]
+    trust_env: bool
+    proxy_url: str | None
+    disable_retry: bool
+    http2_enabled: bool
+
+
+def _resolve_proxy_config(verify: Union[bool, str, None] = None) -> ProxyConfig:
+    """Resolve proxy, SSL, and retry settings from environment.
+
+    This centralizes the logic for detecting proxies, determining SSL verification,
+    and checking if retry transport should be disabled.
+    """
+    if verify is None:
+        verify = get_cert_bundle_path()
+
+    http2_enabled = get_http2()
+
+    disable_retry = os.environ.get(
+        "CODE_PUPPY_DISABLE_RETRY_TRANSPORT", ""
+    ).lower() in ("1", "true", "yes")
+
+    has_proxy = bool(
+        os.environ.get("HTTP_PROXY")
+        or os.environ.get("HTTPS_PROXY")
+        or os.environ.get("http_proxy")
+        or os.environ.get("https_proxy")
+    )
+
+    # Determine trust_env and verify based on proxy/retry settings
+    if disable_retry:
+        # Test mode: disable SSL verification for proxy testing
+        verify = False
+        trust_env = True
+    elif has_proxy:
+        # Production proxy: keep SSL verification enabled
+        trust_env = True
+    else:
+        trust_env = False
+
+    # Extract proxy URL
+    proxy_url = None
+    if has_proxy:
+        proxy_url = (
+            os.environ.get("HTTPS_PROXY")
+            or os.environ.get("https_proxy")
+            or os.environ.get("HTTP_PROXY")
+            or os.environ.get("http_proxy")
+        )
+
+    return ProxyConfig(
+        verify=verify,
+        trust_env=trust_env,
+        proxy_url=proxy_url,
+        disable_retry=disable_retry,
+        http2_enabled=http2_enabled,
+    )
+
 
 try:
     from .reopenable_async_client import ReopenableAsyncClient
@@ -58,14 +120,7 @@ class RetryingAsyncClient(httpx.AsyncClient):
 
         for attempt in range(self.max_retries + 1):
             try:
-                # Clone request for retry (streams might be consumed)
-                # But only if it's not the first attempt
-                req_to_send = request
-                if attempt > 0:
-                    # httpx requests are reusable, but we need to be careful with streams
-                    pass
-
-                response = await super().send(req_to_send, **kwargs)
+                response = await super().send(request, **kwargs)
                 last_response = response
 
                 # Check for retryable status
@@ -128,7 +183,7 @@ class RetryingAsyncClient(httpx.AsyncClient):
         return last_response
 
 
-def get_cert_bundle_path() -> str:
+def get_cert_bundle_path() -> str | None:
     # First check if SSL_CERT_FILE environment variable is set
     ssl_cert_file = os.environ.get("SSL_CERT_FILE")
     if ssl_cert_file and os.path.exists(ssl_cert_file):
@@ -164,66 +219,26 @@ def create_async_client(
     headers: Optional[Dict[str, str]] = None,
     retry_status_codes: tuple = (429, 502, 503, 504),
 ) -> httpx.AsyncClient:
-    if verify is None:
-        verify = get_cert_bundle_path()
-
-    # Check if HTTP/2 is enabled in config
-    http2_enabled = get_http2()
-
-    # Check if custom retry transport should be disabled (e.g., for integration tests with proxies)
-    disable_retry_transport = os.environ.get(
-        "CODE_PUPPY_DISABLE_RETRY_TRANSPORT", ""
-    ).lower() in ("1", "true", "yes")
-
-    # Check if proxy environment variables are set
-    has_proxy = bool(
-        os.environ.get("HTTP_PROXY")
-        or os.environ.get("HTTPS_PROXY")
-        or os.environ.get("http_proxy")
-        or os.environ.get("https_proxy")
-    )
-
-    # When retry transport is disabled (test mode), disable SSL verification
-    # for proxy testing. For production proxies, SSL should still be verified!
-    if disable_retry_transport:
-        verify = False
-        trust_env = True
-    elif has_proxy:
-        # Production proxy detected - keep SSL verification enabled for security
-        trust_env = True
-    else:
-        trust_env = False
-
-    # Extract proxy URL if needed
-    proxy_url = None
-    if has_proxy:
-        proxy_url = (
-            os.environ.get("HTTPS_PROXY")
-            or os.environ.get("https_proxy")
-            or os.environ.get("HTTP_PROXY")
-            or os.environ.get("http_proxy")
-        )
+    config = _resolve_proxy_config(verify)
 
-    # Use RetryingAsyncClient if retries are enabled
-    if not disable_retry_transport:
+    if not config.disable_retry:
         return RetryingAsyncClient(
             retry_status_codes=retry_status_codes,
-            proxy=proxy_url,
-            verify=verify,
+            proxy=config.proxy_url,
+            verify=config.verify,
             headers=headers or {},
             timeout=timeout,
-            http2=http2_enabled,
-            trust_env=trust_env,
+            http2=config.http2_enabled,
+            trust_env=config.trust_env,
         )
     else:
-        # Regular client for testing
         return httpx.AsyncClient(
-            proxy=proxy_url,
-            verify=verify,
+            proxy=config.proxy_url,
+            verify=config.verify,
             headers=headers or {},
             timeout=timeout,
-            http2=http2_enabled,
-            trust_env=trust_env,
+            http2=config.http2_enabled,
+            trust_env=config.trust_env,
         )
 
 
@@ -273,85 +288,33 @@ def create_reopenable_async_client(
     headers: Optional[Dict[str, str]] = None,
     retry_status_codes: tuple = (429, 502, 503, 504),
 ) -> Union[ReopenableAsyncClient, httpx.AsyncClient]:
-    if verify is None:
-        verify = get_cert_bundle_path()
-
-    # Check if HTTP/2 is enabled in config
-    http2_enabled = get_http2()
-
-    # Check if custom retry transport should be disabled (e.g., for integration tests with proxies)
-    disable_retry_transport = os.environ.get(
-        "CODE_PUPPY_DISABLE_RETRY_TRANSPORT", ""
-    ).lower() in ("1", "true", "yes")
-
-    # Check if proxy environment variables are set
-    has_proxy = bool(
-        os.environ.get("HTTP_PROXY")
-        or os.environ.get("HTTPS_PROXY")
-        or os.environ.get("http_proxy")
-        or os.environ.get("https_proxy")
-    )
-
-    # When retry transport is disabled (test mode), disable SSL verification
-    if disable_retry_transport:
-        verify = False
-        trust_env = True
-    elif has_proxy:
-        trust_env = True
-    else:
-        trust_env = False
+    config = _resolve_proxy_config(verify)
 
-    # Extract proxy URL if needed
-    proxy_url = None
-    if has_proxy:
-        proxy_url = (
-            os.environ.get("HTTPS_PROXY")
-            or os.environ.get("https_proxy")
-            or os.environ.get("HTTP_PROXY")
-            or os.environ.get("http_proxy")
-        )
+    base_kwargs = {
+        "proxy": config.proxy_url,
+        "verify": config.verify,
+        "headers": headers or {},
+        "timeout": timeout,
+        "http2": config.http2_enabled,
+        "trust_env": config.trust_env,
+    }
 
     if ReopenableAsyncClient is not None:
-        # Use RetryingAsyncClient if retries are enabled
         client_class = (
-            RetryingAsyncClient if not disable_retry_transport else httpx.AsyncClient
+            RetryingAsyncClient if not config.disable_retry else httpx.AsyncClient
         )
-
-        # Pass retry config only if using RetryingAsyncClient
-        kwargs = {
-            "proxy": proxy_url,
-            "verify": verify,
-            "headers": headers or {},
-            "timeout": timeout,
-            "http2": http2_enabled,
-            "trust_env": trust_env,
-        }
-
-        if not disable_retry_transport:
+        kwargs = {**base_kwargs, "client_class": client_class}
+        if not config.disable_retry:
             kwargs["retry_status_codes"] = retry_status_codes
-
-        return ReopenableAsyncClient(client_class=client_class, **kwargs)
+        return ReopenableAsyncClient(**kwargs)
     else:
-        # Fallback to RetryingAsyncClient
-        if not disable_retry_transport:
+        # Fallback to RetryingAsyncClient or plain AsyncClient
+        if not config.disable_retry:
             return RetryingAsyncClient(
-                retry_status_codes=retry_status_codes,
-                proxy=proxy_url,
-                verify=verify,
-                headers=headers or {},
-                timeout=timeout,
-                http2=http2_enabled,
-                trust_env=trust_env,
+                retry_status_codes=retry_status_codes, **base_kwargs
             )
         else:
-            return httpx.AsyncClient(
-                proxy=proxy_url,
-                verify=verify,
-                headers=headers or {},
-                timeout=timeout,
-                http2=http2_enabled,
-                trust_env=trust_env,
-            )
+            return httpx.AsyncClient(**base_kwargs)
 
 
 def is_cert_bundle_available() -> bool:

code_puppy/mcp_/async_lifecycle.py

@@ -108,10 +108,17 @@ class AsyncServerLifecycleManager:
 
         try:
             logger.info(f"Starting server lifecycle for {server_id}")
+            logger.info(
+                f"Server {server_id} _running_count before enter: {getattr(server, '_running_count', 'N/A')}"
+            )
 
             # Enter the server's context
             await exit_stack.enter_async_context(server)
 
+            logger.info(
+                f"Server {server_id} _running_count after enter: {getattr(server, '_running_count', 'N/A')}"
+            )
+
             # Store the managed context
             async with self._lock:
                 self._servers[server_id] = ManagedServerContext(
@@ -122,26 +129,50 @@ class AsyncServerLifecycleManager:
                     task=asyncio.current_task(),
                 )
 
-            logger.info(f"Server {server_id} started successfully")
+            logger.info(
+                f"Server {server_id} started successfully and stored in _servers"
+            )
 
             # Keep the task alive until cancelled
+            loop_count = 0
             while True:
                 await asyncio.sleep(1)
+                loop_count += 1
 
                 # Check if server is still running
-                if not server.is_running:
-                    logger.warning(f"Server {server_id} stopped unexpectedly")
+                running_count = getattr(server, "_running_count", "N/A")
+                is_running = server.is_running
+                logger.debug(
+                    f"Server {server_id} heartbeat #{loop_count}: "
+                    f"is_running={is_running}, _running_count={running_count}"
+                )
+
+                if not is_running:
+                    logger.warning(
+                        f"Server {server_id} stopped unexpectedly! "
+                        f"_running_count={running_count}"
+                    )
                     break
 
         except asyncio.CancelledError:
             logger.info(f"Server {server_id} lifecycle task cancelled")
             raise
         except Exception as e:
-            logger.error(f"Error in server {server_id} lifecycle: {e}")
+            logger.error(f"Error in server {server_id} lifecycle: {e}", exc_info=True)
         finally:
+            running_count = getattr(server, "_running_count", "N/A")
+            logger.info(
+                f"Server {server_id} lifecycle ending, _running_count={running_count}"
+            )
+
             # Clean up the context
             await exit_stack.aclose()
 
+            running_count_after = getattr(server, "_running_count", "N/A")
+            logger.info(
+                f"Server {server_id} context closed, _running_count={running_count_after}"
+            )
+
             # Remove from managed servers
             async with self._lock:
                 if server_id in self._servers:

code_puppy/mcp_/managed_server.py

@@ -28,6 +28,31 @@ from code_puppy.mcp_.blocking_startup import BlockingMCPServerStdio
 from code_puppy.messaging import emit_info
 
 
+def _expand_env_vars(value: Any) -> Any:
+    """
+    Recursively expand environment variables in config values.
+
+    Supports $VAR and ${VAR} syntax. Works with:
+    - Strings: expands env vars
+    - Dicts: recursively expands all string values
+    - Lists: recursively expands all string elements
+    - Other types: returned as-is
+
+    Args:
+        value: The value to expand env vars in
+
+    Returns:
+        The value with env vars expanded
+    """
+    if isinstance(value, str):
+        return os.path.expandvars(value)
+    elif isinstance(value, dict):
+        return {k: _expand_env_vars(v) for k, v in value.items()}
+    elif isinstance(value, list):
+        return [_expand_env_vars(item) for item in value]
+    return value
+
+
 class ServerState(Enum):
     """Enumeration of possible server states."""
 
@@ -153,9 +178,9 @@ class ManagedMCPServer:
                 if "url" not in config:
                     raise ValueError("SSE server requires 'url' in config")
 
-                # Prepare arguments for MCPServerSSE
+                # Prepare arguments for MCPServerSSE (expand env vars in URL)
                 sse_kwargs = {
-                    "url": config["url"],
+                    "url": _expand_env_vars(config["url"]),
                 }
 
                 # Add optional parameters if provided
@@ -177,23 +202,26 @@ class ManagedMCPServer:
                 if "command" not in config:
                     raise ValueError("Stdio server requires 'command' in config")
 
-                # Handle command and arguments
-                command = config["command"]
+                # Handle command and arguments (expand env vars)
+                command = _expand_env_vars(config["command"])
                 args = config.get("args", [])
                 if isinstance(args, str):
-                    # If args is a string, split it
-                    args = args.split()
+                    # If args is a string, split it then expand
+                    args = [_expand_env_vars(a) for a in args.split()]
+                else:
+                    args = _expand_env_vars(args)
 
                 # Prepare arguments for MCPServerStdio
                 stdio_kwargs = {"command": command, "args": list(args) if args else []}
 
-                # Add optional parameters if provided
+                # Add optional parameters if provided (expand env vars in env and cwd)
                 if "env" in config:
-                    stdio_kwargs["env"] = config["env"]
+                    stdio_kwargs["env"] = _expand_env_vars(config["env"])
                 if "cwd" in config:
-                    stdio_kwargs["cwd"] = config["cwd"]
-                if "timeout" in config:
-                    stdio_kwargs["timeout"] = config["timeout"]
+                    stdio_kwargs["cwd"] = _expand_env_vars(config["cwd"])
+                # Default timeout of 60s for stdio servers - some servers like Serena take a while to start
+                # Users can override this in their config
+                stdio_kwargs["timeout"] = config.get("timeout", 60)
                 if "read_timeout" in config:
                     stdio_kwargs["read_timeout"] = config["read_timeout"]
 
@@ -212,9 +240,9 @@ class ManagedMCPServer:
                 if "url" not in config:
                     raise ValueError("HTTP server requires 'url' in config")
 
-                # Prepare arguments for MCPServerStreamableHTTP
+                # Prepare arguments for MCPServerStreamableHTTP (expand env vars in URL)
                 http_kwargs = {
-                    "url": config["url"],
+                    "url": _expand_env_vars(config["url"]),
                 }
 
                 # Add optional parameters if provided
@@ -222,18 +250,15 @@ class ManagedMCPServer:
                     http_kwargs["timeout"] = config["timeout"]
                 if "read_timeout" in config:
                     http_kwargs["read_timeout"] = config["read_timeout"]
-                if "headers" in config:
+
+                # Pass headers directly instead of creating http_client
+                # Note: There's a bug in MCP 1.25.0 where passing http_client
+                # causes "'_AsyncGeneratorContextManager' object has no attribute 'stream'"
+                # The workaround is to pass headers directly and let pydantic-ai
+                # create the http_client internally.
+                if config.get("headers"):
                     # Expand environment variables in headers
-                    headers = config.get("headers")
-                    resolved_headers = {}
-                    if isinstance(headers, dict):
-                        for k, v in headers.items():
-                            if isinstance(v, str):
-                                resolved_headers[k] = os.path.expandvars(v)
-                            else:
-                                resolved_headers[k] = v
-                    http_kwargs["headers"] = resolved_headers
-                    # Create HTTP client if headers are provided but no client specified
+                    http_kwargs["headers"] = _expand_env_vars(config["headers"])
 
                 self._pydantic_server = MCPServerStreamableHTTP(
                     **http_kwargs, process_tool_call=process_tool_call

code_puppy/mcp_/manager.py

@@ -469,41 +469,57 @@ class MCPManager:
     def start_server_sync(self, server_id: str) -> bool:
         """
         Synchronous wrapper for start_server.
+
+        IMPORTANT: This schedules the server start as a background task.
+        The server subprocess will start asynchronously - it may not be
+        immediately ready when this function returns.
         """
         try:
-            asyncio.get_running_loop()
-            # We're in an async context, but we need to wait for completion
-            # Create a future and schedule the coroutine
+            loop = asyncio.get_running_loop()
+            # We're in an async context - schedule the server start as a background task
+            # DO NOT use blocking time.sleep() here as it freezes the event loop!
+
+            # First, enable the server immediately so it's recognized as "starting"
+            managed_server = self._managed_servers.get(server_id)
+            if managed_server:
+                managed_server.enable()
+                self.status_tracker.set_status(server_id, ServerState.STARTING)
+                self.status_tracker.record_start_time(server_id)
 
-            # Use run_in_executor to run the async function synchronously
-            async def run_async():
-                return await self.start_server(server_id)
+            # Schedule the async start_server to run in the background
+            # This will properly start the subprocess and lifecycle task
+            async def start_server_background():
+                try:
+                    result = await self.start_server(server_id)
+                    if result:
+                        logger.info(f"Background server start completed: {server_id}")
+                    else:
+                        logger.warning(f"Background server start failed: {server_id}")
+                    return result
+                except Exception as e:
+                    logger.error(f"Background server start error for {server_id}: {e}")
+                    self.status_tracker.set_status(server_id, ServerState.ERROR)
+                    return False
 
-            # Schedule the task and wait briefly for it to complete
-            task = asyncio.create_task(run_async())
+            # Create the task - it will run when the event loop gets control
+            task = loop.create_task(
+                start_server_background(), name=f"start_server_{server_id}"
+            )
 
-            # Give it a moment to complete - this fixes the race condition
-            import time
+            # Store task reference to prevent garbage collection
+            if not hasattr(self, "_pending_start_tasks"):
+                self._pending_start_tasks = {}
+            self._pending_start_tasks[server_id] = task
 
-            time.sleep(0.1)  # Small delay to let async tasks progress
+            # Add callback to clean up task reference when done
+            def cleanup_task(t):
+                if hasattr(self, "_pending_start_tasks"):
+                    self._pending_start_tasks.pop(server_id, None)
 
-            # Check if task completed, if not, fall back to sync enable
-            if task.done():
-                try:
-                    result = task.result()
-                    return result
-                except Exception:
-                    pass
+            task.add_done_callback(cleanup_task)
 
-            # If async didn't complete, enable synchronously
-            managed_server = self._managed_servers.get(server_id)
-            if managed_server:
-                managed_server.enable()
-                self.status_tracker.set_status(server_id, ServerState.RUNNING)
-                self.status_tracker.record_start_time(server_id)
-                logger.info(f"Enabled server synchronously: {server_id}")
-                return True
-            return False
+            logger.info(f"Scheduled background start for server: {server_id}")
+            return True  # Return immediately - server will start in background
 
         except RuntimeError:
             # No async loop, just enable the server
@@ -582,39 +598,52 @@ class MCPManager:
     def stop_server_sync(self, server_id: str) -> bool:
         """
         Synchronous wrapper for stop_server.
+
+        IMPORTANT: This schedules the server stop as a background task.
+        The server subprocess will stop asynchronously.
         """
         try:
-            asyncio.get_running_loop()
+            loop = asyncio.get_running_loop()
+            # We're in an async context - schedule the server stop as a background task
+            # DO NOT use blocking time.sleep() here as it freezes the event loop!
 
-            # We're in an async context, but we need to wait for completion
-            async def run_async():
-                return await self.stop_server(server_id)
+            # First, disable the server immediately
+            managed_server = self._managed_servers.get(server_id)
+            if managed_server:
+                managed_server.disable()
+                self.status_tracker.set_status(server_id, ServerState.STOPPING)
+                self.status_tracker.record_stop_time(server_id)
 
-            # Schedule the task and wait briefly for it to complete
-            task = asyncio.create_task(run_async())
+            # Schedule the async stop_server to run in the background
+            async def stop_server_background():
+                try:
+                    result = await self.stop_server(server_id)
+                    if result:
+                        logger.info(f"Background server stop completed: {server_id}")
+                    return result
+                except Exception as e:
+                    logger.error(f"Background server stop error for {server_id}: {e}")
+                    return False
 
-            # Give it a moment to complete - this fixes the race condition
-            import time
+            # Create the task - it will run when the event loop gets control
+            task = loop.create_task(
+                stop_server_background(), name=f"stop_server_{server_id}"
+            )
 
-            time.sleep(0.1)  # Small delay to let async tasks progress
+            # Store task reference to prevent garbage collection
+            if not hasattr(self, "_pending_stop_tasks"):
+                self._pending_stop_tasks = {}
+            self._pending_stop_tasks[server_id] = task
 
-            # Check if task completed, if not, fall back to sync disable
-            if task.done():
-                try:
-                    result = task.result()
-                    return result
-                except Exception:
-                    pass
+            # Add callback to clean up task reference when done
+            def cleanup_task(t):
+                if hasattr(self, "_pending_stop_tasks"):
+                    self._pending_stop_tasks.pop(server_id, None)
 
-            # If async didn't complete, disable synchronously
-            managed_server = self._managed_servers.get(server_id)
-            if managed_server:
-                managed_server.disable()
-                self.status_tracker.set_status(server_id, ServerState.STOPPED)
-                self.status_tracker.record_stop_time(server_id)
-                logger.info(f"Disabled server synchronously: {server_id}")
-                return True
-            return False
+            task.add_done_callback(cleanup_task)
+
+            logger.info(f"Scheduled background stop for server: {server_id}")
+            return True  # Return immediately - server will stop in background
 
         except RuntimeError:
             # No async loop, just disable the server