testmcpy 0.9.0__tar.gz → 0.9.2__tar.gz

{testmcpy-0.9.0/testmcpy.egg-info → testmcpy-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testmcpy
-Version: 0.9.0
+Version: 0.9.2
 Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
 Author: Amin Ghadersohi
 License-Expression: Apache-2.0
@@ -351,6 +351,13 @@ profiles:
 
 The setup command is **idempotent** — safe to run multiple times. Use `--force` to overwrite existing files.
 
+**`TESTMCPY_CHAT_OAUTH_LOGIN`** (default `true`): when a chat message hits an
+OAuth (`oauth_auto_discover`) MCP profile with no cached token, the server opens
+the interactive browser OAuth flow and retries. This assumes a browser is
+available on the machine running the server — in headless deployments set
+`TESTMCPY_CHAT_OAUTH_LOGIN=false` so the request fails fast with a clear error
+instead of blocking on a login that can never complete.
+
 ### 2. Explore Your MCP Service
 
 ```bash

{testmcpy-0.9.0 → testmcpy-0.9.2}/README.md

@@ -271,6 +271,13 @@ profiles:
 
 The setup command is **idempotent** — safe to run multiple times. Use `--force` to overwrite existing files.
 
+**`TESTMCPY_CHAT_OAUTH_LOGIN`** (default `true`): when a chat message hits an
+OAuth (`oauth_auto_discover`) MCP profile with no cached token, the server opens
+the interactive browser OAuth flow and retries. This assumes a browser is
+available on the machine running the server — in headless deployments set
+`TESTMCPY_CHAT_OAUTH_LOGIN=false` so the request fails fast with a clear error
+instead of blocking on a login that can never complete.
+
 ### 2. Explore Your MCP Service
 
 ```bash

{testmcpy-0.9.0 → testmcpy-0.9.2}/pyproject.toml

@@ -93,7 +93,7 @@ testmcpy = [
 
 [project]
 name = "testmcpy"
-version = "0.9.0"
+version = "0.9.2"
 description = "A comprehensive testing framework for validating LLM tool calling capabilities with MCP services"
 authors = [{name = "Amin Ghadersohi"}]
 license = "Apache-2.0"

{testmcpy-0.9.0 → testmcpy-0.9.2}/testmcpy/server/api.py

@@ -151,6 +151,22 @@ def _get_init_lock(cache_key: str) -> asyncio.Lock:
     return _client_init_locks[cache_key]
 
 
+def _primary_mcp_provider_kwargs(
+    clients_to_use: list[tuple[str, str, MCPClient]],
+) -> dict[str, Any]:
+    """mcp_url/auth kwargs from the FIRST selected MCP client.
+
+    SDK providers support a single MCP server; the Chat UI sends exactly one
+    "profileId:mcpName". Without these kwargs the providers fall back to the
+    DEFAULT profile's URL/auth, breaking chat for any other selected profile.
+    create_llm_provider filters these out for providers that don't accept them.
+    """
+    if not clients_to_use:
+        return {}
+    _profile_id, _mcp_name, client = clients_to_use[0]
+    return {"mcp_url": client.base_url, "auth": client.auth_config}
+
+
 async def get_mcp_clients_for_profile(profile_id: str) -> list[tuple[str, MCPClient]]:
     """
     Get or create MCP clients for all MCP servers in a profile.
@@ -299,12 +315,15 @@ async def get_mcp_client_for_server(profile_id: str, mcp_name: str) -> MCPClient
         return client
 
 
-async def clear_cached_client(cache_key: str) -> bool:
+async def clear_cached_client(cache_key: str, record_failure: bool = True) -> bool:
     """
     Clear a cached MCP client by its cache key.
 
     Args:
         cache_key: Cache key in format "{profile_id}:{mcp_name}"
+        record_failure: When True (default), throttle the next reconnect via
+            back-off. Pass False for deliberate re-initialization (e.g. an
+            interactive OAuth re-login) where an immediate reconnect is wanted.
 
     Returns:
         True if a client was cleared, False if no client was cached
@@ -313,8 +332,9 @@ async def clear_cached_client(cache_key: str) -> bool:
 
     client = mcp_clients.pop(cache_key, None)
     if client:
-        # Record a failure so the next reconnect is throttled via back-off.
-        _record_failure(cache_key)
+        if record_failure:
+            # Record a failure so the next reconnect is throttled via back-off.
+            _record_failure(cache_key)
         try:
             await client.close()
             print(f"Cleared cached client '{cache_key}'")
@@ -324,6 +344,69 @@ async def clear_cached_client(cache_key: str) -> bool:
     return False
 
 
+# Marker substring of the ValueError raised by BaseSDKProvider when an
+# oauth_auto_discover profile has no cached token (see
+# llm_integration.BaseSDKProvider._resolve_mcp_bearer_token).
+_OAUTH_TOKEN_ERROR = "No usable cached OAuth token"
+
+
+def _chat_oauth_login_enabled() -> bool:
+    """Feature flag for interactive OAuth login during chat (default ON).
+
+    Disable with TESTMCPY_CHAT_OAUTH_LOGIN=false (or 0/no). Read at call time
+    so tests can monkeypatch the environment.
+    """
+    return os.environ.get("TESTMCPY_CHAT_OAUTH_LOGIN", "true").strip().lower() not in (
+        "0",
+        "false",
+        "no",
+    )
+
+
+async def _relogin_oauth_servers(server_keys: list[str]) -> dict[str, MCPClient]:
+    """Deliberate interactive re-auth for the given "profileId:mcpName" keys.
+
+    Drops cached clients WITHOUT recording back-off, clears any pre-existing
+    back-off state, and re-initializes. MCPClient.initialize() with
+    oauth_auto_discover opens the browser OAuth flow and caches the token via
+    fastmcp FileTokenStorage; duplicate popups are prevented by the per-key
+    init locks.
+
+    Returns the fresh clients keyed by cache key so callers can replace any
+    references to the old, now-closed client objects.
+    """
+    new_clients: dict[str, MCPClient] = {}
+    for cache_key in server_keys:
+        await clear_cached_client(cache_key, record_failure=False)
+        _clear_failure(cache_key)  # earlier failures must not block deliberate re-auth
+        profile_id, mcp_name = cache_key.split(":", 1)
+        client = await get_mcp_client_for_server(profile_id, mcp_name)
+        if client:
+            new_clients[cache_key] = client
+    return new_clients
+
+
+def _refresh_client_refs(
+    new_clients: dict[str, MCPClient],
+    clients_to_use: list[tuple[str, str, MCPClient]],
+    tool_to_client: dict[str, tuple[MCPClient, str, str]],
+) -> tuple[list[tuple[str, str, MCPClient]], dict[str, tuple[MCPClient, str, str]]]:
+    """Swap re-logged-in clients into the chat endpoints' lookup structures.
+
+    After _relogin_oauth_servers the old client objects are closed; tool
+    execution through tool_to_client must use the replacements.
+    """
+    refreshed_clients = [
+        (pid, name, new_clients.get(f"{pid}:{name}", client))
+        for pid, name, client in clients_to_use
+    ]
+    refreshed_tools = {
+        tool: (new_clients.get(f"{pid}:{name}", client), pid, name)
+        for tool, (client, pid, name) in tool_to_client.items()
+    }
+    return refreshed_clients, refreshed_tools
+
+
 def is_auth_error(error_msg: str) -> bool:
     """Check if an error message indicates an authentication failure."""
     error_lower = error_msg.lower()
@@ -936,9 +1019,25 @@ async def chat(request: ChatRequest) -> ChatResponse:
         provider_kwargs = {}
         if api_key:
             provider_kwargs["api_key"] = api_key
-        llm_provider = create_llm_provider(provider, model, **provider_kwargs)
+        provider_kwargs.update(_primary_mcp_provider_kwargs(clients_to_use))
         print("[Chat] Initializing LLM provider...")
-        await llm_provider.initialize()
+        try:
+            llm_provider = create_llm_provider(provider, model, **provider_kwargs)
+            await llm_provider.initialize()
+        except ValueError as e:
+            if not (_chat_oauth_login_enabled() and _OAUTH_TOKEN_ERROR in str(e)):
+                raise
+            print("[Chat] No cached OAuth token; triggering interactive OAuth login...")
+            new_clients = await _relogin_oauth_servers(accessed_servers)
+            # The old client objects are closed now — swap in the replacements
+            # so tool execution doesn't hit a closed client.
+            clients_to_use, tool_to_client = _refresh_client_refs(
+                new_clients, clients_to_use, tool_to_client
+            )
+            provider_kwargs.update(_primary_mcp_provider_kwargs(clients_to_use))
+            llm_provider = create_llm_provider(provider, model, **provider_kwargs)
+            # Single retry; a second failure falls to the existing handlers.
+            await llm_provider.initialize()
         print(
             f"[Chat] LLM provider initialized. Generating response with {len(all_tools)} tools..."
         )
@@ -1209,8 +1308,24 @@ async def chat_stream(request: ChatRequest):
             provider_kwargs: dict = {}
             if api_key:
                 provider_kwargs["api_key"] = api_key
-            llm_provider = create_llm_provider(provider, model, **provider_kwargs)
-            await llm_provider.initialize()
+            provider_kwargs.update(_primary_mcp_provider_kwargs(clients_to_use))
+            try:
+                llm_provider = create_llm_provider(provider, model, **provider_kwargs)
+                await llm_provider.initialize()
+            except ValueError as e:
+                if not (_chat_oauth_login_enabled() and _OAUTH_TOKEN_ERROR in str(e)):
+                    raise
+                yield send_event("status", "Waiting for OAuth login in browser...")
+                new_clients = await _relogin_oauth_servers(accessed_servers)
+                # The old client objects are closed now — swap in the replacements
+                # so tool execution doesn't hit a closed client.
+                clients_to_use, tool_to_client = _refresh_client_refs(
+                    new_clients, clients_to_use, tool_to_client
+                )
+                provider_kwargs.update(_primary_mcp_provider_kwargs(clients_to_use))
+                llm_provider = create_llm_provider(provider, model, **provider_kwargs)
+                # Single retry; a second failure falls to the existing handlers.
+                await llm_provider.initialize()
 
             # --- Detect if provider is SDK-based (handles its own agentic loop) ---
             from testmcpy.src.llm_integration import ClaudeSDKProvider

{testmcpy-0.9.0 → testmcpy-0.9.2}/testmcpy/src/llm_integration.py

@@ -930,6 +930,7 @@ class AnthropicProvider(LLMProvider):
         api_key: str | None = None,
         base_url: str = "https://api.anthropic.com",
         mcp_url: str | None = None,
+        auth: dict[str, Any] | None = None,
     ):
         self.model = model
         # Use config system for API key
@@ -940,11 +941,10 @@ class AnthropicProvider(LLMProvider):
         # Use MCP_URL and auth from default profile if not provided
         if mcp_url is None:
             mcp_url = config.get_mcp_url()
-        # Get auth from default MCP server
-        auth = None
-        default_mcp = config.get_default_mcp_server()
-        if default_mcp and default_mcp.auth:
-            auth = default_mcp.auth.to_dict()
+        if auth is None:
+            default_mcp = config.get_default_mcp_server()
+            if default_mcp and default_mcp.auth:
+                auth = default_mcp.auth.to_dict()
         self.tool_discovery = ToolDiscoveryService(mcp_url, auth=auth)
 
     async def initialize(self):

{testmcpy-0.9.0 → testmcpy-0.9.2}/testmcpy/storage.py

@@ -138,21 +138,38 @@ class TestStorage:
         # a manual `alembic upgrade` on existing DB files.
         self._apply_column_migrations()
 
+    @staticmethod
+    def _sqlite_default_literal(column) -> str | None:
+        """SQLite DDL DEFAULT literal for a mapped column's scalar default.
+
+        Only plain scalar defaults are rendered (bool/int/float/str) — callables
+        and server-side expressions return None so the column is added without
+        a DEFAULT (NULL for existing rows, Python-side default for new ones).
+        """
+        default = column.default
+        if default is None or not getattr(default, "is_scalar", False):
+            return None
+        value = default.arg
+        if isinstance(value, bool):
+            return str(int(value))
+        if isinstance(value, (int, float)):
+            return str(value)
+        if isinstance(value, str):
+            escaped = value.replace("'", "''")
+            return f"'{escaped}'"
+        return None
+
     def _apply_column_migrations(self) -> None:
         """Idempotently add columns that were introduced after the initial schema.
 
         create_all() creates missing *tables* but never alters existing ones, so
         users who created their DB before a new column was added will hit
-        OperationalError on INSERT.  This method uses PRAGMA table_info to detect
-        missing columns and issues ALTER TABLE … ADD COLUMN for each one.
+        OperationalError on SELECT/INSERT.  Missing columns are derived from the
+        ORM metadata (every mapped column not present in the live table) rather
+        than a hand-maintained list — the previous list drifted from the models
+        more than once (e.g. ``manual_false_positive`` was missed, breaking
+        ``GET /api/results/run/{id}`` on older DB files with a 500).
         """
-        # (table, column, DDL type)
-        migrations = [
-            ("question_results", "tool_call_counts", "JSON"),
-            ("question_results", "false_positive_rate", "FLOAT DEFAULT 0.0"),
-            ("test_runs", "total_cost", "FLOAT DEFAULT 0.0"),
-            ("test_runs", "heartbeat_at", "VARCHAR"),
-        ]
         # (index name, table, columns) — indexes added after the initial
         # schema; create_all skips existing tables so these need explicit DDL
         index_migrations = [
@@ -172,14 +189,23 @@ class TestStorage:
             if is_sqlite:
                 # PRAGMA is SQLite-only; non-SQLite backends are expected
                 # to be managed with `alembic upgrade head`.
-                for table, column, col_type in migrations:
-                    result = conn.execute(text(f"PRAGMA table_info({table})"))
+                for table in Base.metadata.sorted_tables:
+                    result = conn.execute(text(f"PRAGMA table_info({table.name})"))
                     existing = {row[1] for row in result}
-                    if column not in existing:
-                        conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {col_type}"))
-            for index_name, table, columns in index_migrations:
+                    if not existing:
+                        continue  # create_all just made it; nothing to alter
+                    for column in table.columns:
+                        if column.name in existing:
+                            continue
+                        ddl_type = column.type.compile(self._engine.dialect)
+                        ddl = f"ALTER TABLE {table.name} ADD COLUMN {column.name} {ddl_type}"
+                        default = self._sqlite_default_literal(column)
+                        if default is not None:
+                            ddl += f" DEFAULT {default}"
+                        conn.execute(text(ddl))
+            for index_name, table_name, columns in index_migrations:
                 conn.execute(
-                    text(f"CREATE INDEX IF NOT EXISTS {index_name} ON {table} ({columns})")
+                    text(f"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name} ({columns})")
                 )
             conn.commit()
 

{testmcpy-0.9.0 → testmcpy-0.9.2/testmcpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testmcpy
-Version: 0.9.0
+Version: 0.9.2
 Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
 Author: Amin Ghadersohi
 License-Expression: Apache-2.0
@@ -351,6 +351,13 @@ profiles:
 
 The setup command is **idempotent** — safe to run multiple times. Use `--force` to overwrite existing files.
 
+**`TESTMCPY_CHAT_OAUTH_LOGIN`** (default `true`): when a chat message hits an
+OAuth (`oauth_auto_discover`) MCP profile with no cached token, the server opens
+the interactive browser OAuth flow and retries. This assumes a browser is
+available on the machine running the server — in headless deployments set
+`TESTMCPY_CHAT_OAUTH_LOGIN=false` so the request fails fast with a clear error
+instead of blocking on a login that can never complete.
+
 ### 2. Explore Your MCP Service
 
 ```bash