testmcpy 0.7.0__tar.gz → 0.7.2__tar.gz

{testmcpy-0.7.0/testmcpy.egg-info → testmcpy-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testmcpy
-Version: 0.7.0
+Version: 0.7.2
 Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
 Author: Amin Ghadersohi
 License-Expression: Apache-2.0

{testmcpy-0.7.0 → testmcpy-0.7.2}/pyproject.toml

@@ -93,7 +93,7 @@ testmcpy = [
 
 [project]
 name = "testmcpy"
-version = "0.7.0"
+version = "0.7.2"
 description = "A comprehensive testing framework for validating LLM tool calling capabilities with MCP services"
 authors = [{name = "Amin Ghadersohi"}]
 license = "Apache-2.0"

{testmcpy-0.7.0 → testmcpy-0.7.2}/testmcpy/__init__.py

@@ -11,6 +11,6 @@ try:
     __version__ = version("testmcpy")
 except Exception:
     # Fallback for development or when package not installed
-    __version__ = "0.7.0"
+    __version__ = "0.7.2"
 
 __author__ = "testmcpy Contributors"

{testmcpy-0.7.0 → testmcpy-0.7.2}/testmcpy/src/llm_integration.py

@@ -1356,6 +1356,30 @@ class BedrockProvider(LLMProvider):
 _claude_sdk_logger = logging.getLogger(__name__ + ".ClaudeSDKProvider")
 
 
+# Substrings that strongly indicate a tool-result is an error, even when
+# the SDK didn't flag is_error=True. Used by the retry-budget guard to
+# detect cases like MCP-side validation errors that come back as "successful"
+# tool results with error text in the body.
+_ERROR_PAYLOAD_MARKERS = (
+    "validation error",
+    "Unexpected keyword argument",
+    "Missing required argument",
+    "missing_argument",
+    "unexpected_keyword_argument",
+    '"error":',
+    "'error':",
+    "Error:",
+)
+
+
+def _looks_like_error_payload(content: Any) -> bool:
+    """Heuristic: does this tool-result content look like an error?"""
+    text = str(content) if content is not None else ""
+    if not text:
+        return False
+    return any(marker in text for marker in _ERROR_PAYLOAD_MARKERS)
+
+
 class ClaudeSDKProvider(LLMProvider):
     """Claude Agent SDK provider with native MCP integration.
 
@@ -1712,10 +1736,21 @@ class ClaudeSDKProvider(LLMProvider):
             cost = 0.0
             raw_events = []
 
+            # Retry budget: if the model keeps calling the SAME tool with the
+            # SAME arguments and getting the SAME error, abort the query
+            # rather than letting it spin until the wall-clock timeout fires.
+            # Counts how many times each (tool_name, args, error) signature
+            # has been seen; we abort when any signature crosses the
+            # threshold below.
+            error_signature_counts: dict[tuple[str, str, str], int] = {}
+            max_repeats_per_signature = 3
+            retry_budget_aborted = False
+
             log(f"[ClaudeSDK] Starting query (model={self.model}, timeout={timeout}s)...")
 
             async def execute_query():
                 nonlocal response_text, thinking_text, token_usage, cost
+                nonlocal retry_budget_aborted
                 message_count = 0
                 # Track all text blocks per AssistantMessage so we can
                 # identify the FINAL text response (after all tool calls)
@@ -1805,6 +1840,50 @@ class ClaudeSDKProvider(LLMProvider):
                                             f"[ClaudeSDK] Tool Result ({status}): {content_preview}"
                                         )
 
+                                        # Retry-budget enforcement: if the model
+                                        # keeps making the same call with the
+                                        # same args and getting the same error,
+                                        # abort to break out of the loop.
+                                        if is_error or _looks_like_error_payload(content):
+                                            matching_call = next(
+                                                (
+                                                    tc
+                                                    for tc in tool_calls
+                                                    if tc.get("id") == tool_use_id
+                                                ),
+                                                None,
+                                            )
+                                            if matching_call:
+                                                sig_args = json.dumps(
+                                                    matching_call.get("arguments", {}),
+                                                    sort_keys=True,
+                                                    default=str,
+                                                )[:200]
+                                                # Use a normalized prefix of the
+                                                # error text — exact byte match
+                                                # would be too brittle.
+                                                sig_err = str(content)[:120]
+                                                sig = (
+                                                    matching_call.get("name", ""),
+                                                    sig_args,
+                                                    sig_err,
+                                                )
+                                                error_signature_counts[sig] = (
+                                                    error_signature_counts.get(sig, 0) + 1
+                                                )
+                                                if (
+                                                    error_signature_counts[sig]
+                                                    >= max_repeats_per_signature
+                                                ):
+                                                    log(
+                                                        f"[ClaudeSDK] Retry budget exhausted: "
+                                                        f"same call+error repeated "
+                                                        f"{max_repeats_per_signature}× — aborting "
+                                                        f"(tool={sig[0]}, error={sig_err[:80]!r})"
+                                                    )
+                                                    retry_budget_aborted = True
+                                                    return
+
                         elif isinstance(message, ResultMessage):
                             if message.usage:
                                 usage = message.usage
@@ -1882,9 +1961,25 @@ class ClaudeSDKProvider(LLMProvider):
                     mcp_tool_results.append(mcp_result)
 
             duration = time.time() - start_time
+
+            # If we aborted via the retry budget, surface that in the
+            # response so evaluators see a clear, actionable error
+            # rather than an empty / partial response.
+            if retry_budget_aborted and not response_text:
+                response_text = (
+                    f"Error: aborted after the model repeated the same tool call "
+                    f"and got the same error {max_repeats_per_signature}× in a row. "
+                    f"This usually means the prompt is priming the model toward a "
+                    f"wrong parameter name, the tool's schema mismatches the model's "
+                    f"expectation, or the resource being queried doesn't exist. See "
+                    f"the log lines marked '[ClaudeSDK] Tool Result (Error)' for the "
+                    f"specific error pattern."
+                )
+
             log(
                 f"[ClaudeSDK] Done: {len(response_text)} chars, "
                 f"{len(tool_calls)} tool calls, {len(mcp_tool_results)} results"
+                + (" [retry budget aborted]" if retry_budget_aborted else "")
             )
 
             # Estimate cost from tokens if SDK didn't provide it (subscription billing)
@@ -1955,6 +2050,21 @@ class ClaudeSDKProvider(LLMProvider):
 _assistant_logger = logging.getLogger(__name__ + ".AssistantProvider")
 
 
+def _format_seconds(seconds: float) -> str:
+    """Format a duration so sub-second values aren't rounded to ``0s``.
+
+    The SSE idle threshold is configurable down to fractions of a second
+    (the unit tests override it to 0.3s). Using ``f"{x:.0f}s"`` would
+    produce a misleading ``0s`` in those messages — switch to ms below
+    1s, decimals below 10s, and integer seconds otherwise.
+    """
+    if seconds < 1.0:
+        return f"{seconds * 1000:.0f}ms"
+    if seconds < 10.0:
+        return f"{seconds:.1f}s"
+    return f"{seconds:.0f}s"
+
+
 @dataclass
 class _SSEStreamState:
     """Mutable state accumulated as we parse a chatbot SSE response."""
@@ -2016,6 +2126,13 @@ class AssistantProvider(LLMProvider):
     _DEFAULT_CONVERSATIONS_PATH = "/api/v1/copilot/conversations"
     _DEFAULT_COMPLETIONS_PATH = "/api/v1/copilot/completions"
 
+    # If the SSE stream emits no recognized event for this many seconds,
+    # abort the stream. Defends against a chatbot backend that keeps the
+    # connection open (preventing httpx's per-event read timeout from
+    # firing) but stops emitting real progress. Observed in c29
+    # (SC-105915). Class-level so subclasses / tests can override.
+    SSE_IDLE_ABORT_SECONDS: float = 90.0
+
     def __init__(
         self,
         model: str = "default",
@@ -2140,6 +2257,16 @@ class AssistantProvider(LLMProvider):
 
         log(f"[Assistant] POST {completions_url} (conversation={self._conversation_id})")
 
+        # Idle abort: if the SSE stream emits NO recognized event within
+        # this many seconds, give up. This catches a chatbot backend that
+        # keeps the connection open (so httpx's per-event read timeout
+        # never fires) but stops sending real progress — observed in
+        # eval cycle c29 (SC-105915) where C00_9, C01_9, C02_7 hung
+        # despite the per-test wall-clock added in v0.7.1.
+        sse_idle_abort_seconds = self.SSE_IDLE_ABORT_SECONDS
+        last_event_at = time.time()
+        idle_aborted = False
+
         state = _SSEStreamState()
         try:
             async with self._client.stream(
@@ -2153,7 +2280,40 @@ class AssistantProvider(LLMProvider):
                     )
 
                 current_event: str | None = None
-                async for raw_line in resp.aiter_lines():
+                # Drive the line iterator manually so we can wrap each
+                # await in asyncio.wait_for(...). httpx's aiter_lines()
+                # blocks inside __anext__ when no bytes arrive — a plain
+                # `async for` would be suspended forever. The wait_for
+                # catches the case where the SSE connection stays open
+                # but never sends another byte (real-world c29 hang).
+                line_iter = resp.aiter_lines().__aiter__()
+                budget_str = _format_seconds(sse_idle_abort_seconds)
+                while True:
+                    elapsed = time.time() - last_event_at
+                    remaining = sse_idle_abort_seconds - elapsed
+                    if remaining <= 0:
+                        log(
+                            f"[Assistant] SSE idle abort: no recognized event for "
+                            f"{budget_str} — closing stream"
+                        )
+                        idle_aborted = True
+                        break
+                    try:
+                        raw_line = await asyncio.wait_for(line_iter.__anext__(), timeout=remaining)
+                    except StopAsyncIteration:
+                        break
+                    except asyncio.TimeoutError:
+                        # Budget is measured since the last *recognized* event.
+                        # Unrecognized noise (keepalives, malformed events) does
+                        # NOT reset last_event_at, so this fires correctly even
+                        # if bytes are arriving without real progress.
+                        log(
+                            f"[Assistant] SSE idle abort: no recognized event for "
+                            f"{budget_str} — closing stream"
+                        )
+                        idle_aborted = True
+                        break
+
                     line = raw_line.strip()
                     if not line:
                         current_event = None
@@ -2178,6 +2338,8 @@ class AssistantProvider(LLMProvider):
                         continue
 
                     self._handle_sse_event(current_event, data, state, log)
+                    # A real event arrived — reset the idle timer.
+                    last_event_at = time.time()
 
         except httpx.TimeoutException:
             duration = time.time() - start_time
@@ -2201,13 +2363,22 @@ class AssistantProvider(LLMProvider):
         duration = time.time() - start_time
         if state.got_error and not state.response_text:
             state.response_text = f"Error: {state.error_message}"
+        elif idle_aborted and not state.response_text:
+            # Surface the idle abort cleanly so evaluators don't see an
+            # empty response with no explanation.
+            state.response_text = (
+                f"Error: SSE stream went idle for "
+                f"{_format_seconds(sse_idle_abort_seconds)} without sending a "
+                "final / error event. The chatbot backend kept the connection "
+                "open but stopped emitting progress. Aborted to free the runner."
+            )
 
         log(
             f"[Assistant] Done: {len(state.response_text)} chars, "
             f"{len(state.tool_calls)} tool calls, {state.token_event_count} tokens, "
             f"final={'yes' if state.got_final else 'no'}, "
             f"error={'yes' if state.got_error else 'no'}, "
-            f"{duration:.2f}s"
+            f"{duration:.2f}s" + (" [SSE idle aborted]" if idle_aborted else "")
         )
 
         return LLMResult(

{testmcpy-0.7.0 → testmcpy-0.7.2}/testmcpy/src/test_runner.py

@@ -1360,20 +1360,68 @@ class TestRunner:
 
         return results
 
+    # Safety margin added to each test's per-call timeout to compute the
+    # wall-clock budget. Covers auth + conversation create + tool execution
+    # + evaluator runtime. Class-level so tests can shorten it.
+    WALL_CLOCK_SLACK_SECONDS: float = 60.0
+
     async def _run_test_with_retry(
         self, test_case: TestCase, max_test_retries: int = 2
     ) -> TestResult:
-        """Run a test with retry logic for rate limit failures."""
-        for attempt in range(max_test_retries + 1):
-            # Dispatch to appropriate runner
+        """Run a test with retry logic for rate limit failures.
+
+        Each test is wrapped in a wall-clock timeout so it can never hang
+        the runner indefinitely. The wall-clock timeout is the test's
+        declared ``timeout`` plus :attr:`WALL_CLOCK_SLACK_SECONDS` to
+        absorb provider-side overhead (auth, conversation creation,
+        evaluators); for CLI providers we also bump it up like
+        :meth:`run_test` does for the inner LLM call.
+
+        Without this, providers that stream events (e.g. the assistant
+        chatbot endpoint) can keep the test alive forever — the per-event
+        httpx timeout resets on every received chunk, so a chatbot stuck
+        in an infinite tool-call retry loop keeps streaming until the
+        runner is killed externally.
+        """
+        # Compute the wall-clock budget once per test (used across retries).
+        cli_providers = ("claude-sdk", "claude-cli", "claude-code", "codex-cli", "codex")
+        per_call_timeout = test_case.timeout
+        if self.provider in cli_providers:
+            per_call_timeout = max(per_call_timeout, 120.0)
+        wall_clock_timeout = per_call_timeout + self.WALL_CLOCK_SLACK_SECONDS
+
+        async def _dispatch() -> TestResult:
             if test_case.is_auth_only:
-                result = await self.run_auth_only_test(test_case)
-            elif test_case.is_load_test:
-                result = await self.run_load_test(test_case)
-            elif test_case.is_multi_turn:
-                result = await self.run_multi_turn_test(test_case)
-            else:
-                result = await self.run_test(test_case)
+                return await self.run_auth_only_test(test_case)
+            if test_case.is_load_test:
+                return await self.run_load_test(test_case)
+            if test_case.is_multi_turn:
+                return await self.run_multi_turn_test(test_case)
+            return await self.run_test(test_case)
+
+        for attempt in range(max_test_retries + 1):
+            try:
+                result = await asyncio.wait_for(_dispatch(), timeout=wall_clock_timeout)
+            except asyncio.TimeoutError:
+                if self.verbose:
+                    self._log(
+                        f"  Test wall-clock timeout after {wall_clock_timeout:.1f}s — "
+                        "the provider was streaming or retrying without making progress"
+                    )
+                result = TestResult(
+                    test_name=test_case.name,
+                    passed=False,
+                    score=0.0,
+                    duration=wall_clock_timeout,
+                    response=(
+                        f"Error: test wall-clock timeout after {wall_clock_timeout:.1f}s. "
+                        "The provider did not return a final result in time. This is a "
+                        "hard cap independent of the per-call timeout, used to break out "
+                        "of provider-side retry loops or stuck SSE streams."
+                    ),
+                    reason="wall-clock timeout",
+                    error="wall-clock timeout",
+                )
 
             # Check if this was a rate limit failure
             is_rate_limit_failure = (

{testmcpy-0.7.0 → testmcpy-0.7.2/testmcpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testmcpy
-Version: 0.7.0
+Version: 0.7.2
 Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
 Author: Amin Ghadersohi
 License-Expression: Apache-2.0