testmcpy 0.3.1__tar.gz → 0.3.2__tar.gz

{testmcpy-0.3.1/testmcpy.egg-info → testmcpy-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testmcpy
-Version: 0.3.1
+Version: 0.3.2
 Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
 Author: Amin Ghadersohi
 License-Expression: Apache-2.0

{testmcpy-0.3.1 → testmcpy-0.3.2}/pyproject.toml

@@ -93,7 +93,7 @@ testmcpy = [
 
 [project]
 name = "testmcpy"
-version = "0.3.1"
+version = "0.3.2"
 description = "A comprehensive testing framework for validating LLM tool calling capabilities with MCP services"
 authors = [{name = "Amin Ghadersohi"}]
 license = "Apache-2.0"

{testmcpy-0.3.1 → testmcpy-0.3.2}/testmcpy/__init__.py

@@ -11,6 +11,6 @@ try:
     __version__ = version("testmcpy")
 except Exception:
     # Fallback for development or when package not installed
-    __version__ = "0.3.1"
+    __version__ = "0.3.2"
 
 __author__ = "testmcpy Contributors"

{testmcpy-0.3.1 → testmcpy-0.3.2}/testmcpy/evals/base_evaluators.py

@@ -4,6 +4,7 @@ Base evaluation functions for testmcpy.
 These evaluators can be used to validate LLM responses and tool calling behavior.
 """
 
+import json
 import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
@@ -2531,11 +2532,11 @@ class UnnecessaryToolCalls(BaseEvaluator):
                 continue
 
             if self.check_args:
-                # Include args in the signature (sorted for consistency)
+                # Canonical serialization for stable duplicate detection
                 args = tc.get("arguments", {})
                 try:
-                    args_key = str(sorted(args.items())) if isinstance(args, dict) else str(args)
-                except TypeError:
+                    args_key = json.dumps(args, sort_keys=True, default=str)
+                except (TypeError, ValueError):
                     args_key = str(args)
                 sig = f"{tool_name}({args_key})"
             else:
@@ -2560,11 +2561,18 @@ class UnnecessaryToolCalls(BaseEvaluator):
         total_calls = len(tool_calls)
         score = max(0.0, 1.0 - (total_excess / total_calls))
 
+        # Truncate signatures in reason to avoid leaking large args
+        short_sigs = []
+        for sig, count in duplicates.items():
+            if len(sig) > 80:
+                sig = sig[:77] + "..."
+            short_sigs.append(f"{sig} ({count}x)")
+
         return EvalResult(
             passed=False,
             score=score,
             reason=f"Found {len(duplicates)} duplicate tool call pattern(s): "
-            + ", ".join(f"{sig} ({count}x)" for sig, count in duplicates.items()),
+            + ", ".join(short_sigs),
             details={
                 "duplicates": duplicates,
                 "total_excess_calls": total_excess,

{testmcpy-0.3.1 → testmcpy-0.3.2}/testmcpy/src/test_runner.py

@@ -419,21 +419,6 @@ class TestRunner:
             # Ensure initialized
             await self.initialize()
 
-            # Run setup hooks (MCP tool calls before test)
-            if test_case.setup:
-                for setup_call in test_case.setup:
-                    tool_name = setup_call.get("tool", "")
-                    tool_args = setup_call.get("args", {})
-                    if tool_name and self.mcp_client:
-                        try:
-                            mcp_call = MCPToolCall(name=tool_name, arguments=tool_args)
-                            await self.mcp_client.call_tool(mcp_call)
-                            if self.verbose:
-                                self._log(f"  Setup: {tool_name}({tool_args})")
-                        except Exception as setup_err:
-                            if self.verbose:
-                                self._log(f"  Setup failed: {tool_name}: {setup_err}")
-
             # If test has auth config, create a temporary MCP client with that auth
             test_mcp_client = self.mcp_client
             if test_case.auth:
@@ -484,6 +469,17 @@ class TestRunner:
                     if self.verbose:
                         print(f"  Authentication failed: {auth_error}")
 
+            # Run setup hooks against the resolved MCP client (after auth)
+            if test_case.setup:
+                for setup_call in test_case.setup:
+                    tool_name = setup_call.get("tool", "")
+                    tool_args = setup_call.get("args", {})
+                    if tool_name and test_mcp_client:
+                        mcp_call = MCPToolCall(name=tool_name, arguments=tool_args)
+                        await test_mcp_client.call_tool(mcp_call)
+                        if self.verbose:
+                            self._log(f"  Setup: {tool_name}({tool_args})")
+
             # Get available MCP tools
             mcp_tools = await test_mcp_client.list_tools()
 
@@ -729,15 +725,16 @@ class TestRunner:
             )
 
         finally:
-            # Run teardown hooks (MCP tool calls after test)
-            if test_case.teardown and self.mcp_client:
+            # Run teardown hooks against the same client used for the test
+            teardown_client = test_mcp_client if "test_mcp_client" in locals() else self.mcp_client
+            if test_case.teardown and teardown_client:
                 for teardown_call in test_case.teardown:
                     tool_name = teardown_call.get("tool", "")
                     tool_args = teardown_call.get("args", {})
                     if tool_name:
                         try:
                             mcp_call = MCPToolCall(name=tool_name, arguments=tool_args)
-                            await self.mcp_client.call_tool(mcp_call)
+                            await teardown_client.call_tool(mcp_call)
                             if self.verbose:
                                 self._log(f"  Teardown: {tool_name}({tool_args})")
                         except Exception as teardown_err:

{testmcpy-0.3.1 → testmcpy-0.3.2/testmcpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testmcpy
-Version: 0.3.1
+Version: 0.3.2
 Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
 Author: Amin Ghadersohi
 License-Expression: Apache-2.0

{testmcpy-0.3.1 → testmcpy-0.3.2}/testmcpy.egg-info/SOURCES.txt

@@ -122,9 +122,6 @@ testmcpy/ui/package.json
 testmcpy/ui/postcss.config.js
 testmcpy/ui/tailwind.config.js
 testmcpy/ui/vite.config.js
-testmcpy/ui/dist/index.html
-testmcpy/ui/dist/assets/index-B1_waOsr.js
-testmcpy/ui/dist/assets/index-Do693nG2.css
 testmcpy/ui/src/App.jsx
 testmcpy/ui/src/index.css
 testmcpy/ui/src/main.jsx