testmcpy 0.2.3__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testmcpy-0.2.3/testmcpy.egg-info → testmcpy-0.2.6}/PKG-INFO +182 -30
- {testmcpy-0.2.3 → testmcpy-0.2.6}/README.md +171 -24
- {testmcpy-0.2.3 → testmcpy-0.2.6}/pyproject.toml +12 -6
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/__init__.py +1 -1
- testmcpy-0.2.6/testmcpy/auth_debugger.py +985 -0
- testmcpy-0.2.6/testmcpy/auth_flow_recorder.py +688 -0
- testmcpy-0.2.6/testmcpy/cli/__init__.py +14 -0
- testmcpy-0.2.6/testmcpy/cli/app.py +103 -0
- testmcpy-0.2.6/testmcpy/cli/commands/__init__.py +1 -0
- testmcpy-0.2.6/testmcpy/cli/commands/mcp.py +365 -0
- testmcpy-0.2.6/testmcpy/cli/commands/run.py +933 -0
- testmcpy-0.2.6/testmcpy/cli/commands/server.py +1276 -0
- testmcpy-0.2.6/testmcpy/cli/commands/tools.py +438 -0
- testmcpy-0.2.6/testmcpy/cli/commands/tui.py +207 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/config.py +159 -136
- testmcpy-0.2.6/testmcpy/core/__init__.py +20 -0
- testmcpy-0.2.6/testmcpy/core/chat_session.py +391 -0
- testmcpy-0.2.6/testmcpy/core/docs_optimizer.py +208 -0
- testmcpy-0.2.6/testmcpy/core/mcp_manager.py +328 -0
- testmcpy-0.2.6/testmcpy/core/tool_comparison.py +443 -0
- testmcpy-0.2.6/testmcpy/core/tool_discovery.py +361 -0
- testmcpy-0.2.6/testmcpy/error_handlers.py +70 -0
- testmcpy-0.2.6/testmcpy/evals/__init__.py +57 -0
- testmcpy-0.2.6/testmcpy/evals/auth_evaluators.py +556 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/evals/base_evaluators.py +747 -15
- testmcpy-0.2.6/testmcpy/formatters/__init__.py +46 -0
- testmcpy-0.2.6/testmcpy/formatters/base.py +232 -0
- testmcpy-0.2.6/testmcpy/formatters/curl.py +82 -0
- testmcpy-0.2.6/testmcpy/formatters/graphql.py +136 -0
- testmcpy-0.2.6/testmcpy/formatters/javascript_client.py +115 -0
- testmcpy-0.2.6/testmcpy/formatters/json_yaml.py +61 -0
- testmcpy-0.2.6/testmcpy/formatters/protobuf.py +136 -0
- testmcpy-0.2.6/testmcpy/formatters/python.py +157 -0
- testmcpy-0.2.6/testmcpy/formatters/python_client.py +143 -0
- testmcpy-0.2.6/testmcpy/formatters/thrift.py +117 -0
- testmcpy-0.2.6/testmcpy/formatters/typescript.py +144 -0
- testmcpy-0.2.6/testmcpy/formatters/typescript_client.py +140 -0
- testmcpy-0.2.6/testmcpy/llm_profiles.py +243 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/mcp_profiles.py +160 -18
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/claude_sdk_poc.py +1 -1
- testmcpy-0.2.6/testmcpy/server/api.py +865 -0
- testmcpy-0.2.6/testmcpy/server/api.py.bak +3150 -0
- testmcpy-0.2.6/testmcpy/server/helpers/__init__.py +19 -0
- testmcpy-0.2.6/testmcpy/server/helpers/mcp_config.py +236 -0
- testmcpy-0.2.6/testmcpy/server/models.py +261 -0
- testmcpy-0.2.6/testmcpy/server/routers/__init__.py +8 -0
- testmcpy-0.2.6/testmcpy/server/routers/auth.py +338 -0
- testmcpy-0.2.6/testmcpy/server/routers/generation_logs.py +185 -0
- testmcpy-0.2.6/testmcpy/server/routers/llm.py +414 -0
- testmcpy-0.2.6/testmcpy/server/routers/mcp_profiles.py +833 -0
- testmcpy-0.2.6/testmcpy/server/routers/results.py +277 -0
- testmcpy-0.2.6/testmcpy/server/routers/smoke_reports.py +130 -0
- testmcpy-0.2.6/testmcpy/server/routers/test_profiles.py +187 -0
- testmcpy-0.2.6/testmcpy/server/routers/tests.py +1277 -0
- testmcpy-0.2.6/testmcpy/server/routers/tools.py +801 -0
- testmcpy-0.2.6/testmcpy/server/state.py +437 -0
- testmcpy-0.2.6/testmcpy/server/tool_compare_endpoint.py +127 -0
- testmcpy-0.2.6/testmcpy/server/websocket.py +410 -0
- testmcpy-0.2.6/testmcpy/smoke_test.py +428 -0
- testmcpy-0.2.6/testmcpy/src/llm_integration.py +2411 -0
- testmcpy-0.2.6/testmcpy/src/mcp_client.py +856 -0
- testmcpy-0.2.6/testmcpy/src/model_registry.py +509 -0
- testmcpy-0.2.6/testmcpy/src/models.py +274 -0
- testmcpy-0.2.6/testmcpy/src/runner_tools.py +535 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/src/test_runner.py +485 -56
- testmcpy-0.2.6/testmcpy/storage.py +1050 -0
- testmcpy-0.2.6/testmcpy/test_profiles.py +238 -0
- testmcpy-0.2.6/testmcpy/ui/dist/assets/index-CaEBvXci.css +1 -0
- testmcpy-0.2.6/testmcpy/ui/dist/assets/index-DbhZ0GnU.js +649 -0
- testmcpy-0.2.6/testmcpy/ui/dist/index.html +14 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/package-lock.json +1740 -130
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/package.json +5 -1
- testmcpy-0.2.6/testmcpy/ui/src/App.jsx +465 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/CompareToolsTab.jsx +144 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/ErrorAlert.jsx +68 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/ErrorBoundary.jsx +127 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/LLMProfileSelector.jsx +184 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/LoadingSpinner.jsx +65 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/MCPProfileSelector.jsx +205 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/OptimizeDocsModal.jsx +396 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/SchemaCodeViewer.jsx +206 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/SkeletonLoader.jsx +146 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/TestGenerationModal.jsx +534 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/TestProfileSelector.jsx +207 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/TestResultPanel.jsx +75 -26
- testmcpy-0.2.6/testmcpy/ui/src/components/ToolComparison.jsx +291 -0
- testmcpy-0.2.6/testmcpy/ui/src/components/ToolDebugModal.jsx +569 -0
- testmcpy-0.2.6/testmcpy/ui/src/contexts/TestRunContext.jsx +381 -0
- testmcpy-0.2.6/testmcpy/ui/src/hooks/useKeyboardShortcuts.js +168 -0
- testmcpy-0.2.6/testmcpy/ui/src/hooks/useSafeFetch.js +182 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/index.css +106 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/AuthDebugger.jsx +1434 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/ChatInterface.jsx +1056 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/pages/Configuration.jsx +23 -18
- testmcpy-0.2.6/testmcpy/ui/src/pages/GenerationHistory.jsx +634 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/LLMProfiles.jsx +1226 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/MCPExplorer.jsx +1995 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/MCPProfiles.jsx +1252 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/ProfilesManager.jsx +146 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/Reports.jsx +572 -0
- testmcpy-0.2.6/testmcpy/ui/src/pages/TestManager.jsx +1557 -0
- testmcpy-0.2.6/testmcpy/ui/src/utils/__tests__/formatConverters.test.js +170 -0
- testmcpy-0.2.6/testmcpy/ui/src/utils/formatConverters.js +807 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/tailwind.config.js +3 -1
- {testmcpy-0.2.3 → testmcpy-0.2.6/testmcpy.egg-info}/PKG-INFO +182 -30
- testmcpy-0.2.6/testmcpy.egg-info/SOURCES.txt +125 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/requires.txt +11 -5
- testmcpy-0.2.3/testmcpy/cli.py +0 -1983
- testmcpy-0.2.3/testmcpy/evals/__init__.py +0 -1
- testmcpy-0.2.3/testmcpy/server/api.py +0 -681
- testmcpy-0.2.3/testmcpy/server/websocket.py +0 -167
- testmcpy-0.2.3/testmcpy/src/llm_integration.py +0 -1209
- testmcpy-0.2.3/testmcpy/src/mcp_client.py +0 -342
- testmcpy-0.2.3/testmcpy/ui/src/App.jsx +0 -148
- testmcpy-0.2.3/testmcpy/ui/src/pages/ChatInterface.jsx +0 -676
- testmcpy-0.2.3/testmcpy/ui/src/pages/MCPExplorer.jsx +0 -278
- testmcpy-0.2.3/testmcpy/ui/src/pages/TestManager.jsx +0 -485
- testmcpy-0.2.3/testmcpy.egg-info/SOURCES.txt +0 -47
- testmcpy-0.2.3/tests/test_url_protection.py +0 -379
- {testmcpy-0.2.3 → testmcpy-0.2.6}/LICENSE +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/MANIFEST.in +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/NOTICE +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/setup.cfg +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/claude_sdk_detailed_exploration.py +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/claude_sdk_working_poc.py +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/test_ollama_tools.py +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/server/__init__.py +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/src/__init__.py +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/README.md +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/index.html +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/postcss.config.js +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/ParameterCard.jsx +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/TestStatusIndicator.jsx +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/TypeBadge.jsx +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/main.jsx +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/vite.config.js +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/dependency_links.txt +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/entry_points.txt +0 -0
- {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: testmcpy
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
|
|
5
5
|
Author: Amin Ghadersohi
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
|
|
|
18
18
|
License-File: LICENSE
|
|
19
19
|
License-File: NOTICE
|
|
20
20
|
Requires-Dist: typer<1.0.0,>=0.9.0
|
|
21
|
-
Requires-Dist: rich<
|
|
21
|
+
Requires-Dist: rich<15.0.0,>=13.0.0
|
|
22
22
|
Requires-Dist: pyyaml<7.0,>=6.0
|
|
23
23
|
Requires-Dist: requests<3.0.0,>=2.28.0
|
|
24
24
|
Requires-Dist: aiohttp<4.0.0,>=3.8.0
|
|
@@ -29,6 +29,7 @@ Requires-Dist: httpx<1.0.0,>=0.27.0
|
|
|
29
29
|
Requires-Dist: python-dotenv<2.0.0,>=1.0.0
|
|
30
30
|
Requires-Dist: click<9.0.0,>=8.0.0
|
|
31
31
|
Requires-Dist: shellingham<2.0.0,>=1.3.0
|
|
32
|
+
Requires-Dist: textual<1.0.0,>=0.47.0
|
|
32
33
|
Provides-Extra: dev
|
|
33
34
|
Requires-Dist: ruff>=0.8.0; extra == "dev"
|
|
34
35
|
Requires-Dist: mypy>=1.13.0; extra == "dev"
|
|
@@ -40,28 +41,40 @@ Requires-Dist: build>=1.0.0; extra == "dev"
|
|
|
40
41
|
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
41
42
|
Requires-Dist: types-pyyaml>=6.0.0; extra == "dev"
|
|
42
43
|
Requires-Dist: types-requests>=2.28.0; extra == "dev"
|
|
44
|
+
Requires-Dist: textual-dev>=1.0.0; extra == "dev"
|
|
43
45
|
Provides-Extra: server
|
|
44
46
|
Requires-Dist: fastapi<1.0.0,>=0.104.0; extra == "server"
|
|
45
|
-
Requires-Dist: uvicorn<1.0.0,>=0.24.0; extra == "server"
|
|
46
|
-
Requires-Dist: websockets<
|
|
47
|
+
Requires-Dist: uvicorn[standard]<1.0.0,>=0.24.0; extra == "server"
|
|
48
|
+
Requires-Dist: websockets<15.0,>=14.0; extra == "server"
|
|
47
49
|
Provides-Extra: sdk
|
|
48
50
|
Requires-Dist: claude-agent-sdk>=0.1.0; extra == "sdk"
|
|
51
|
+
Provides-Extra: tui
|
|
52
|
+
Requires-Dist: textual>=0.85.0; extra == "tui"
|
|
49
53
|
Provides-Extra: all
|
|
50
54
|
Requires-Dist: fastapi<1.0.0,>=0.104.0; extra == "all"
|
|
51
|
-
Requires-Dist: uvicorn<1.0.0,>=0.24.0; extra == "all"
|
|
52
|
-
Requires-Dist: websockets<
|
|
55
|
+
Requires-Dist: uvicorn[standard]<1.0.0,>=0.24.0; extra == "all"
|
|
56
|
+
Requires-Dist: websockets<15.0,>=14.0; extra == "all"
|
|
53
57
|
Requires-Dist: claude-agent-sdk>=0.1.0; extra == "all"
|
|
58
|
+
Requires-Dist: textual>=0.85.0; extra == "all"
|
|
54
59
|
Dynamic: license-file
|
|
55
60
|
|
|
56
|
-
|
|
61
|
+
<p align="center">
|
|
62
|
+
<img src="docs/logos/logo.svg" alt="testmcpy logo" width="600">
|
|
63
|
+
</p>
|
|
57
64
|
|
|
58
|
-
|
|
65
|
+
<p align="center">
|
|
66
|
+
<strong>Test and benchmark LLMs with MCP tools in minutes.</strong>
|
|
67
|
+
</p>
|
|
59
68
|
|
|
60
|
-
|
|
69
|
+
<p align="center">
|
|
70
|
+
A testing framework for validating how LLMs call tools via Model Context Protocol (MCP) - compare Claude, GPT-4, Llama, and other models' accuracy, cost, and performance.
|
|
71
|
+
</p>
|
|
61
72
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
73
|
+
<p align="center">
|
|
74
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
|
|
75
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License"></a>
|
|
76
|
+
<a href="https://pypi.org/project/testmcpy/"><img src="https://img.shields.io/badge/pypi-testmcpy-blue" alt="PyPI"></a>
|
|
77
|
+
</p>
|
|
65
78
|
|
|
66
79
|
[Screenshot: CLI test runner with colorful progress bars and results]
|
|
67
80
|
|
|
@@ -101,6 +114,33 @@ That's it! No complex configuration needed to get started.
|
|
|
101
114
|
|
|
102
115
|
## Key Features
|
|
103
116
|
|
|
117
|
+
### Interactive TUI Dashboard (NEW!)
|
|
118
|
+
Beautiful terminal interface for MCP testing - no browser required:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
testmcpy dash # Launch interactive dashboard
|
|
122
|
+
testmcpy dash --auto-refresh # Live connection monitoring
|
|
123
|
+
testmcpy dash --profile prod # Use specific MCP profile
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
**TUI Features:**
|
|
127
|
+
- Real-time MCP connection status
|
|
128
|
+
- Interactive tool exploration
|
|
129
|
+
- Live test execution with progress
|
|
130
|
+
- Configuration editor
|
|
131
|
+
- Global search across tools, tests, and settings
|
|
132
|
+
- Help system with keyboard shortcuts (press `?`)
|
|
133
|
+
- Multiple themes (default, light, high contrast)
|
|
134
|
+
|
|
135
|
+
**Quick CLI Commands (no TUI):**
|
|
136
|
+
```bash
|
|
137
|
+
testmcpy profiles # List MCP profiles (table)
|
|
138
|
+
testmcpy status # Connection status check
|
|
139
|
+
testmcpy explore-cli # Browse tools (non-interactive)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
[Screenshot: TUI dashboard showing profiles, quick actions, and keyboard shortcuts]
|
|
143
|
+
|
|
104
144
|
### Multi-Provider Support
|
|
105
145
|
Test with **Claude**, **GPT-4**, **Llama**, and other models. Works with both paid APIs and free local models via Ollama.
|
|
106
146
|
|
|
@@ -121,6 +161,16 @@ Comprehensive validation out of the box:
|
|
|
121
161
|
- **Optional web interface**: Visual tool explorer and interactive chat
|
|
122
162
|
- **Real-time feedback**: Watch tests execute with live updates
|
|
123
163
|
|
|
164
|
+
When you start testmcpy, you're greeted with a beautiful terminal interface:
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
▀█▀ █▀▀ █▀ ▀█▀ █▀▄▀█ █▀▀ █▀█ █▄█
|
|
168
|
+
█ ██▄ ▄█ █ █ ▀ █ █▄▄ █▀▀ █
|
|
169
|
+
|
|
170
|
+
🧪 Test • 📊 Benchmark • ✓ Validate
|
|
171
|
+
MCP Testing Framework
|
|
172
|
+
```
|
|
173
|
+
|
|
124
174
|
[Screenshot: Split view of CLI and Web UI running the same test]
|
|
125
175
|
|
|
126
176
|
### YAML Test Definitions
|
|
@@ -224,26 +274,62 @@ pip install 'testmcpy[all]'
|
|
|
224
274
|
|
|
225
275
|
### 1. Configuration
|
|
226
276
|
|
|
227
|
-
Run the interactive setup wizard:
|
|
277
|
+
Run the interactive setup wizard to create configuration files:
|
|
228
278
|
|
|
229
279
|
```bash
|
|
230
280
|
testmcpy setup
|
|
231
281
|
```
|
|
232
282
|
|
|
233
|
-
|
|
283
|
+
This will guide you through:
|
|
284
|
+
- **LLM Provider setup**: Choose between Claude (Anthropic), GPT-4 (OpenAI), or local Ollama models
|
|
285
|
+
- **MCP Service setup**: Configure your MCP server URL and authentication
|
|
286
|
+
- **API Key management**: Detects keys from environment and saves them to `.llm_providers.yaml`
|
|
234
287
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
288
|
+
The setup command creates two files in your current directory:
|
|
289
|
+
|
|
290
|
+
**`.llm_providers.yaml`** - LLM configuration with API keys:
|
|
291
|
+
|
|
292
|
+
```yaml
|
|
293
|
+
default: prod
|
|
294
|
+
|
|
295
|
+
profiles:
|
|
296
|
+
prod:
|
|
297
|
+
name: "Production"
|
|
298
|
+
description: "High-quality models for production use"
|
|
299
|
+
providers:
|
|
300
|
+
- name: "Claude claude-sonnet-4-5"
|
|
301
|
+
provider: "anthropic"
|
|
302
|
+
model: "claude-sonnet-4-5"
|
|
303
|
+
api_key: "your-anthropic-api-key-here" # API key stored directly
|
|
304
|
+
timeout: 60
|
|
305
|
+
default: true
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
**`.mcp_services.yaml`** - MCP server profiles:
|
|
309
|
+
|
|
310
|
+
```yaml
|
|
311
|
+
default: prod
|
|
312
|
+
|
|
313
|
+
profiles:
|
|
314
|
+
prod:
|
|
315
|
+
name: "Production"
|
|
316
|
+
description: "Production MCP service"
|
|
317
|
+
mcps:
|
|
318
|
+
- name: "Preset Superset"
|
|
319
|
+
mcp_url: "https://your-workspace.preset.io/mcp"
|
|
320
|
+
auth:
|
|
321
|
+
auth_type: "jwt" # or "bearer" or "none"
|
|
322
|
+
api_url: "https://api.app.preset.io/v1/auth/"
|
|
323
|
+
api_token: "your-api-token"
|
|
324
|
+
api_secret: "your-api-secret"
|
|
325
|
+
timeout: 30
|
|
326
|
+
rate_limit_rpm: 60
|
|
327
|
+
default: true
|
|
244
328
|
```
|
|
245
329
|
|
|
246
|
-
**Configuration priority:** CLI options >
|
|
330
|
+
**Configuration priority:** CLI options > LLM Profile (.llm_providers.yaml) > MCP Profile (.mcp_services.yaml) > `.env` > Environment variables
|
|
331
|
+
|
|
332
|
+
**Note:** The setup command is **idempotent** - it's safe to run multiple times. Use `--force` to overwrite existing files.
|
|
247
333
|
|
|
248
334
|
### 2. Test Your MCP Service
|
|
249
335
|
|
|
@@ -301,7 +387,12 @@ testmcpy run tests/ --model claude-haiku-4-5
|
|
|
301
387
|
|
|
302
388
|
| Command | Description |
|
|
303
389
|
|---------|-------------|
|
|
390
|
+
| `testmcpy dash` | **Launch interactive TUI dashboard** |
|
|
304
391
|
| `testmcpy setup` | Interactive configuration wizard |
|
|
392
|
+
| `testmcpy profiles` | List MCP profiles (table) |
|
|
393
|
+
| `testmcpy status` | Show MCP connection status |
|
|
394
|
+
| `testmcpy explore-cli` | Browse tools (non-interactive) |
|
|
395
|
+
| `testmcpy explorer` | Launch TUI tool explorer |
|
|
305
396
|
| `testmcpy tools` | List available MCP tools |
|
|
306
397
|
| `testmcpy research` | Test LLM tool-calling capabilities |
|
|
307
398
|
| `testmcpy run <path>` | Execute test suite |
|
|
@@ -311,14 +402,55 @@ testmcpy run tests/ --model claude-haiku-4-5
|
|
|
311
402
|
| `testmcpy config-cmd` | View current configuration |
|
|
312
403
|
| `testmcpy doctor` | Diagnose installation issues |
|
|
313
404
|
|
|
405
|
+
### TUI Keyboard Shortcuts
|
|
406
|
+
|
|
407
|
+
**Global Navigation:**
|
|
408
|
+
- `h` - Home screen
|
|
409
|
+
- `e` - Explorer (MCP tools)
|
|
410
|
+
- `5` - Configuration
|
|
411
|
+
- `?` - Help modal
|
|
412
|
+
- `/` - Global search
|
|
413
|
+
- `q` - Quit (with confirmation)
|
|
414
|
+
- `F5` - Refresh
|
|
415
|
+
|
|
416
|
+
**Home Screen:**
|
|
417
|
+
- `1-5` - Quick actions (Tests, Explorer, Chat, Optimize, Config)
|
|
418
|
+
- `p` - Switch profile
|
|
419
|
+
- `Space` - Connect/disconnect
|
|
420
|
+
|
|
421
|
+
**Explorer:**
|
|
422
|
+
- `↑↓` or `j/k` - Navigate
|
|
423
|
+
- `Enter` - View details
|
|
424
|
+
- `t` - Create test
|
|
425
|
+
- `o` - Optimize docs
|
|
426
|
+
|
|
427
|
+
**Configuration:**
|
|
428
|
+
- `Tab` - Next field
|
|
429
|
+
- `s` - Save changes
|
|
430
|
+
- `q` - Quit without saving
|
|
431
|
+
|
|
314
432
|
## LLM Providers
|
|
315
433
|
|
|
434
|
+
Configure LLM providers in `.llm_providers.yaml`. See `.llm_providers.yaml.example` for examples.
|
|
435
|
+
|
|
316
436
|
### Anthropic (Recommended)
|
|
317
437
|
Best tool-calling accuracy, native MCP support:
|
|
318
438
|
|
|
319
439
|
```bash
|
|
440
|
+
# Set API key in .env or ~/.testmcpy
|
|
320
441
|
ANTHROPIC_API_KEY=sk-ant-your-key
|
|
321
|
-
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
```yaml
|
|
445
|
+
# Configure in .llm_providers.yaml
|
|
446
|
+
prod:
|
|
447
|
+
name: "Production"
|
|
448
|
+
providers:
|
|
449
|
+
- name: "Claude Sonnet 4.5"
|
|
450
|
+
provider: "anthropic"
|
|
451
|
+
model: "claude-sonnet-4-5"
|
|
452
|
+
api_key_env: "ANTHROPIC_API_KEY"
|
|
453
|
+
default: true
|
|
322
454
|
```
|
|
323
455
|
|
|
324
456
|
**Available models:** `claude-haiku-4-5`, `claude-sonnet-4-5`, `claude-opus-4-1`
|
|
@@ -334,16 +466,36 @@ brew install ollama # macOS
|
|
|
334
466
|
# Start Ollama and pull a model
|
|
335
467
|
ollama serve
|
|
336
468
|
ollama pull llama3.1:8b
|
|
469
|
+
```
|
|
337
470
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
471
|
+
```yaml
|
|
472
|
+
# Configure in .llm_providers.yaml
|
|
473
|
+
local:
|
|
474
|
+
name: "Local Only"
|
|
475
|
+
providers:
|
|
476
|
+
- name: "Ollama Llama"
|
|
477
|
+
provider: "ollama"
|
|
478
|
+
model: "llama3.1:8b"
|
|
479
|
+
base_url: "http://localhost:11434"
|
|
480
|
+
default: true
|
|
341
481
|
```
|
|
342
482
|
|
|
343
483
|
### OpenAI
|
|
344
484
|
```bash
|
|
485
|
+
# Set API key in .env or ~/.testmcpy
|
|
345
486
|
OPENAI_API_KEY=sk-your-key
|
|
346
|
-
|
|
487
|
+
```
|
|
488
|
+
|
|
489
|
+
```yaml
|
|
490
|
+
# Configure in .llm_providers.yaml
|
|
491
|
+
openai:
|
|
492
|
+
name: "OpenAI"
|
|
493
|
+
providers:
|
|
494
|
+
- name: "GPT-4"
|
|
495
|
+
provider: "openai"
|
|
496
|
+
model: "gpt-4-turbo"
|
|
497
|
+
api_key_env: "OPENAI_API_KEY"
|
|
498
|
+
default: true
|
|
347
499
|
```
|
|
348
500
|
|
|
349
501
|
## Built-in Evaluators
|
|
@@ -469,6 +621,6 @@ By contributing, you agree that your contributions will be licensed under Apache
|
|
|
469
621
|
|
|
470
622
|
## Acknowledgments
|
|
471
623
|
|
|
472
|
-
Built
|
|
624
|
+
Built to enable better LLM testing and integration with Model Context Protocol services.
|
|
473
625
|
|
|
474
626
|
Special thanks to the MCP community and all our contributors!
|
|
@@ -1,12 +1,20 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/logos/logo.svg" alt="testmcpy logo" width="600">
|
|
3
|
+
</p>
|
|
2
4
|
|
|
3
|
-
|
|
5
|
+
<p align="center">
|
|
6
|
+
<strong>Test and benchmark LLMs with MCP tools in minutes.</strong>
|
|
7
|
+
</p>
|
|
4
8
|
|
|
5
|
-
|
|
9
|
+
<p align="center">
|
|
10
|
+
A testing framework for validating how LLMs call tools via Model Context Protocol (MCP) - compare Claude, GPT-4, Llama, and other models' accuracy, cost, and performance.
|
|
11
|
+
</p>
|
|
6
12
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
13
|
+
<p align="center">
|
|
14
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
|
|
15
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License"></a>
|
|
16
|
+
<a href="https://pypi.org/project/testmcpy/"><img src="https://img.shields.io/badge/pypi-testmcpy-blue" alt="PyPI"></a>
|
|
17
|
+
</p>
|
|
10
18
|
|
|
11
19
|
[Screenshot: CLI test runner with colorful progress bars and results]
|
|
12
20
|
|
|
@@ -46,6 +54,33 @@ That's it! No complex configuration needed to get started.
|
|
|
46
54
|
|
|
47
55
|
## Key Features
|
|
48
56
|
|
|
57
|
+
### Interactive TUI Dashboard (NEW!)
|
|
58
|
+
Beautiful terminal interface for MCP testing - no browser required:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
testmcpy dash # Launch interactive dashboard
|
|
62
|
+
testmcpy dash --auto-refresh # Live connection monitoring
|
|
63
|
+
testmcpy dash --profile prod # Use specific MCP profile
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
**TUI Features:**
|
|
67
|
+
- Real-time MCP connection status
|
|
68
|
+
- Interactive tool exploration
|
|
69
|
+
- Live test execution with progress
|
|
70
|
+
- Configuration editor
|
|
71
|
+
- Global search across tools, tests, and settings
|
|
72
|
+
- Help system with keyboard shortcuts (press `?`)
|
|
73
|
+
- Multiple themes (default, light, high contrast)
|
|
74
|
+
|
|
75
|
+
**Quick CLI Commands (no TUI):**
|
|
76
|
+
```bash
|
|
77
|
+
testmcpy profiles # List MCP profiles (table)
|
|
78
|
+
testmcpy status # Connection status check
|
|
79
|
+
testmcpy explore-cli # Browse tools (non-interactive)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
[Screenshot: TUI dashboard showing profiles, quick actions, and keyboard shortcuts]
|
|
83
|
+
|
|
49
84
|
### Multi-Provider Support
|
|
50
85
|
Test with **Claude**, **GPT-4**, **Llama**, and other models. Works with both paid APIs and free local models via Ollama.
|
|
51
86
|
|
|
@@ -66,6 +101,16 @@ Comprehensive validation out of the box:
|
|
|
66
101
|
- **Optional web interface**: Visual tool explorer and interactive chat
|
|
67
102
|
- **Real-time feedback**: Watch tests execute with live updates
|
|
68
103
|
|
|
104
|
+
When you start testmcpy, you're greeted with a beautiful terminal interface:
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
▀█▀ █▀▀ █▀ ▀█▀ █▀▄▀█ █▀▀ █▀█ █▄█
|
|
108
|
+
█ ██▄ ▄█ █ █ ▀ █ █▄▄ █▀▀ █
|
|
109
|
+
|
|
110
|
+
🧪 Test • 📊 Benchmark • ✓ Validate
|
|
111
|
+
MCP Testing Framework
|
|
112
|
+
```
|
|
113
|
+
|
|
69
114
|
[Screenshot: Split view of CLI and Web UI running the same test]
|
|
70
115
|
|
|
71
116
|
### YAML Test Definitions
|
|
@@ -169,26 +214,62 @@ pip install 'testmcpy[all]'
|
|
|
169
214
|
|
|
170
215
|
### 1. Configuration
|
|
171
216
|
|
|
172
|
-
Run the interactive setup wizard:
|
|
217
|
+
Run the interactive setup wizard to create configuration files:
|
|
173
218
|
|
|
174
219
|
```bash
|
|
175
220
|
testmcpy setup
|
|
176
221
|
```
|
|
177
222
|
|
|
178
|
-
|
|
223
|
+
This will guide you through:
|
|
224
|
+
- **LLM Provider setup**: Choose between Claude (Anthropic), GPT-4 (OpenAI), or local Ollama models
|
|
225
|
+
- **MCP Service setup**: Configure your MCP server URL and authentication
|
|
226
|
+
- **API Key management**: Detects keys from environment and saves them to `.llm_providers.yaml`
|
|
179
227
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
228
|
+
The setup command creates two files in your current directory:
|
|
229
|
+
|
|
230
|
+
**`.llm_providers.yaml`** - LLM configuration with API keys:
|
|
231
|
+
|
|
232
|
+
```yaml
|
|
233
|
+
default: prod
|
|
234
|
+
|
|
235
|
+
profiles:
|
|
236
|
+
prod:
|
|
237
|
+
name: "Production"
|
|
238
|
+
description: "High-quality models for production use"
|
|
239
|
+
providers:
|
|
240
|
+
- name: "Claude claude-sonnet-4-5"
|
|
241
|
+
provider: "anthropic"
|
|
242
|
+
model: "claude-sonnet-4-5"
|
|
243
|
+
api_key: "your-anthropic-api-key-here" # API key stored directly
|
|
244
|
+
timeout: 60
|
|
245
|
+
default: true
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
**`.mcp_services.yaml`** - MCP server profiles:
|
|
249
|
+
|
|
250
|
+
```yaml
|
|
251
|
+
default: prod
|
|
252
|
+
|
|
253
|
+
profiles:
|
|
254
|
+
prod:
|
|
255
|
+
name: "Production"
|
|
256
|
+
description: "Production MCP service"
|
|
257
|
+
mcps:
|
|
258
|
+
- name: "Preset Superset"
|
|
259
|
+
mcp_url: "https://your-workspace.preset.io/mcp"
|
|
260
|
+
auth:
|
|
261
|
+
auth_type: "jwt" # or "bearer" or "none"
|
|
262
|
+
api_url: "https://api.app.preset.io/v1/auth/"
|
|
263
|
+
api_token: "your-api-token"
|
|
264
|
+
api_secret: "your-api-secret"
|
|
265
|
+
timeout: 30
|
|
266
|
+
rate_limit_rpm: 60
|
|
267
|
+
default: true
|
|
189
268
|
```
|
|
190
269
|
|
|
191
|
-
**Configuration priority:** CLI options >
|
|
270
|
+
**Configuration priority:** CLI options > LLM Profile (.llm_providers.yaml) > MCP Profile (.mcp_services.yaml) > `.env` > Environment variables
|
|
271
|
+
|
|
272
|
+
**Note:** The setup command is **idempotent** - it's safe to run multiple times. Use `--force` to overwrite existing files.
|
|
192
273
|
|
|
193
274
|
### 2. Test Your MCP Service
|
|
194
275
|
|
|
@@ -246,7 +327,12 @@ testmcpy run tests/ --model claude-haiku-4-5
|
|
|
246
327
|
|
|
247
328
|
| Command | Description |
|
|
248
329
|
|---------|-------------|
|
|
330
|
+
| `testmcpy dash` | **Launch interactive TUI dashboard** |
|
|
249
331
|
| `testmcpy setup` | Interactive configuration wizard |
|
|
332
|
+
| `testmcpy profiles` | List MCP profiles (table) |
|
|
333
|
+
| `testmcpy status` | Show MCP connection status |
|
|
334
|
+
| `testmcpy explore-cli` | Browse tools (non-interactive) |
|
|
335
|
+
| `testmcpy explorer` | Launch TUI tool explorer |
|
|
250
336
|
| `testmcpy tools` | List available MCP tools |
|
|
251
337
|
| `testmcpy research` | Test LLM tool-calling capabilities |
|
|
252
338
|
| `testmcpy run <path>` | Execute test suite |
|
|
@@ -256,14 +342,55 @@ testmcpy run tests/ --model claude-haiku-4-5
|
|
|
256
342
|
| `testmcpy config-cmd` | View current configuration |
|
|
257
343
|
| `testmcpy doctor` | Diagnose installation issues |
|
|
258
344
|
|
|
345
|
+
### TUI Keyboard Shortcuts
|
|
346
|
+
|
|
347
|
+
**Global Navigation:**
|
|
348
|
+
- `h` - Home screen
|
|
349
|
+
- `e` - Explorer (MCP tools)
|
|
350
|
+
- `5` - Configuration
|
|
351
|
+
- `?` - Help modal
|
|
352
|
+
- `/` - Global search
|
|
353
|
+
- `q` - Quit (with confirmation)
|
|
354
|
+
- `F5` - Refresh
|
|
355
|
+
|
|
356
|
+
**Home Screen:**
|
|
357
|
+
- `1-5` - Quick actions (Tests, Explorer, Chat, Optimize, Config)
|
|
358
|
+
- `p` - Switch profile
|
|
359
|
+
- `Space` - Connect/disconnect
|
|
360
|
+
|
|
361
|
+
**Explorer:**
|
|
362
|
+
- `↑↓` or `j/k` - Navigate
|
|
363
|
+
- `Enter` - View details
|
|
364
|
+
- `t` - Create test
|
|
365
|
+
- `o` - Optimize docs
|
|
366
|
+
|
|
367
|
+
**Configuration:**
|
|
368
|
+
- `Tab` - Next field
|
|
369
|
+
- `s` - Save changes
|
|
370
|
+
- `q` - Quit without saving
|
|
371
|
+
|
|
259
372
|
## LLM Providers
|
|
260
373
|
|
|
374
|
+
Configure LLM providers in `.llm_providers.yaml`. See `.llm_providers.yaml.example` for examples.
|
|
375
|
+
|
|
261
376
|
### Anthropic (Recommended)
|
|
262
377
|
Best tool-calling accuracy, native MCP support:
|
|
263
378
|
|
|
264
379
|
```bash
|
|
380
|
+
# Set API key in .env or ~/.testmcpy
|
|
265
381
|
ANTHROPIC_API_KEY=sk-ant-your-key
|
|
266
|
-
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
```yaml
|
|
385
|
+
# Configure in .llm_providers.yaml
|
|
386
|
+
prod:
|
|
387
|
+
name: "Production"
|
|
388
|
+
providers:
|
|
389
|
+
- name: "Claude Sonnet 4.5"
|
|
390
|
+
provider: "anthropic"
|
|
391
|
+
model: "claude-sonnet-4-5"
|
|
392
|
+
api_key_env: "ANTHROPIC_API_KEY"
|
|
393
|
+
default: true
|
|
267
394
|
```
|
|
268
395
|
|
|
269
396
|
**Available models:** `claude-haiku-4-5`, `claude-sonnet-4-5`, `claude-opus-4-1`
|
|
@@ -279,16 +406,36 @@ brew install ollama # macOS
|
|
|
279
406
|
# Start Ollama and pull a model
|
|
280
407
|
ollama serve
|
|
281
408
|
ollama pull llama3.1:8b
|
|
409
|
+
```
|
|
282
410
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
411
|
+
```yaml
|
|
412
|
+
# Configure in .llm_providers.yaml
|
|
413
|
+
local:
|
|
414
|
+
name: "Local Only"
|
|
415
|
+
providers:
|
|
416
|
+
- name: "Ollama Llama"
|
|
417
|
+
provider: "ollama"
|
|
418
|
+
model: "llama3.1:8b"
|
|
419
|
+
base_url: "http://localhost:11434"
|
|
420
|
+
default: true
|
|
286
421
|
```
|
|
287
422
|
|
|
288
423
|
### OpenAI
|
|
289
424
|
```bash
|
|
425
|
+
# Set API key in .env or ~/.testmcpy
|
|
290
426
|
OPENAI_API_KEY=sk-your-key
|
|
291
|
-
|
|
427
|
+
```
|
|
428
|
+
|
|
429
|
+
```yaml
|
|
430
|
+
# Configure in .llm_providers.yaml
|
|
431
|
+
openai:
|
|
432
|
+
name: "OpenAI"
|
|
433
|
+
providers:
|
|
434
|
+
- name: "GPT-4"
|
|
435
|
+
provider: "openai"
|
|
436
|
+
model: "gpt-4-turbo"
|
|
437
|
+
api_key_env: "OPENAI_API_KEY"
|
|
438
|
+
default: true
|
|
292
439
|
```
|
|
293
440
|
|
|
294
441
|
## Built-in Evaluators
|
|
@@ -414,6 +561,6 @@ By contributing, you agree that your contributions will be licensed under Apache
|
|
|
414
561
|
|
|
415
562
|
## Acknowledgments
|
|
416
563
|
|
|
417
|
-
Built
|
|
564
|
+
Built to enable better LLM testing and integration with Model Context Protocol services.
|
|
418
565
|
|
|
419
566
|
Special thanks to the MCP community and all our contributors!
|
|
@@ -88,7 +88,7 @@ testmcpy = [
|
|
|
88
88
|
|
|
89
89
|
[project]
|
|
90
90
|
name = "testmcpy"
|
|
91
|
-
version = "0.2.
|
|
91
|
+
version = "0.2.6"
|
|
92
92
|
description = "A comprehensive testing framework for validating LLM tool calling capabilities with MCP services"
|
|
93
93
|
authors = [{name = "Amin Ghadersohi"}]
|
|
94
94
|
license = "Apache-2.0"
|
|
@@ -104,7 +104,7 @@ classifiers = [
|
|
|
104
104
|
]
|
|
105
105
|
dependencies = [
|
|
106
106
|
"typer>=0.9.0,<1.0.0",
|
|
107
|
-
"rich>=13.0.0,<
|
|
107
|
+
"rich>=13.0.0,<15.0.0",
|
|
108
108
|
"pyyaml>=6.0,<7.0",
|
|
109
109
|
"requests>=2.28.0,<3.0.0",
|
|
110
110
|
"aiohttp>=3.8.0,<4.0.0",
|
|
@@ -115,6 +115,7 @@ dependencies = [
|
|
|
115
115
|
"python-dotenv>=1.0.0,<2.0.0",
|
|
116
116
|
"click>=8.0.0,<9.0.0",
|
|
117
117
|
"shellingham>=1.3.0,<2.0.0",
|
|
118
|
+
"textual>=0.47.0,<1.0.0",
|
|
118
119
|
]
|
|
119
120
|
|
|
120
121
|
[project.optional-dependencies]
|
|
@@ -129,20 +130,25 @@ dev = [
|
|
|
129
130
|
"twine>=5.0.0",
|
|
130
131
|
"types-pyyaml>=6.0.0",
|
|
131
132
|
"types-requests>=2.28.0",
|
|
133
|
+
"textual-dev>=1.0.0",
|
|
132
134
|
]
|
|
133
135
|
server = [
|
|
134
136
|
"fastapi>=0.104.0,<1.0.0",
|
|
135
|
-
"uvicorn>=0.24.0,<1.0.0",
|
|
136
|
-
"websockets>=
|
|
137
|
+
"uvicorn[standard]>=0.24.0,<1.0.0",
|
|
138
|
+
"websockets>=14.0,<15.0",
|
|
137
139
|
]
|
|
138
140
|
sdk = [
|
|
139
141
|
"claude-agent-sdk>=0.1.0",
|
|
140
142
|
]
|
|
143
|
+
tui = [
|
|
144
|
+
"textual>=0.85.0",
|
|
145
|
+
]
|
|
141
146
|
all = [
|
|
142
147
|
"fastapi>=0.104.0,<1.0.0",
|
|
143
|
-
"uvicorn>=0.24.0,<1.0.0",
|
|
144
|
-
"websockets>=
|
|
148
|
+
"uvicorn[standard]>=0.24.0,<1.0.0",
|
|
149
|
+
"websockets>=14.0,<15.0",
|
|
145
150
|
"claude-agent-sdk>=0.1.0",
|
|
151
|
+
"textual>=0.85.0",
|
|
146
152
|
]
|
|
147
153
|
|
|
148
154
|
[project.scripts]
|