testmcpy 0.2.3__tar.gz → 0.2.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. {testmcpy-0.2.3/testmcpy.egg-info → testmcpy-0.2.6}/PKG-INFO +182 -30
  2. {testmcpy-0.2.3 → testmcpy-0.2.6}/README.md +171 -24
  3. {testmcpy-0.2.3 → testmcpy-0.2.6}/pyproject.toml +12 -6
  4. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/__init__.py +1 -1
  5. testmcpy-0.2.6/testmcpy/auth_debugger.py +985 -0
  6. testmcpy-0.2.6/testmcpy/auth_flow_recorder.py +688 -0
  7. testmcpy-0.2.6/testmcpy/cli/__init__.py +14 -0
  8. testmcpy-0.2.6/testmcpy/cli/app.py +103 -0
  9. testmcpy-0.2.6/testmcpy/cli/commands/__init__.py +1 -0
  10. testmcpy-0.2.6/testmcpy/cli/commands/mcp.py +365 -0
  11. testmcpy-0.2.6/testmcpy/cli/commands/run.py +933 -0
  12. testmcpy-0.2.6/testmcpy/cli/commands/server.py +1276 -0
  13. testmcpy-0.2.6/testmcpy/cli/commands/tools.py +438 -0
  14. testmcpy-0.2.6/testmcpy/cli/commands/tui.py +207 -0
  15. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/config.py +159 -136
  16. testmcpy-0.2.6/testmcpy/core/__init__.py +20 -0
  17. testmcpy-0.2.6/testmcpy/core/chat_session.py +391 -0
  18. testmcpy-0.2.6/testmcpy/core/docs_optimizer.py +208 -0
  19. testmcpy-0.2.6/testmcpy/core/mcp_manager.py +328 -0
  20. testmcpy-0.2.6/testmcpy/core/tool_comparison.py +443 -0
  21. testmcpy-0.2.6/testmcpy/core/tool_discovery.py +361 -0
  22. testmcpy-0.2.6/testmcpy/error_handlers.py +70 -0
  23. testmcpy-0.2.6/testmcpy/evals/__init__.py +57 -0
  24. testmcpy-0.2.6/testmcpy/evals/auth_evaluators.py +556 -0
  25. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/evals/base_evaluators.py +747 -15
  26. testmcpy-0.2.6/testmcpy/formatters/__init__.py +46 -0
  27. testmcpy-0.2.6/testmcpy/formatters/base.py +232 -0
  28. testmcpy-0.2.6/testmcpy/formatters/curl.py +82 -0
  29. testmcpy-0.2.6/testmcpy/formatters/graphql.py +136 -0
  30. testmcpy-0.2.6/testmcpy/formatters/javascript_client.py +115 -0
  31. testmcpy-0.2.6/testmcpy/formatters/json_yaml.py +61 -0
  32. testmcpy-0.2.6/testmcpy/formatters/protobuf.py +136 -0
  33. testmcpy-0.2.6/testmcpy/formatters/python.py +157 -0
  34. testmcpy-0.2.6/testmcpy/formatters/python_client.py +143 -0
  35. testmcpy-0.2.6/testmcpy/formatters/thrift.py +117 -0
  36. testmcpy-0.2.6/testmcpy/formatters/typescript.py +144 -0
  37. testmcpy-0.2.6/testmcpy/formatters/typescript_client.py +140 -0
  38. testmcpy-0.2.6/testmcpy/llm_profiles.py +243 -0
  39. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/mcp_profiles.py +160 -18
  40. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/claude_sdk_poc.py +1 -1
  41. testmcpy-0.2.6/testmcpy/server/api.py +865 -0
  42. testmcpy-0.2.6/testmcpy/server/api.py.bak +3150 -0
  43. testmcpy-0.2.6/testmcpy/server/helpers/__init__.py +19 -0
  44. testmcpy-0.2.6/testmcpy/server/helpers/mcp_config.py +236 -0
  45. testmcpy-0.2.6/testmcpy/server/models.py +261 -0
  46. testmcpy-0.2.6/testmcpy/server/routers/__init__.py +8 -0
  47. testmcpy-0.2.6/testmcpy/server/routers/auth.py +338 -0
  48. testmcpy-0.2.6/testmcpy/server/routers/generation_logs.py +185 -0
  49. testmcpy-0.2.6/testmcpy/server/routers/llm.py +414 -0
  50. testmcpy-0.2.6/testmcpy/server/routers/mcp_profiles.py +833 -0
  51. testmcpy-0.2.6/testmcpy/server/routers/results.py +277 -0
  52. testmcpy-0.2.6/testmcpy/server/routers/smoke_reports.py +130 -0
  53. testmcpy-0.2.6/testmcpy/server/routers/test_profiles.py +187 -0
  54. testmcpy-0.2.6/testmcpy/server/routers/tests.py +1277 -0
  55. testmcpy-0.2.6/testmcpy/server/routers/tools.py +801 -0
  56. testmcpy-0.2.6/testmcpy/server/state.py +437 -0
  57. testmcpy-0.2.6/testmcpy/server/tool_compare_endpoint.py +127 -0
  58. testmcpy-0.2.6/testmcpy/server/websocket.py +410 -0
  59. testmcpy-0.2.6/testmcpy/smoke_test.py +428 -0
  60. testmcpy-0.2.6/testmcpy/src/llm_integration.py +2411 -0
  61. testmcpy-0.2.6/testmcpy/src/mcp_client.py +856 -0
  62. testmcpy-0.2.6/testmcpy/src/model_registry.py +509 -0
  63. testmcpy-0.2.6/testmcpy/src/models.py +274 -0
  64. testmcpy-0.2.6/testmcpy/src/runner_tools.py +535 -0
  65. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/src/test_runner.py +485 -56
  66. testmcpy-0.2.6/testmcpy/storage.py +1050 -0
  67. testmcpy-0.2.6/testmcpy/test_profiles.py +238 -0
  68. testmcpy-0.2.6/testmcpy/ui/dist/assets/index-CaEBvXci.css +1 -0
  69. testmcpy-0.2.6/testmcpy/ui/dist/assets/index-DbhZ0GnU.js +649 -0
  70. testmcpy-0.2.6/testmcpy/ui/dist/index.html +14 -0
  71. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/package-lock.json +1740 -130
  72. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/package.json +5 -1
  73. testmcpy-0.2.6/testmcpy/ui/src/App.jsx +465 -0
  74. testmcpy-0.2.6/testmcpy/ui/src/components/CompareToolsTab.jsx +144 -0
  75. testmcpy-0.2.6/testmcpy/ui/src/components/ErrorAlert.jsx +68 -0
  76. testmcpy-0.2.6/testmcpy/ui/src/components/ErrorBoundary.jsx +127 -0
  77. testmcpy-0.2.6/testmcpy/ui/src/components/LLMProfileSelector.jsx +184 -0
  78. testmcpy-0.2.6/testmcpy/ui/src/components/LoadingSpinner.jsx +65 -0
  79. testmcpy-0.2.6/testmcpy/ui/src/components/MCPProfileSelector.jsx +205 -0
  80. testmcpy-0.2.6/testmcpy/ui/src/components/OptimizeDocsModal.jsx +396 -0
  81. testmcpy-0.2.6/testmcpy/ui/src/components/SchemaCodeViewer.jsx +206 -0
  82. testmcpy-0.2.6/testmcpy/ui/src/components/SkeletonLoader.jsx +146 -0
  83. testmcpy-0.2.6/testmcpy/ui/src/components/TestGenerationModal.jsx +534 -0
  84. testmcpy-0.2.6/testmcpy/ui/src/components/TestProfileSelector.jsx +207 -0
  85. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/TestResultPanel.jsx +75 -26
  86. testmcpy-0.2.6/testmcpy/ui/src/components/ToolComparison.jsx +291 -0
  87. testmcpy-0.2.6/testmcpy/ui/src/components/ToolDebugModal.jsx +569 -0
  88. testmcpy-0.2.6/testmcpy/ui/src/contexts/TestRunContext.jsx +381 -0
  89. testmcpy-0.2.6/testmcpy/ui/src/hooks/useKeyboardShortcuts.js +168 -0
  90. testmcpy-0.2.6/testmcpy/ui/src/hooks/useSafeFetch.js +182 -0
  91. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/index.css +106 -0
  92. testmcpy-0.2.6/testmcpy/ui/src/pages/AuthDebugger.jsx +1434 -0
  93. testmcpy-0.2.6/testmcpy/ui/src/pages/ChatInterface.jsx +1056 -0
  94. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/pages/Configuration.jsx +23 -18
  95. testmcpy-0.2.6/testmcpy/ui/src/pages/GenerationHistory.jsx +634 -0
  96. testmcpy-0.2.6/testmcpy/ui/src/pages/LLMProfiles.jsx +1226 -0
  97. testmcpy-0.2.6/testmcpy/ui/src/pages/MCPExplorer.jsx +1995 -0
  98. testmcpy-0.2.6/testmcpy/ui/src/pages/MCPProfiles.jsx +1252 -0
  99. testmcpy-0.2.6/testmcpy/ui/src/pages/ProfilesManager.jsx +146 -0
  100. testmcpy-0.2.6/testmcpy/ui/src/pages/Reports.jsx +572 -0
  101. testmcpy-0.2.6/testmcpy/ui/src/pages/TestManager.jsx +1557 -0
  102. testmcpy-0.2.6/testmcpy/ui/src/utils/__tests__/formatConverters.test.js +170 -0
  103. testmcpy-0.2.6/testmcpy/ui/src/utils/formatConverters.js +807 -0
  104. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/tailwind.config.js +3 -1
  105. {testmcpy-0.2.3 → testmcpy-0.2.6/testmcpy.egg-info}/PKG-INFO +182 -30
  106. testmcpy-0.2.6/testmcpy.egg-info/SOURCES.txt +125 -0
  107. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/requires.txt +11 -5
  108. testmcpy-0.2.3/testmcpy/cli.py +0 -1983
  109. testmcpy-0.2.3/testmcpy/evals/__init__.py +0 -1
  110. testmcpy-0.2.3/testmcpy/server/api.py +0 -681
  111. testmcpy-0.2.3/testmcpy/server/websocket.py +0 -167
  112. testmcpy-0.2.3/testmcpy/src/llm_integration.py +0 -1209
  113. testmcpy-0.2.3/testmcpy/src/mcp_client.py +0 -342
  114. testmcpy-0.2.3/testmcpy/ui/src/App.jsx +0 -148
  115. testmcpy-0.2.3/testmcpy/ui/src/pages/ChatInterface.jsx +0 -676
  116. testmcpy-0.2.3/testmcpy/ui/src/pages/MCPExplorer.jsx +0 -278
  117. testmcpy-0.2.3/testmcpy/ui/src/pages/TestManager.jsx +0 -485
  118. testmcpy-0.2.3/testmcpy.egg-info/SOURCES.txt +0 -47
  119. testmcpy-0.2.3/tests/test_url_protection.py +0 -379
  120. {testmcpy-0.2.3 → testmcpy-0.2.6}/LICENSE +0 -0
  121. {testmcpy-0.2.3 → testmcpy-0.2.6}/MANIFEST.in +0 -0
  122. {testmcpy-0.2.3 → testmcpy-0.2.6}/NOTICE +0 -0
  123. {testmcpy-0.2.3 → testmcpy-0.2.6}/setup.cfg +0 -0
  124. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/claude_sdk_detailed_exploration.py +0 -0
  125. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/claude_sdk_working_poc.py +0 -0
  126. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/research/test_ollama_tools.py +0 -0
  127. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/server/__init__.py +0 -0
  128. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/src/__init__.py +0 -0
  129. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/README.md +0 -0
  130. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/index.html +0 -0
  131. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/postcss.config.js +0 -0
  132. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/ParameterCard.jsx +0 -0
  133. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/TestStatusIndicator.jsx +0 -0
  134. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/components/TypeBadge.jsx +0 -0
  135. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/src/main.jsx +0 -0
  136. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy/ui/vite.config.js +0 -0
  137. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/dependency_links.txt +0 -0
  138. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/entry_points.txt +0 -0
  139. {testmcpy-0.2.3 → testmcpy-0.2.6}/testmcpy.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: testmcpy
3
- Version: 0.2.3
3
+ Version: 0.2.6
4
4
  Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
5
5
  Author: Amin Ghadersohi
6
6
  License-Expression: Apache-2.0
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
18
18
  License-File: LICENSE
19
19
  License-File: NOTICE
20
20
  Requires-Dist: typer<1.0.0,>=0.9.0
21
- Requires-Dist: rich<14.0.0,>=13.0.0
21
+ Requires-Dist: rich<15.0.0,>=13.0.0
22
22
  Requires-Dist: pyyaml<7.0,>=6.0
23
23
  Requires-Dist: requests<3.0.0,>=2.28.0
24
24
  Requires-Dist: aiohttp<4.0.0,>=3.8.0
@@ -29,6 +29,7 @@ Requires-Dist: httpx<1.0.0,>=0.27.0
29
29
  Requires-Dist: python-dotenv<2.0.0,>=1.0.0
30
30
  Requires-Dist: click<9.0.0,>=8.0.0
31
31
  Requires-Dist: shellingham<2.0.0,>=1.3.0
32
+ Requires-Dist: textual<1.0.0,>=0.47.0
32
33
  Provides-Extra: dev
33
34
  Requires-Dist: ruff>=0.8.0; extra == "dev"
34
35
  Requires-Dist: mypy>=1.13.0; extra == "dev"
@@ -40,28 +41,40 @@ Requires-Dist: build>=1.0.0; extra == "dev"
40
41
  Requires-Dist: twine>=5.0.0; extra == "dev"
41
42
  Requires-Dist: types-pyyaml>=6.0.0; extra == "dev"
42
43
  Requires-Dist: types-requests>=2.28.0; extra == "dev"
44
+ Requires-Dist: textual-dev>=1.0.0; extra == "dev"
43
45
  Provides-Extra: server
44
46
  Requires-Dist: fastapi<1.0.0,>=0.104.0; extra == "server"
45
- Requires-Dist: uvicorn<1.0.0,>=0.24.0; extra == "server"
46
- Requires-Dist: websockets<13.0,>=12.0; extra == "server"
47
+ Requires-Dist: uvicorn[standard]<1.0.0,>=0.24.0; extra == "server"
48
+ Requires-Dist: websockets<15.0,>=14.0; extra == "server"
47
49
  Provides-Extra: sdk
48
50
  Requires-Dist: claude-agent-sdk>=0.1.0; extra == "sdk"
51
+ Provides-Extra: tui
52
+ Requires-Dist: textual>=0.85.0; extra == "tui"
49
53
  Provides-Extra: all
50
54
  Requires-Dist: fastapi<1.0.0,>=0.104.0; extra == "all"
51
- Requires-Dist: uvicorn<1.0.0,>=0.24.0; extra == "all"
52
- Requires-Dist: websockets<13.0,>=12.0; extra == "all"
55
+ Requires-Dist: uvicorn[standard]<1.0.0,>=0.24.0; extra == "all"
56
+ Requires-Dist: websockets<15.0,>=14.0; extra == "all"
53
57
  Requires-Dist: claude-agent-sdk>=0.1.0; extra == "all"
58
+ Requires-Dist: textual>=0.85.0; extra == "all"
54
59
  Dynamic: license-file
55
60
 
56
- # testmcpy
61
+ <p align="center">
62
+ <img src="docs/logos/logo.svg" alt="testmcpy logo" width="600">
63
+ </p>
57
64
 
58
- **Test and benchmark LLMs with MCP tools in minutes.**
65
+ <p align="center">
66
+ <strong>Test and benchmark LLMs with MCP tools in minutes.</strong>
67
+ </p>
59
68
 
60
- A testing framework for validating how LLMs call tools via Model Context Protocol (MCP) - compare Claude, GPT-4, Llama, and other models' accuracy, cost, and performance.
69
+ <p align="center">
70
+ A testing framework for validating how LLMs call tools via Model Context Protocol (MCP) - compare Claude, GPT-4, Llama, and other models' accuracy, cost, and performance.
71
+ </p>
61
72
 
62
- [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
63
- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
64
- [![PyPI](https://img.shields.io/badge/pypi-testmcpy-blue)](https://pypi.org/project/testmcpy/)
73
+ <p align="center">
74
+ <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
75
+ <a href="LICENSE"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License"></a>
76
+ <a href="https://pypi.org/project/testmcpy/"><img src="https://img.shields.io/badge/pypi-testmcpy-blue" alt="PyPI"></a>
77
+ </p>
65
78
 
66
79
  [Screenshot: CLI test runner with colorful progress bars and results]
67
80
 
@@ -101,6 +114,33 @@ That's it! No complex configuration needed to get started.
101
114
 
102
115
  ## Key Features
103
116
 
117
+ ### Interactive TUI Dashboard (NEW!)
118
+ Beautiful terminal interface for MCP testing - no browser required:
119
+
120
+ ```bash
121
+ testmcpy dash # Launch interactive dashboard
122
+ testmcpy dash --auto-refresh # Live connection monitoring
123
+ testmcpy dash --profile prod # Use specific MCP profile
124
+ ```
125
+
126
+ **TUI Features:**
127
+ - Real-time MCP connection status
128
+ - Interactive tool exploration
129
+ - Live test execution with progress
130
+ - Configuration editor
131
+ - Global search across tools, tests, and settings
132
+ - Help system with keyboard shortcuts (press `?`)
133
+ - Multiple themes (default, light, high contrast)
134
+
135
+ **Quick CLI Commands (no TUI):**
136
+ ```bash
137
+ testmcpy profiles # List MCP profiles (table)
138
+ testmcpy status # Connection status check
139
+ testmcpy explore-cli # Browse tools (non-interactive)
140
+ ```
141
+
142
+ [Screenshot: TUI dashboard showing profiles, quick actions, and keyboard shortcuts]
143
+
104
144
  ### Multi-Provider Support
105
145
  Test with **Claude**, **GPT-4**, **Llama**, and other models. Works with both paid APIs and free local models via Ollama.
106
146
 
@@ -121,6 +161,16 @@ Comprehensive validation out of the box:
121
161
  - **Optional web interface**: Visual tool explorer and interactive chat
122
162
  - **Real-time feedback**: Watch tests execute with live updates
123
163
 
164
+ When you start testmcpy, you're greeted with a beautiful terminal interface:
165
+
166
+ ```
167
+ ▀█▀ █▀▀ █▀ ▀█▀ █▀▄▀█ █▀▀ █▀█ █▄█
168
+ █ ██▄ ▄█ █ █ ▀ █ █▄▄ █▀▀ █
169
+
170
+ 🧪 Test • 📊 Benchmark • ✓ Validate
171
+ MCP Testing Framework
172
+ ```
173
+
124
174
  [Screenshot: Split view of CLI and Web UI running the same test]
125
175
 
126
176
  ### YAML Test Definitions
@@ -224,26 +274,62 @@ pip install 'testmcpy[all]'
224
274
 
225
275
  ### 1. Configuration
226
276
 
227
- Run the interactive setup wizard:
277
+ Run the interactive setup wizard to create configuration files:
228
278
 
229
279
  ```bash
230
280
  testmcpy setup
231
281
  ```
232
282
 
233
- Or manually create `~/.testmcpy`:
283
+ This will guide you through:
284
+ - **LLM Provider setup**: Choose between Claude (Anthropic), GPT-4 (OpenAI), or local Ollama models
285
+ - **MCP Service setup**: Configure your MCP server URL and authentication
286
+ - **API Key management**: Detects keys from environment and saves them to `.llm_providers.yaml`
234
287
 
235
- ```bash
236
- # MCP Service
237
- MCP_URL=http://localhost:5008/mcp/
238
- MCP_AUTH_TOKEN=your_bearer_token
239
-
240
- # LLM Provider (choose one)
241
- DEFAULT_PROVIDER=anthropic
242
- DEFAULT_MODEL=claude-haiku-4-5
243
- ANTHROPIC_API_KEY=sk-ant-...
288
+ The setup command creates two files in your current directory:
289
+
290
+ **`.llm_providers.yaml`** - LLM configuration with API keys:
291
+
292
+ ```yaml
293
+ default: prod
294
+
295
+ profiles:
296
+ prod:
297
+ name: "Production"
298
+ description: "High-quality models for production use"
299
+ providers:
300
+ - name: "Claude claude-sonnet-4-5"
301
+ provider: "anthropic"
302
+ model: "claude-sonnet-4-5"
303
+ api_key: "your-anthropic-api-key-here" # API key stored directly
304
+ timeout: 60
305
+ default: true
306
+ ```
307
+
308
+ **`.mcp_services.yaml`** - MCP server profiles:
309
+
310
+ ```yaml
311
+ default: prod
312
+
313
+ profiles:
314
+ prod:
315
+ name: "Production"
316
+ description: "Production MCP service"
317
+ mcps:
318
+ - name: "Preset Superset"
319
+ mcp_url: "https://your-workspace.preset.io/mcp"
320
+ auth:
321
+ auth_type: "jwt" # or "bearer" or "none"
322
+ api_url: "https://api.app.preset.io/v1/auth/"
323
+ api_token: "your-api-token"
324
+ api_secret: "your-api-secret"
325
+ timeout: 30
326
+ rate_limit_rpm: 60
327
+ default: true
244
328
  ```
245
329
 
246
- **Configuration priority:** CLI options > `.env` > `~/.testmcpy` > Environment variables > Defaults
330
+ **Configuration priority:** CLI options > LLM Profile (.llm_providers.yaml) > MCP Profile (.mcp_services.yaml) > `.env` > Environment variables
331
+
332
+ **Note:** The setup command is **idempotent** - it's safe to run multiple times. Use `--force` to overwrite existing files.
247
333
 
248
334
  ### 2. Test Your MCP Service
249
335
 
@@ -301,7 +387,12 @@ testmcpy run tests/ --model claude-haiku-4-5
301
387
 
302
388
  | Command | Description |
303
389
  |---------|-------------|
390
+ | `testmcpy dash` | **Launch interactive TUI dashboard** |
304
391
  | `testmcpy setup` | Interactive configuration wizard |
392
+ | `testmcpy profiles` | List MCP profiles (table) |
393
+ | `testmcpy status` | Show MCP connection status |
394
+ | `testmcpy explore-cli` | Browse tools (non-interactive) |
395
+ | `testmcpy explorer` | Launch TUI tool explorer |
305
396
  | `testmcpy tools` | List available MCP tools |
306
397
  | `testmcpy research` | Test LLM tool-calling capabilities |
307
398
  | `testmcpy run <path>` | Execute test suite |
@@ -311,14 +402,55 @@ testmcpy run tests/ --model claude-haiku-4-5
311
402
  | `testmcpy config-cmd` | View current configuration |
312
403
  | `testmcpy doctor` | Diagnose installation issues |
313
404
 
405
+ ### TUI Keyboard Shortcuts
406
+
407
+ **Global Navigation:**
408
+ - `h` - Home screen
409
+ - `e` - Explorer (MCP tools)
410
+ - `5` - Configuration
411
+ - `?` - Help modal
412
+ - `/` - Global search
413
+ - `q` - Quit (with confirmation)
414
+ - `F5` - Refresh
415
+
416
+ **Home Screen:**
417
+ - `1-5` - Quick actions (Tests, Explorer, Chat, Optimize, Config)
418
+ - `p` - Switch profile
419
+ - `Space` - Connect/disconnect
420
+
421
+ **Explorer:**
422
+ - `↑↓` or `j/k` - Navigate
423
+ - `Enter` - View details
424
+ - `t` - Create test
425
+ - `o` - Optimize docs
426
+
427
+ **Configuration:**
428
+ - `Tab` - Next field
429
+ - `s` - Save changes
430
+ - `q` - Quit without saving
431
+
314
432
  ## LLM Providers
315
433
 
434
+ Configure LLM providers in `.llm_providers.yaml`. See `.llm_providers.yaml.example` for examples.
435
+
316
436
  ### Anthropic (Recommended)
317
437
  Best tool-calling accuracy, native MCP support:
318
438
 
319
439
  ```bash
440
+ # Set API key in .env or ~/.testmcpy
320
441
  ANTHROPIC_API_KEY=sk-ant-your-key
321
- DEFAULT_MODEL=claude-haiku-4-5 # Fast & cost-effective
442
+ ```
443
+
444
+ ```yaml
445
+ # Configure in .llm_providers.yaml
446
+ prod:
447
+ name: "Production"
448
+ providers:
449
+ - name: "Claude Sonnet 4.5"
450
+ provider: "anthropic"
451
+ model: "claude-sonnet-4-5"
452
+ api_key_env: "ANTHROPIC_API_KEY"
453
+ default: true
322
454
  ```
323
455
 
324
456
  **Available models:** `claude-haiku-4-5`, `claude-sonnet-4-5`, `claude-opus-4-1`
@@ -334,16 +466,36 @@ brew install ollama # macOS
334
466
  # Start Ollama and pull a model
335
467
  ollama serve
336
468
  ollama pull llama3.1:8b
469
+ ```
337
470
 
338
- # Configure testmcpy
339
- DEFAULT_PROVIDER=ollama
340
- DEFAULT_MODEL=llama3.1:8b
471
+ ```yaml
472
+ # Configure in .llm_providers.yaml
473
+ local:
474
+ name: "Local Only"
475
+ providers:
476
+ - name: "Ollama Llama"
477
+ provider: "ollama"
478
+ model: "llama3.1:8b"
479
+ base_url: "http://localhost:11434"
480
+ default: true
341
481
  ```
342
482
 
343
483
  ### OpenAI
344
484
  ```bash
485
+ # Set API key in .env or ~/.testmcpy
345
486
  OPENAI_API_KEY=sk-your-key
346
- DEFAULT_MODEL=gpt-4-turbo
487
+ ```
488
+
489
+ ```yaml
490
+ # Configure in .llm_providers.yaml
491
+ openai:
492
+ name: "OpenAI"
493
+ providers:
494
+ - name: "GPT-4"
495
+ provider: "openai"
496
+ model: "gpt-4-turbo"
497
+ api_key_env: "OPENAI_API_KEY"
498
+ default: true
347
499
  ```
348
500
 
349
501
  ## Built-in Evaluators
@@ -469,6 +621,6 @@ By contributing, you agree that your contributions will be licensed under Apache
469
621
 
470
622
  ## Acknowledgments
471
623
 
472
- Built by the team at [Preset](https://preset.io) to enable better LLM testing and integration with Apache Superset and beyond.
624
+ Built to enable better LLM testing and integration with Model Context Protocol services.
473
625
 
474
626
  Special thanks to the MCP community and all our contributors!
@@ -1,12 +1,20 @@
1
- # testmcpy
1
+ <p align="center">
2
+ <img src="docs/logos/logo.svg" alt="testmcpy logo" width="600">
3
+ </p>
2
4
 
3
- **Test and benchmark LLMs with MCP tools in minutes.**
5
+ <p align="center">
6
+ <strong>Test and benchmark LLMs with MCP tools in minutes.</strong>
7
+ </p>
4
8
 
5
- A testing framework for validating how LLMs call tools via Model Context Protocol (MCP) - compare Claude, GPT-4, Llama, and other models' accuracy, cost, and performance.
9
+ <p align="center">
10
+ A testing framework for validating how LLMs call tools via Model Context Protocol (MCP) - compare Claude, GPT-4, Llama, and other models' accuracy, cost, and performance.
11
+ </p>
6
12
 
7
- [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
8
- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
9
- [![PyPI](https://img.shields.io/badge/pypi-testmcpy-blue)](https://pypi.org/project/testmcpy/)
13
+ <p align="center">
14
+ <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.9+-blue.svg" alt="Python 3.9+"></a>
15
+ <a href="LICENSE"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="License"></a>
16
+ <a href="https://pypi.org/project/testmcpy/"><img src="https://img.shields.io/badge/pypi-testmcpy-blue" alt="PyPI"></a>
17
+ </p>
10
18
 
11
19
  [Screenshot: CLI test runner with colorful progress bars and results]
12
20
 
@@ -46,6 +54,33 @@ That's it! No complex configuration needed to get started.
46
54
 
47
55
  ## Key Features
48
56
 
57
+ ### Interactive TUI Dashboard (NEW!)
58
+ Beautiful terminal interface for MCP testing - no browser required:
59
+
60
+ ```bash
61
+ testmcpy dash # Launch interactive dashboard
62
+ testmcpy dash --auto-refresh # Live connection monitoring
63
+ testmcpy dash --profile prod # Use specific MCP profile
64
+ ```
65
+
66
+ **TUI Features:**
67
+ - Real-time MCP connection status
68
+ - Interactive tool exploration
69
+ - Live test execution with progress
70
+ - Configuration editor
71
+ - Global search across tools, tests, and settings
72
+ - Help system with keyboard shortcuts (press `?`)
73
+ - Multiple themes (default, light, high contrast)
74
+
75
+ **Quick CLI Commands (no TUI):**
76
+ ```bash
77
+ testmcpy profiles # List MCP profiles (table)
78
+ testmcpy status # Connection status check
79
+ testmcpy explore-cli # Browse tools (non-interactive)
80
+ ```
81
+
82
+ [Screenshot: TUI dashboard showing profiles, quick actions, and keyboard shortcuts]
83
+
49
84
  ### Multi-Provider Support
50
85
  Test with **Claude**, **GPT-4**, **Llama**, and other models. Works with both paid APIs and free local models via Ollama.
51
86
 
@@ -66,6 +101,16 @@ Comprehensive validation out of the box:
66
101
  - **Optional web interface**: Visual tool explorer and interactive chat
67
102
  - **Real-time feedback**: Watch tests execute with live updates
68
103
 
104
+ When you start testmcpy, you're greeted with a beautiful terminal interface:
105
+
106
+ ```
107
+ ▀█▀ █▀▀ █▀ ▀█▀ █▀▄▀█ █▀▀ █▀█ █▄█
108
+ █ ██▄ ▄█ █ █ ▀ █ █▄▄ █▀▀ █
109
+
110
+ 🧪 Test • 📊 Benchmark • ✓ Validate
111
+ MCP Testing Framework
112
+ ```
113
+
69
114
  [Screenshot: Split view of CLI and Web UI running the same test]
70
115
 
71
116
  ### YAML Test Definitions
@@ -169,26 +214,62 @@ pip install 'testmcpy[all]'
169
214
 
170
215
  ### 1. Configuration
171
216
 
172
- Run the interactive setup wizard:
217
+ Run the interactive setup wizard to create configuration files:
173
218
 
174
219
  ```bash
175
220
  testmcpy setup
176
221
  ```
177
222
 
178
- Or manually create `~/.testmcpy`:
223
+ This will guide you through:
224
+ - **LLM Provider setup**: Choose between Claude (Anthropic), GPT-4 (OpenAI), or local Ollama models
225
+ - **MCP Service setup**: Configure your MCP server URL and authentication
226
+ - **API Key management**: Detects keys from environment and saves them to `.llm_providers.yaml`
179
227
 
180
- ```bash
181
- # MCP Service
182
- MCP_URL=http://localhost:5008/mcp/
183
- MCP_AUTH_TOKEN=your_bearer_token
184
-
185
- # LLM Provider (choose one)
186
- DEFAULT_PROVIDER=anthropic
187
- DEFAULT_MODEL=claude-haiku-4-5
188
- ANTHROPIC_API_KEY=sk-ant-...
228
+ The setup command creates two files in your current directory:
229
+
230
+ **`.llm_providers.yaml`** - LLM configuration with API keys:
231
+
232
+ ```yaml
233
+ default: prod
234
+
235
+ profiles:
236
+ prod:
237
+ name: "Production"
238
+ description: "High-quality models for production use"
239
+ providers:
240
+ - name: "Claude claude-sonnet-4-5"
241
+ provider: "anthropic"
242
+ model: "claude-sonnet-4-5"
243
+ api_key: "your-anthropic-api-key-here" # API key stored directly
244
+ timeout: 60
245
+ default: true
246
+ ```
247
+
248
+ **`.mcp_services.yaml`** - MCP server profiles:
249
+
250
+ ```yaml
251
+ default: prod
252
+
253
+ profiles:
254
+ prod:
255
+ name: "Production"
256
+ description: "Production MCP service"
257
+ mcps:
258
+ - name: "Preset Superset"
259
+ mcp_url: "https://your-workspace.preset.io/mcp"
260
+ auth:
261
+ auth_type: "jwt" # or "bearer" or "none"
262
+ api_url: "https://api.app.preset.io/v1/auth/"
263
+ api_token: "your-api-token"
264
+ api_secret: "your-api-secret"
265
+ timeout: 30
266
+ rate_limit_rpm: 60
267
+ default: true
189
268
  ```
190
269
 
191
- **Configuration priority:** CLI options > `.env` > `~/.testmcpy` > Environment variables > Defaults
270
+ **Configuration priority:** CLI options > LLM Profile (.llm_providers.yaml) > MCP Profile (.mcp_services.yaml) > `.env` > Environment variables
271
+
272
+ **Note:** The setup command is **idempotent** - it's safe to run multiple times. Use `--force` to overwrite existing files.
192
273
 
193
274
  ### 2. Test Your MCP Service
194
275
 
@@ -246,7 +327,12 @@ testmcpy run tests/ --model claude-haiku-4-5
246
327
 
247
328
  | Command | Description |
248
329
  |---------|-------------|
330
+ | `testmcpy dash` | **Launch interactive TUI dashboard** |
249
331
  | `testmcpy setup` | Interactive configuration wizard |
332
+ | `testmcpy profiles` | List MCP profiles (table) |
333
+ | `testmcpy status` | Show MCP connection status |
334
+ | `testmcpy explore-cli` | Browse tools (non-interactive) |
335
+ | `testmcpy explorer` | Launch TUI tool explorer |
250
336
  | `testmcpy tools` | List available MCP tools |
251
337
  | `testmcpy research` | Test LLM tool-calling capabilities |
252
338
  | `testmcpy run <path>` | Execute test suite |
@@ -256,14 +342,55 @@ testmcpy run tests/ --model claude-haiku-4-5
256
342
  | `testmcpy config-cmd` | View current configuration |
257
343
  | `testmcpy doctor` | Diagnose installation issues |
258
344
 
345
+ ### TUI Keyboard Shortcuts
346
+
347
+ **Global Navigation:**
348
+ - `h` - Home screen
349
+ - `e` - Explorer (MCP tools)
350
+ - `5` - Configuration
351
+ - `?` - Help modal
352
+ - `/` - Global search
353
+ - `q` - Quit (with confirmation)
354
+ - `F5` - Refresh
355
+
356
+ **Home Screen:**
357
+ - `1-5` - Quick actions (Tests, Explorer, Chat, Optimize, Config)
358
+ - `p` - Switch profile
359
+ - `Space` - Connect/disconnect
360
+
361
+ **Explorer:**
362
+ - `↑↓` or `j/k` - Navigate
363
+ - `Enter` - View details
364
+ - `t` - Create test
365
+ - `o` - Optimize docs
366
+
367
+ **Configuration:**
368
+ - `Tab` - Next field
369
+ - `s` - Save changes
370
+ - `q` - Quit without saving
371
+
259
372
  ## LLM Providers
260
373
 
374
+ Configure LLM providers in `.llm_providers.yaml`. See `.llm_providers.yaml.example` for examples.
375
+
261
376
  ### Anthropic (Recommended)
262
377
  Best tool-calling accuracy, native MCP support:
263
378
 
264
379
  ```bash
380
+ # Set API key in .env or ~/.testmcpy
265
381
  ANTHROPIC_API_KEY=sk-ant-your-key
266
- DEFAULT_MODEL=claude-haiku-4-5 # Fast & cost-effective
382
+ ```
383
+
384
+ ```yaml
385
+ # Configure in .llm_providers.yaml
386
+ prod:
387
+ name: "Production"
388
+ providers:
389
+ - name: "Claude Sonnet 4.5"
390
+ provider: "anthropic"
391
+ model: "claude-sonnet-4-5"
392
+ api_key_env: "ANTHROPIC_API_KEY"
393
+ default: true
267
394
  ```
268
395
 
269
396
  **Available models:** `claude-haiku-4-5`, `claude-sonnet-4-5`, `claude-opus-4-1`
@@ -279,16 +406,36 @@ brew install ollama # macOS
279
406
  # Start Ollama and pull a model
280
407
  ollama serve
281
408
  ollama pull llama3.1:8b
409
+ ```
282
410
 
283
- # Configure testmcpy
284
- DEFAULT_PROVIDER=ollama
285
- DEFAULT_MODEL=llama3.1:8b
411
+ ```yaml
412
+ # Configure in .llm_providers.yaml
413
+ local:
414
+ name: "Local Only"
415
+ providers:
416
+ - name: "Ollama Llama"
417
+ provider: "ollama"
418
+ model: "llama3.1:8b"
419
+ base_url: "http://localhost:11434"
420
+ default: true
286
421
  ```
287
422
 
288
423
  ### OpenAI
289
424
  ```bash
425
+ # Set API key in .env or ~/.testmcpy
290
426
  OPENAI_API_KEY=sk-your-key
291
- DEFAULT_MODEL=gpt-4-turbo
427
+ ```
428
+
429
+ ```yaml
430
+ # Configure in .llm_providers.yaml
431
+ openai:
432
+ name: "OpenAI"
433
+ providers:
434
+ - name: "GPT-4"
435
+ provider: "openai"
436
+ model: "gpt-4-turbo"
437
+ api_key_env: "OPENAI_API_KEY"
438
+ default: true
292
439
  ```
293
440
 
294
441
  ## Built-in Evaluators
@@ -414,6 +561,6 @@ By contributing, you agree that your contributions will be licensed under Apache
414
561
 
415
562
  ## Acknowledgments
416
563
 
417
- Built by the team at [Preset](https://preset.io) to enable better LLM testing and integration with Apache Superset and beyond.
564
+ Built to enable better LLM testing and integration with Model Context Protocol services.
418
565
 
419
566
  Special thanks to the MCP community and all our contributors!
@@ -88,7 +88,7 @@ testmcpy = [
88
88
 
89
89
  [project]
90
90
  name = "testmcpy"
91
- version = "0.2.3"
91
+ version = "0.2.6"
92
92
  description = "A comprehensive testing framework for validating LLM tool calling capabilities with MCP services"
93
93
  authors = [{name = "Amin Ghadersohi"}]
94
94
  license = "Apache-2.0"
@@ -104,7 +104,7 @@ classifiers = [
104
104
  ]
105
105
  dependencies = [
106
106
  "typer>=0.9.0,<1.0.0",
107
- "rich>=13.0.0,<14.0.0",
107
+ "rich>=13.0.0,<15.0.0",
108
108
  "pyyaml>=6.0,<7.0",
109
109
  "requests>=2.28.0,<3.0.0",
110
110
  "aiohttp>=3.8.0,<4.0.0",
@@ -115,6 +115,7 @@ dependencies = [
115
115
  "python-dotenv>=1.0.0,<2.0.0",
116
116
  "click>=8.0.0,<9.0.0",
117
117
  "shellingham>=1.3.0,<2.0.0",
118
+ "textual>=0.47.0,<1.0.0",
118
119
  ]
119
120
 
120
121
  [project.optional-dependencies]
@@ -129,20 +130,25 @@ dev = [
129
130
  "twine>=5.0.0",
130
131
  "types-pyyaml>=6.0.0",
131
132
  "types-requests>=2.28.0",
133
+ "textual-dev>=1.0.0",
132
134
  ]
133
135
  server = [
134
136
  "fastapi>=0.104.0,<1.0.0",
135
- "uvicorn>=0.24.0,<1.0.0",
136
- "websockets>=12.0,<13.0",
137
+ "uvicorn[standard]>=0.24.0,<1.0.0",
138
+ "websockets>=14.0,<15.0",
137
139
  ]
138
140
  sdk = [
139
141
  "claude-agent-sdk>=0.1.0",
140
142
  ]
143
+ tui = [
144
+ "textual>=0.85.0",
145
+ ]
141
146
  all = [
142
147
  "fastapi>=0.104.0,<1.0.0",
143
- "uvicorn>=0.24.0,<1.0.0",
144
- "websockets>=12.0,<13.0",
148
+ "uvicorn[standard]>=0.24.0,<1.0.0",
149
+ "websockets>=14.0,<15.0",
145
150
  "claude-agent-sdk>=0.1.0",
151
+ "textual>=0.85.0",
146
152
  ]
147
153
 
148
154
  [project.scripts]
@@ -6,4 +6,4 @@ capabilities with MCP (Model Context Protocol) services.
6
6
  """
7
7
 
8
8
  __version__ = "0.2.0"
9
- __author__ = "Amin Ghadersohi"
9
+ __author__ = "testmcpy Contributors"