testmcpy 0.7.3__tar.gz → 0.7.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testmcpy-0.7.3/testmcpy.egg-info → testmcpy-0.7.4}/PKG-INFO +31 -30
- {testmcpy-0.7.3 → testmcpy-0.7.4}/README.md +30 -29
- {testmcpy-0.7.3 → testmcpy-0.7.4}/pyproject.toml +1 -1
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/__init__.py +1 -1
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/run.py +60 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/mcp_client.py +31 -1
- {testmcpy-0.7.3 → testmcpy-0.7.4/testmcpy.egg-info}/PKG-INFO +31 -30
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy.egg-info/SOURCES.txt +0 -3
- testmcpy-0.7.3/testmcpy/ui/dist/assets/index-30Ed2JCz.css +0 -1
- testmcpy-0.7.3/testmcpy/ui/dist/assets/index-6JiH0p1L.js +0 -291
- testmcpy-0.7.3/testmcpy/ui/dist/index.html +0 -22
- {testmcpy-0.7.3 → testmcpy-0.7.4}/LICENSE +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/MANIFEST.in +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/NOTICE +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/setup.cfg +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/agent/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/agent/hooks.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/agent/models.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/agent/orchestrator.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/agent/prompts.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/agent/tools.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/auth_debugger.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/auth_flow_recorder.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/app.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/agent.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/baseline.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/export_db.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/mcp.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/metamorphic.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/multi_env.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/mutate.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/push.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/server.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/tools.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/tui.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/cli/commands/wizard.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/config.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/core/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/core/chat_session.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/core/docs_optimizer.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/core/mcp_manager.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/core/tool_comparison.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/core/tool_discovery.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/db.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/error_handlers.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/evals/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/evals/auth_evaluators.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/evals/base_evaluators.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/evals/evaluator_packs.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/base.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/curl.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/graphql.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/javascript_client.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/json_yaml.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/protobuf.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/python.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/python_client.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/thrift.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/typescript.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/formatters/typescript_client.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/llm_profiles.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/mcp_profiles.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/migrate_json.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/models.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/research/claude_sdk_detailed_exploration.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/research/claude_sdk_poc.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/research/claude_sdk_working_poc.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/research/test_ollama_tools.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/api.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/api.py.bak +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/auth_middleware.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/helpers/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/helpers/mcp_config.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/models.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/agent.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/auth.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/compare.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/compatibility.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/generation_logs.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/health.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/llm.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/mcp_profiles.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/metrics.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/results.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/search.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/security.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/smoke_reports.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/test_profiles.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/tests.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/routers/tools.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/state.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/server/websocket.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/smoke_test.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/__init__.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/baseline.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/ci_gate.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/comparison_runner.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/coverage_analyzer.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/html_report.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/llm_integration.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/metamorphic.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/model_registry.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/models.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/multi_env.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/oauth_flows.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/prompt_mutation.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/report_generator.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/runner_tools.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/schema_diff.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/test_runner.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/src/token_manager.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/storage.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/test_profiles.py +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/README.md +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/index.html +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/package-lock.json +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/package.json +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/postcss.config.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/App.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/CommandPalette.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/CompareToolsTab.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/EditorStatusBar.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/EditorTabStrip.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/ErrorAlert.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/ErrorBoundary.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/LLMProfileSelector.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/LoadingSpinner.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/MCPProfileSelector.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/NotificationProvider.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/OptimizeDocsModal.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/OutputDiff.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/ParameterCard.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/SchemaCodeViewer.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/SkeletonLoader.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/StreamingLogViewer.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/TestGenerationModal.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/TestProfileSelector.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/TestResultPanel.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/TestStatusIndicator.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/ToolCallTimeline.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/ToolComparison.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/ToolDebugModal.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/TraceView.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/TypeBadge.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/components/Wizard.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/contexts/TestRunContext.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/contexts/ThemeContext.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/hooks/useEditorTheme.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/hooks/useKeyboardShortcuts.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/hooks/useSafeFetch.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/index.css +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/main.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/AuthDebugger.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/ChatInterface.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/CompatibilityMatrix.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/Configuration.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/GenerationHistory.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/LLMProfiles.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/MCPExplorer.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/MCPHealth.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/MCPProfiles.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/MetricsDashboard.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/ProfilesManager.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/Reports.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/RunComparison.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/SecurityDashboard.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/pages/TestManager.jsx +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/utils/__tests__/formatConverters.test.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/src/utils/formatConverters.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/tailwind.config.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy/ui/vite.config.js +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy.egg-info/dependency_links.txt +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy.egg-info/entry_points.txt +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy.egg-info/requires.txt +0 -0
- {testmcpy-0.7.3 → testmcpy-0.7.4}/testmcpy.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: testmcpy
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.4
|
|
4
4
|
Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
|
|
5
5
|
Author: Amin Ghadersohi
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -86,7 +86,7 @@ Dynamic: license-file
|
|
|
86
86
|
<a href="https://pypi.org/project/testmcpy/"><img src="https://img.shields.io/badge/pypi-testmcpy-blue" alt="PyPI"></a>
|
|
87
87
|
</p>
|
|
88
88
|
|
|
89
|
-

|
|
90
90
|
|
|
91
91
|
---
|
|
92
92
|
|
|
@@ -131,7 +131,7 @@ Test with **Claude**, **GPT-4**, **Llama**, and other models. Works with both pa
|
|
|
131
131
|
| Ollama | Llama, Mistral, etc. (local) | Free, local execution, no API costs |
|
|
132
132
|
| Claude SDK | claude-cli, claude-code | Subprocess-based, full MCP support |
|
|
133
133
|
|
|
134
|
-

|
|
135
135
|
|
|
136
136
|
### Built-in Evaluators
|
|
137
137
|
|
|
@@ -154,7 +154,7 @@ Comprehensive validation out of the box. Each evaluator returns a score from 0.0
|
|
|
154
154
|
|
|
155
155
|
**Extensible:** Extend `BaseEvaluator` and implement `evaluate(context) -> EvalResult` to create custom evaluators for your domain.
|
|
156
156
|
|
|
157
|
-

|
|
158
158
|
|
|
159
159
|
### YAML Test Definitions
|
|
160
160
|
|
|
@@ -200,32 +200,13 @@ tests:
|
|
|
200
200
|
duration: 60
|
|
201
201
|
```
|
|
202
202
|
|
|
203
|
-
### Interactive TUI Dashboard
|
|
204
|
-
|
|
205
|
-
Beautiful terminal interface for MCP testing — no browser required:
|
|
206
|
-
|
|
207
|
-
```bash
|
|
208
|
-
testmcpy dash # Launch interactive dashboard
|
|
209
|
-
testmcpy dash --auto-refresh # Live connection monitoring
|
|
210
|
-
testmcpy dash --profile prod # Use specific MCP profile
|
|
211
|
-
```
|
|
212
|
-
|
|
213
|
-
**TUI Features:**
|
|
214
|
-
- Real-time MCP connection status
|
|
215
|
-
- Interactive tool exploration
|
|
216
|
-
- Live test execution with progress
|
|
217
|
-
- Configuration editor
|
|
218
|
-
- Global search across tools, tests, and settings
|
|
219
|
-
- Help system with keyboard shortcuts (press `?`)
|
|
220
|
-
- Multiple themes (default, light, high contrast)
|
|
221
|
-
|
|
222
203
|
### CLI & Web UI
|
|
223
204
|
|
|
224
205
|
- **Rich terminal UI**: Progress bars, colored output, formatted tables
|
|
225
206
|
- **Optional web interface**: Visual tool explorer, interactive chat, analytics dashboards
|
|
226
207
|
- **Real-time feedback**: Watch tests execute with live updates via WebSocket
|
|
227
208
|
|
|
228
|
-

|
|
229
210
|
|
|
230
211
|
## Architecture
|
|
231
212
|
|
|
@@ -394,13 +375,20 @@ testmcpy run tests/ --model claude-haiku-4-5
|
|
|
394
375
|
| `testmcpy chat` | Interactive chat with MCP tools |
|
|
395
376
|
| `testmcpy compare` | Multi-model comparison |
|
|
396
377
|
| **Advanced** | |
|
|
397
|
-
| `testmcpy baseline` | Save
|
|
378
|
+
| `testmcpy baseline-save` | Save current test results as a named baseline |
|
|
379
|
+
| `testmcpy baseline-compare` | Compare a run against a saved baseline |
|
|
380
|
+
| `testmcpy baseline-list` | List saved baselines |
|
|
398
381
|
| `testmcpy mutate` | Prompt mutation testing |
|
|
399
382
|
| `testmcpy metamorphic` | Metamorphic testing |
|
|
383
|
+
| `testmcpy generate` | AI-assisted test generation |
|
|
384
|
+
| `testmcpy smoke-test` | Quick smoke test against an MCP service |
|
|
385
|
+
| `testmcpy coverage` | Tool coverage report for a test suite |
|
|
386
|
+
| `testmcpy multi-env` | Run the same suite against multiple MCP profiles |
|
|
387
|
+
| `testmcpy export-db` | Export the SQLite results database |
|
|
400
388
|
| **UI** | |
|
|
401
|
-
| `testmcpy serve` | Start web UI server (port 8000) |
|
|
402
|
-
| `testmcpy dash` | Launch terminal UI dashboard |
|
|
389
|
+
| `testmcpy serve` | Start web UI server (default port 8000) |
|
|
403
390
|
| `testmcpy config-cmd` | View current configuration |
|
|
391
|
+
| `testmcpy config-mcp` | Print MCP client snippets for Claude Desktop / Code |
|
|
404
392
|
|
|
405
393
|
**Common options:** `--profile`, `--llm-profile`, `--model`, `--provider`, `--timeout`, `--verbose`, `--output`
|
|
406
394
|
|
|
@@ -433,9 +421,9 @@ Environment variables are also supported: `MCP_AUTH_TOKEN`, `MCP_JWT_URL`, `MCP_
|
|
|
433
421
|
|
|
434
422
|
## Web Interface
|
|
435
423
|
|
|
436
|
-
Optional React-based UI with
|
|
424
|
+
Optional React-based UI with 14 pages for visual testing and analytics:
|
|
437
425
|
|
|
438
|
-

|
|
439
427
|
|
|
440
428
|
```bash
|
|
441
429
|
# Install with UI support
|
|
@@ -462,7 +450,20 @@ testmcpy serve
|
|
|
462
450
|
| `/mcp-profiles` | MCP Profiles | MCP server configuration |
|
|
463
451
|
| `/llm-profiles` | LLM Profiles | LLM provider configuration |
|
|
464
452
|
|
|
465
|
-
Access at `http://localhost:8000
|
|
453
|
+
Access at `http://localhost:8000`.
|
|
454
|
+
|
|
455
|
+
#### More screenshots
|
|
456
|
+
|
|
457
|
+
<table>
|
|
458
|
+
<tr>
|
|
459
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/generation-history.png" alt="Generation History page"><br><sub>Generation History — AI-assisted test generation runs</sub></td>
|
|
460
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/auth-debugger.png" alt="Auth Debugger page"><br><sub>Auth Debugger — step through OAuth / JWT / Bearer flows</sub></td>
|
|
461
|
+
</tr>
|
|
462
|
+
<tr>
|
|
463
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/mcp-profiles.png" alt="MCP Profiles page"><br><sub>MCP Profiles — manage MCP service connections</sub></td>
|
|
464
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/config.png" alt="Configuration page"><br><sub>Configuration — current settings and client snippets</sub></td>
|
|
465
|
+
</tr>
|
|
466
|
+
</table>
|
|
466
467
|
|
|
467
468
|
## LLM Providers
|
|
468
469
|
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
<a href="https://pypi.org/project/testmcpy/"><img src="https://img.shields.io/badge/pypi-testmcpy-blue" alt="PyPI"></a>
|
|
17
17
|
</p>
|
|
18
18
|
|
|
19
|
-

|
|
20
20
|
|
|
21
21
|
---
|
|
22
22
|
|
|
@@ -61,7 +61,7 @@ Test with **Claude**, **GPT-4**, **Llama**, and other models. Works with both pa
|
|
|
61
61
|
| Ollama | Llama, Mistral, etc. (local) | Free, local execution, no API costs |
|
|
62
62
|
| Claude SDK | claude-cli, claude-code | Subprocess-based, full MCP support |
|
|
63
63
|
|
|
64
|
-

|
|
65
65
|
|
|
66
66
|
### Built-in Evaluators
|
|
67
67
|
|
|
@@ -84,7 +84,7 @@ Comprehensive validation out of the box. Each evaluator returns a score from 0.0
|
|
|
84
84
|
|
|
85
85
|
**Extensible:** Extend `BaseEvaluator` and implement `evaluate(context) -> EvalResult` to create custom evaluators for your domain.
|
|
86
86
|
|
|
87
|
-

|
|
88
88
|
|
|
89
89
|
### YAML Test Definitions
|
|
90
90
|
|
|
@@ -130,32 +130,13 @@ tests:
|
|
|
130
130
|
duration: 60
|
|
131
131
|
```
|
|
132
132
|
|
|
133
|
-
### Interactive TUI Dashboard
|
|
134
|
-
|
|
135
|
-
Beautiful terminal interface for MCP testing — no browser required:
|
|
136
|
-
|
|
137
|
-
```bash
|
|
138
|
-
testmcpy dash # Launch interactive dashboard
|
|
139
|
-
testmcpy dash --auto-refresh # Live connection monitoring
|
|
140
|
-
testmcpy dash --profile prod # Use specific MCP profile
|
|
141
|
-
```
|
|
142
|
-
|
|
143
|
-
**TUI Features:**
|
|
144
|
-
- Real-time MCP connection status
|
|
145
|
-
- Interactive tool exploration
|
|
146
|
-
- Live test execution with progress
|
|
147
|
-
- Configuration editor
|
|
148
|
-
- Global search across tools, tests, and settings
|
|
149
|
-
- Help system with keyboard shortcuts (press `?`)
|
|
150
|
-
- Multiple themes (default, light, high contrast)
|
|
151
|
-
|
|
152
133
|
### CLI & Web UI
|
|
153
134
|
|
|
154
135
|
- **Rich terminal UI**: Progress bars, colored output, formatted tables
|
|
155
136
|
- **Optional web interface**: Visual tool explorer, interactive chat, analytics dashboards
|
|
156
137
|
- **Real-time feedback**: Watch tests execute with live updates via WebSocket
|
|
157
138
|
|
|
158
|
-

|
|
159
140
|
|
|
160
141
|
## Architecture
|
|
161
142
|
|
|
@@ -324,13 +305,20 @@ testmcpy run tests/ --model claude-haiku-4-5
|
|
|
324
305
|
| `testmcpy chat` | Interactive chat with MCP tools |
|
|
325
306
|
| `testmcpy compare` | Multi-model comparison |
|
|
326
307
|
| **Advanced** | |
|
|
327
|
-
| `testmcpy baseline` | Save
|
|
308
|
+
| `testmcpy baseline-save` | Save current test results as a named baseline |
|
|
309
|
+
| `testmcpy baseline-compare` | Compare a run against a saved baseline |
|
|
310
|
+
| `testmcpy baseline-list` | List saved baselines |
|
|
328
311
|
| `testmcpy mutate` | Prompt mutation testing |
|
|
329
312
|
| `testmcpy metamorphic` | Metamorphic testing |
|
|
313
|
+
| `testmcpy generate` | AI-assisted test generation |
|
|
314
|
+
| `testmcpy smoke-test` | Quick smoke test against an MCP service |
|
|
315
|
+
| `testmcpy coverage` | Tool coverage report for a test suite |
|
|
316
|
+
| `testmcpy multi-env` | Run the same suite against multiple MCP profiles |
|
|
317
|
+
| `testmcpy export-db` | Export the SQLite results database |
|
|
330
318
|
| **UI** | |
|
|
331
|
-
| `testmcpy serve` | Start web UI server (port 8000) |
|
|
332
|
-
| `testmcpy dash` | Launch terminal UI dashboard |
|
|
319
|
+
| `testmcpy serve` | Start web UI server (default port 8000) |
|
|
333
320
|
| `testmcpy config-cmd` | View current configuration |
|
|
321
|
+
| `testmcpy config-mcp` | Print MCP client snippets for Claude Desktop / Code |
|
|
334
322
|
|
|
335
323
|
**Common options:** `--profile`, `--llm-profile`, `--model`, `--provider`, `--timeout`, `--verbose`, `--output`
|
|
336
324
|
|
|
@@ -363,9 +351,9 @@ Environment variables are also supported: `MCP_AUTH_TOKEN`, `MCP_JWT_URL`, `MCP_
|
|
|
363
351
|
|
|
364
352
|
## Web Interface
|
|
365
353
|
|
|
366
|
-
Optional React-based UI with
|
|
354
|
+
Optional React-based UI with 14 pages for visual testing and analytics:
|
|
367
355
|
|
|
368
|
-

|
|
369
357
|
|
|
370
358
|
```bash
|
|
371
359
|
# Install with UI support
|
|
@@ -392,7 +380,20 @@ testmcpy serve
|
|
|
392
380
|
| `/mcp-profiles` | MCP Profiles | MCP server configuration |
|
|
393
381
|
| `/llm-profiles` | LLM Profiles | LLM provider configuration |
|
|
394
382
|
|
|
395
|
-
Access at `http://localhost:8000
|
|
383
|
+
Access at `http://localhost:8000`.
|
|
384
|
+
|
|
385
|
+
#### More screenshots
|
|
386
|
+
|
|
387
|
+
<table>
|
|
388
|
+
<tr>
|
|
389
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/generation-history.png" alt="Generation History page"><br><sub>Generation History — AI-assisted test generation runs</sub></td>
|
|
390
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/auth-debugger.png" alt="Auth Debugger page"><br><sub>Auth Debugger — step through OAuth / JWT / Bearer flows</sub></td>
|
|
391
|
+
</tr>
|
|
392
|
+
<tr>
|
|
393
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/mcp-profiles.png" alt="MCP Profiles page"><br><sub>MCP Profiles — manage MCP service connections</sub></td>
|
|
394
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/config.png" alt="Configuration page"><br><sub>Configuration — current settings and client snippets</sub></td>
|
|
395
|
+
</tr>
|
|
396
|
+
</table>
|
|
396
397
|
|
|
397
398
|
## LLM Providers
|
|
398
399
|
|
|
@@ -93,7 +93,7 @@ testmcpy = [
|
|
|
93
93
|
|
|
94
94
|
[project]
|
|
95
95
|
name = "testmcpy"
|
|
96
|
-
version = "0.7.
|
|
96
|
+
version = "0.7.4"
|
|
97
97
|
description = "A comprehensive testing framework for validating LLM tool calling capabilities with MCP services"
|
|
98
98
|
authors = [{name = "Amin Ghadersohi"}]
|
|
99
99
|
license = "Apache-2.0"
|
|
@@ -540,6 +540,47 @@ def run(
|
|
|
540
540
|
|
|
541
541
|
# Run tests with progress output
|
|
542
542
|
results = []
|
|
543
|
+
|
|
544
|
+
# Progressive checkpoint: dump partial results to a JSON file under
|
|
545
|
+
# tests/.results/.checkpoints/ after every test completes, so an
|
|
546
|
+
# outer harness (or this process after a crash) can recover what
|
|
547
|
+
# finished even if the run is killed mid-stream. The matching
|
|
548
|
+
# <session_id>.done sentinel is written immediately after the run
|
|
549
|
+
# summary prints, before optional post-processing (DB save, report
|
|
550
|
+
# generation) — so harnesses see "done" as soon as the test loop
|
|
551
|
+
# itself finishes, even if a later non-test step hangs or fails.
|
|
552
|
+
# (SC-107284 / c33 issues 3 & 6)
|
|
553
|
+
checkpoint_dir = Path("tests/.results/.checkpoints")
|
|
554
|
+
checkpoint_path: Optional[Path] = None
|
|
555
|
+
done_path: Optional[Path] = None
|
|
556
|
+
try:
|
|
557
|
+
checkpoint_dir.mkdir(parents=True, exist_ok=True)
|
|
558
|
+
checkpoint_path = checkpoint_dir / f"{session_id}.json"
|
|
559
|
+
done_path = checkpoint_dir / f"{session_id}.done"
|
|
560
|
+
except OSError as e:
|
|
561
|
+
console.print(f"[dim]Note: Could not create checkpoint dir: {e}[/dim]")
|
|
562
|
+
|
|
563
|
+
def _write_checkpoint(completed: list, total: int) -> None:
|
|
564
|
+
"""Atomically write current results to the checkpoint file."""
|
|
565
|
+
if checkpoint_path is None:
|
|
566
|
+
return
|
|
567
|
+
payload = {
|
|
568
|
+
"session_id": session_id,
|
|
569
|
+
"test_file": str(test_path),
|
|
570
|
+
"provider": effective_provider,
|
|
571
|
+
"model": effective_model,
|
|
572
|
+
"mcp_profile": profile or "default",
|
|
573
|
+
"completed": len(completed),
|
|
574
|
+
"total": total,
|
|
575
|
+
"results": [r.to_dict() for r in completed],
|
|
576
|
+
}
|
|
577
|
+
try:
|
|
578
|
+
tmp = checkpoint_path.with_suffix(".json.tmp")
|
|
579
|
+
tmp.write_text(json.dumps(payload, indent=2, default=str))
|
|
580
|
+
tmp.replace(checkpoint_path)
|
|
581
|
+
except OSError as e:
|
|
582
|
+
console.print(f"[dim]Note: Could not write checkpoint: {e}[/dim]")
|
|
583
|
+
|
|
543
584
|
# Only initialize LLM provider if there are non-auth-only tests
|
|
544
585
|
has_non_auth_only = any(not tc.is_auth_only for tc in test_cases)
|
|
545
586
|
if has_non_auth_only:
|
|
@@ -589,6 +630,7 @@ def run(
|
|
|
589
630
|
_status.stop()
|
|
590
631
|
|
|
591
632
|
results.append(result)
|
|
633
|
+
_write_checkpoint(results, len(test_cases))
|
|
592
634
|
|
|
593
635
|
# Show immediate result
|
|
594
636
|
if result.passed:
|
|
@@ -664,6 +706,24 @@ def run(
|
|
|
664
706
|
|
|
665
707
|
console.print(f"\n[bold]Summary:[/bold] {' | '.join(summary_parts)}")
|
|
666
708
|
|
|
709
|
+
# Drop a .done sentinel immediately after the summary, before any
|
|
710
|
+
# optional post-processing (DB save, report gen) so outer harnesses
|
|
711
|
+
# can treat the run as finished even if a later step fails or hangs.
|
|
712
|
+
if done_path is not None:
|
|
713
|
+
try:
|
|
714
|
+
done_path.write_text(
|
|
715
|
+
json.dumps(
|
|
716
|
+
{
|
|
717
|
+
"session_id": session_id,
|
|
718
|
+
"total": len(results),
|
|
719
|
+
"passed": total_passed,
|
|
720
|
+
"failed": len(results) - total_passed,
|
|
721
|
+
}
|
|
722
|
+
)
|
|
723
|
+
)
|
|
724
|
+
except OSError as e:
|
|
725
|
+
console.print(f"[dim]Note: Could not write .done marker: {e}[/dim]")
|
|
726
|
+
|
|
667
727
|
# Always auto-save results to tests/.results/ so the UI can see them
|
|
668
728
|
try:
|
|
669
729
|
from testmcpy.server.routers.results import save_test_run_to_file
|
|
@@ -874,6 +874,16 @@ class MCPClient:
|
|
|
874
874
|
except Exception as e:
|
|
875
875
|
raise MCPError(f"Failed to list tools: {e}")
|
|
876
876
|
|
|
877
|
+
# Hard cap on total characters collected from search_tools responses
|
|
878
|
+
# during gateway discovery. Some MCP servers return very large tool
|
|
879
|
+
# catalogs (full schemas inline) and accumulating every broad-query
|
|
880
|
+
# response can blow out memory or downstream context budgets. 200k
|
|
881
|
+
# chars is enough to discover a few hundred tools while staying well
|
|
882
|
+
# under any sane context limit. The cap is measured in chars (Python
|
|
883
|
+
# str length) — not bytes — to match how the JSON response is held in
|
|
884
|
+
# memory before parsing. (SC-107284 / c33 issue 4)
|
|
885
|
+
_GATEWAY_DISCOVERY_MAX_CHARS = 200_000
|
|
886
|
+
|
|
877
887
|
async def _expand_gateway_tools(self, timeout: float = 30.0) -> list["MCPTool"]:
|
|
878
888
|
"""Expand tools via the search_tools gateway (FastMCP 3.x pattern).
|
|
879
889
|
|
|
@@ -896,9 +906,12 @@ class MCPClient:
|
|
|
896
906
|
"list_charts generate_dashboard get_chart_data",
|
|
897
907
|
"list_databases get_database_info generate_chart",
|
|
898
908
|
]
|
|
899
|
-
all_content = []
|
|
909
|
+
all_content: list[str] = []
|
|
910
|
+
total_chars = 0
|
|
900
911
|
|
|
901
912
|
for q in broad_queries:
|
|
913
|
+
if total_chars >= self._GATEWAY_DISCOVERY_MAX_CHARS:
|
|
914
|
+
break
|
|
902
915
|
try:
|
|
903
916
|
result = await asyncio.wait_for(
|
|
904
917
|
self.client.call_tool("search_tools", {"query": q}),
|
|
@@ -913,7 +926,17 @@ class MCPClient:
|
|
|
913
926
|
elif isinstance(result.content, str):
|
|
914
927
|
content = result.content
|
|
915
928
|
if content:
|
|
929
|
+
# Don't truncate mid-string — search_tools returns JSON
|
|
930
|
+
# and a mid-character slice would corrupt it, causing the
|
|
931
|
+
# downstream json.loads() to fail and discarding the
|
|
932
|
+
# whole response (including any tools we could have
|
|
933
|
+
# discovered from it). Skip oversize responses entirely
|
|
934
|
+
# instead; the per-query early-exit above bounds the
|
|
935
|
+
# worst case to roughly one response past the cap.
|
|
936
|
+
if total_chars + len(content) > self._GATEWAY_DISCOVERY_MAX_CHARS:
|
|
937
|
+
continue
|
|
916
938
|
all_content.append(content)
|
|
939
|
+
total_chars += len(content)
|
|
917
940
|
except (asyncio.TimeoutError, TypeError, ValueError):
|
|
918
941
|
continue
|
|
919
942
|
|
|
@@ -955,6 +978,8 @@ class MCPClient:
|
|
|
955
978
|
second_pass_queries.append(batch)
|
|
956
979
|
|
|
957
980
|
for q in second_pass_queries[:5]: # Limit to 5 extra queries
|
|
981
|
+
if total_chars >= self._GATEWAY_DISCOVERY_MAX_CHARS:
|
|
982
|
+
break
|
|
958
983
|
try:
|
|
959
984
|
result = await asyncio.wait_for(
|
|
960
985
|
self.client.call_tool("search_tools", {"query": q}),
|
|
@@ -966,6 +991,11 @@ class MCPClient:
|
|
|
966
991
|
if hasattr(item, "text"):
|
|
967
992
|
content += item.text
|
|
968
993
|
if content:
|
|
994
|
+
# Same reasoning as the first pass: skip oversize
|
|
995
|
+
# responses rather than truncating mid-JSON.
|
|
996
|
+
if total_chars + len(content) > self._GATEWAY_DISCOVERY_MAX_CHARS:
|
|
997
|
+
continue
|
|
998
|
+
total_chars += len(content)
|
|
969
999
|
import json as _json2
|
|
970
1000
|
|
|
971
1001
|
try:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: testmcpy
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.4
|
|
4
4
|
Summary: A comprehensive testing framework for validating LLM tool calling capabilities with MCP services
|
|
5
5
|
Author: Amin Ghadersohi
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -86,7 +86,7 @@ Dynamic: license-file
|
|
|
86
86
|
<a href="https://pypi.org/project/testmcpy/"><img src="https://img.shields.io/badge/pypi-testmcpy-blue" alt="PyPI"></a>
|
|
87
87
|
</p>
|
|
88
88
|
|
|
89
|
-

|
|
90
90
|
|
|
91
91
|
---
|
|
92
92
|
|
|
@@ -131,7 +131,7 @@ Test with **Claude**, **GPT-4**, **Llama**, and other models. Works with both pa
|
|
|
131
131
|
| Ollama | Llama, Mistral, etc. (local) | Free, local execution, no API costs |
|
|
132
132
|
| Claude SDK | claude-cli, claude-code | Subprocess-based, full MCP support |
|
|
133
133
|
|
|
134
|
-

|
|
135
135
|
|
|
136
136
|
### Built-in Evaluators
|
|
137
137
|
|
|
@@ -154,7 +154,7 @@ Comprehensive validation out of the box. Each evaluator returns a score from 0.0
|
|
|
154
154
|
|
|
155
155
|
**Extensible:** Extend `BaseEvaluator` and implement `evaluate(context) -> EvalResult` to create custom evaluators for your domain.
|
|
156
156
|
|
|
157
|
-

|
|
158
158
|
|
|
159
159
|
### YAML Test Definitions
|
|
160
160
|
|
|
@@ -200,32 +200,13 @@ tests:
|
|
|
200
200
|
duration: 60
|
|
201
201
|
```
|
|
202
202
|
|
|
203
|
-
### Interactive TUI Dashboard
|
|
204
|
-
|
|
205
|
-
Beautiful terminal interface for MCP testing — no browser required:
|
|
206
|
-
|
|
207
|
-
```bash
|
|
208
|
-
testmcpy dash # Launch interactive dashboard
|
|
209
|
-
testmcpy dash --auto-refresh # Live connection monitoring
|
|
210
|
-
testmcpy dash --profile prod # Use specific MCP profile
|
|
211
|
-
```
|
|
212
|
-
|
|
213
|
-
**TUI Features:**
|
|
214
|
-
- Real-time MCP connection status
|
|
215
|
-
- Interactive tool exploration
|
|
216
|
-
- Live test execution with progress
|
|
217
|
-
- Configuration editor
|
|
218
|
-
- Global search across tools, tests, and settings
|
|
219
|
-
- Help system with keyboard shortcuts (press `?`)
|
|
220
|
-
- Multiple themes (default, light, high contrast)
|
|
221
|
-
|
|
222
203
|
### CLI & Web UI
|
|
223
204
|
|
|
224
205
|
- **Rich terminal UI**: Progress bars, colored output, formatted tables
|
|
225
206
|
- **Optional web interface**: Visual tool explorer, interactive chat, analytics dashboards
|
|
226
207
|
- **Real-time feedback**: Watch tests execute with live updates via WebSocket
|
|
227
208
|
|
|
228
|
-

|
|
229
210
|
|
|
230
211
|
## Architecture
|
|
231
212
|
|
|
@@ -394,13 +375,20 @@ testmcpy run tests/ --model claude-haiku-4-5
|
|
|
394
375
|
| `testmcpy chat` | Interactive chat with MCP tools |
|
|
395
376
|
| `testmcpy compare` | Multi-model comparison |
|
|
396
377
|
| **Advanced** | |
|
|
397
|
-
| `testmcpy baseline` | Save
|
|
378
|
+
| `testmcpy baseline-save` | Save current test results as a named baseline |
|
|
379
|
+
| `testmcpy baseline-compare` | Compare a run against a saved baseline |
|
|
380
|
+
| `testmcpy baseline-list` | List saved baselines |
|
|
398
381
|
| `testmcpy mutate` | Prompt mutation testing |
|
|
399
382
|
| `testmcpy metamorphic` | Metamorphic testing |
|
|
383
|
+
| `testmcpy generate` | AI-assisted test generation |
|
|
384
|
+
| `testmcpy smoke-test` | Quick smoke test against an MCP service |
|
|
385
|
+
| `testmcpy coverage` | Tool coverage report for a test suite |
|
|
386
|
+
| `testmcpy multi-env` | Run the same suite against multiple MCP profiles |
|
|
387
|
+
| `testmcpy export-db` | Export the SQLite results database |
|
|
400
388
|
| **UI** | |
|
|
401
|
-
| `testmcpy serve` | Start web UI server (port 8000) |
|
|
402
|
-
| `testmcpy dash` | Launch terminal UI dashboard |
|
|
389
|
+
| `testmcpy serve` | Start web UI server (default port 8000) |
|
|
403
390
|
| `testmcpy config-cmd` | View current configuration |
|
|
391
|
+
| `testmcpy config-mcp` | Print MCP client snippets for Claude Desktop / Code |
|
|
404
392
|
|
|
405
393
|
**Common options:** `--profile`, `--llm-profile`, `--model`, `--provider`, `--timeout`, `--verbose`, `--output`
|
|
406
394
|
|
|
@@ -433,9 +421,9 @@ Environment variables are also supported: `MCP_AUTH_TOKEN`, `MCP_JWT_URL`, `MCP_
|
|
|
433
421
|
|
|
434
422
|
## Web Interface
|
|
435
423
|
|
|
436
|
-
Optional React-based UI with
|
|
424
|
+
Optional React-based UI with 14 pages for visual testing and analytics:
|
|
437
425
|
|
|
438
|
-

|
|
439
427
|
|
|
440
428
|
```bash
|
|
441
429
|
# Install with UI support
|
|
@@ -462,7 +450,20 @@ testmcpy serve
|
|
|
462
450
|
| `/mcp-profiles` | MCP Profiles | MCP server configuration |
|
|
463
451
|
| `/llm-profiles` | LLM Profiles | LLM provider configuration |
|
|
464
452
|
|
|
465
|
-
Access at `http://localhost:8000
|
|
453
|
+
Access at `http://localhost:8000`.
|
|
454
|
+
|
|
455
|
+
#### More screenshots
|
|
456
|
+
|
|
457
|
+
<table>
|
|
458
|
+
<tr>
|
|
459
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/generation-history.png" alt="Generation History page"><br><sub>Generation History — AI-assisted test generation runs</sub></td>
|
|
460
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/auth-debugger.png" alt="Auth Debugger page"><br><sub>Auth Debugger — step through OAuth / JWT / Bearer flows</sub></td>
|
|
461
|
+
</tr>
|
|
462
|
+
<tr>
|
|
463
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/mcp-profiles.png" alt="MCP Profiles page"><br><sub>MCP Profiles — manage MCP service connections</sub></td>
|
|
464
|
+
<td align="center"><img src="https://raw.githubusercontent.com/preset-io/testmcpy/main/docs/screenshots/config.png" alt="Configuration page"><br><sub>Configuration — current settings and client snippets</sub></td>
|
|
465
|
+
</tr>
|
|
466
|
+
</table>
|
|
466
467
|
|
|
467
468
|
## LLM Providers
|
|
468
469
|
|
|
@@ -122,9 +122,6 @@ testmcpy/ui/package.json
|
|
|
122
122
|
testmcpy/ui/postcss.config.js
|
|
123
123
|
testmcpy/ui/tailwind.config.js
|
|
124
124
|
testmcpy/ui/vite.config.js
|
|
125
|
-
testmcpy/ui/dist/index.html
|
|
126
|
-
testmcpy/ui/dist/assets/index-30Ed2JCz.css
|
|
127
|
-
testmcpy/ui/dist/assets/index-6JiH0p1L.js
|
|
128
125
|
testmcpy/ui/src/App.jsx
|
|
129
126
|
testmcpy/ui/src/index.css
|
|
130
127
|
testmcpy/ui/src/main.jsx
|