amagi-mcp-probe 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- amagi_mcp_probe-0.1.0/LICENSE +21 -0
- amagi_mcp_probe-0.1.0/PKG-INFO +222 -0
- amagi_mcp_probe-0.1.0/README.md +200 -0
- amagi_mcp_probe-0.1.0/pyproject.toml +39 -0
- amagi_mcp_probe-0.1.0/setup.cfg +4 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/__init__.py +3 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/__main__.py +5 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/cli.py +231 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/config.py +105 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/model.py +106 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/probe.py +512 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/report.py +230 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/samples.py +578 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/schemas.py +126 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/transport.py +443 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/PKG-INFO +222 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/SOURCES.txt +24 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/dependency_links.txt +1 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/entry_points.txt +2 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/requires.txt +4 -0
- amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/top_level.txt +1 -0
- amagi_mcp_probe-0.1.0/tests/test_cli.py +214 -0
- amagi_mcp_probe-0.1.0/tests/test_http.py +59 -0
- amagi_mcp_probe-0.1.0/tests/test_probe.py +181 -0
- amagi_mcp_probe-0.1.0/tests/test_samples.py +118 -0
- amagi_mcp_probe-0.1.0/tests/test_schemas_results.py +67 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Amagisolutionsgroup, Inc. (AmagiLabs)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: amagi-mcp-probe
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Headless smoke test for MCP servers: starts the server, checks every tool schema, calls every tool with generated arguments, and fails CI on crashes, hangs and schema violations.
|
|
5
|
+
Author-email: "AmagiLabs (Amagisolutionsgroup, Inc.)" <dev@amagisolutionsgroup.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/amagilabs/amagi-mcp-probe
|
|
8
|
+
Keywords: mcp,model-context-protocol,testing,smoke-test,ci,github-actions,json-schema,claude
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Topic :: Software Development :: Testing
|
|
14
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: jsonschema>=4.18
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# amagi-mcp-probe
|
|
24
|
+
|
|
25
|
+
Smoke-test any MCP server in one command: it starts the server, checks every tool schema, calls every tool with valid generated arguments, and fails your CI build on crashes, hangs, and schema violations.
|
|
26
|
+
|
|
27
|
+

|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pipx install amagi-mcp-probe
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Python 3.9 or newer. The only dependency is jsonschema.
|
|
36
|
+
|
|
37
|
+
## Usage
|
|
38
|
+
|
|
39
|
+
Put the command that starts your server after `--`:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
amagi-mcp-probe -- python my_server.py
|
|
43
|
+
amagi-mcp-probe -- npx -y @modelcontextprotocol/server-everything
|
|
44
|
+
amagi-mcp-probe --url http://localhost:3001/mcp
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
The probe runs the `initialize` handshake, pages through `tools/list`, validates each `inputSchema` and `outputSchema`, then calls each tool once with arguments built from its schema. Every result is checked as a `CallToolResult`, and `structuredContent` is checked against the tool's `outputSchema`.
|
|
48
|
+
|
|
49
|
+
Real output from `examples/servers/buggy_server.py`, a small server in this repo with one bug per tool:
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
$ amagi-mcp-probe --timeout 3 -- python examples/servers/buggy_server.py
|
|
53
|
+
buggy_server.py: python examples/servers/buggy_server.py
|
|
54
|
+
buggy 0.3.0, protocol 2025-11-25, ready in 1312 ms
|
|
55
|
+
|
|
56
|
+
PASS echo <1 ms
|
|
57
|
+
FAIL resize_image 156 ms
|
|
58
|
+
error server-crash: server process exited (code 1) while running this tool
|
|
59
|
+
serve("buggy", "0.3.0", TOOLS)
|
|
60
|
+
~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
61
|
+
File "examples\servers\minimcp.py", line 65, in serve
|
|
62
|
+
result = tool["handler"](params.get("arguments") or {})
|
|
63
|
+
File "examples\servers\buggy_server.py", line 22, in resize_image
|
|
64
|
+
scale = 100 / (args["width"] - 1)
|
|
65
|
+
~~~~^~~~~~~~~~~~~~~~~~~~~
|
|
66
|
+
ZeroDivisionError: division by zero
|
|
67
|
+
FAIL fetch_page <1 ms
|
|
68
|
+
error stdout-noise: 1 line(s) on stdout (log to stderr instead) were not JSON-RPC while this tool ran
|
|
69
|
+
fetching https://example.com/
|
|
70
|
+
FAIL weather <1 ms
|
|
71
|
+
error structured-content-invalid: structuredContent (root): 'temp_c' is a required property
|
|
72
|
+
FAIL lookup_user <1 ms
|
|
73
|
+
error call-rpc-error: JSON-RPC error -32602: user_id must be a string (args match inputSchema: schema and handler disagree)
|
|
74
|
+
FAIL slow_report 3014 ms
|
|
75
|
+
warn tool-description: no description; models pick tools by their description
|
|
76
|
+
error call-timeout: no answer within 3s
|
|
77
|
+
FAIL count_rows
|
|
78
|
+
error input-schema-invalid: properties/limit/type: 'int' is not valid under any of the given schemas
|
|
79
|
+
SKIP drop_table destructiveHint: true (use --allow-destructive to call it)
|
|
80
|
+
server restarted 2 times after a crash or hang
|
|
81
|
+
|
|
82
|
+
8 tools: 1 passed, 6 failed, 1 skipped. FAIL
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The exit code is 1. After the crash and the hang, the probe restarted the server and kept going, so one broken tool doesn't hide the rest. The crash detail is the tail of the server's stderr.
|
|
86
|
+
|
|
87
|
+
A well-behaved server looks like this:
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
$ amagi-mcp-probe -- python examples/servers/notes_server.py
|
|
91
|
+
notes_server.py: python examples/servers/notes_server.py
|
|
92
|
+
notes 1.0.0, protocol 2025-11-25, ready in 2235 ms
|
|
93
|
+
|
|
94
|
+
PASS add_note <1 ms
|
|
95
|
+
WARN get_note <1 ms
|
|
96
|
+
warn tool-error: isError: true: no note with id note-000
|
|
97
|
+
PASS search_notes <1 ms
|
|
98
|
+
|
|
99
|
+
3 tools: 2 passed, 1 warned. PASS
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
`get_note` got a schema-valid id that doesn't exist, so the tool answered with `isError: true`. The probe counts that as a warning and the run still passes, because the server handled bad input the way MCP intends. Give it a real id with `--args` (below) and all three pass.
|
|
103
|
+
|
|
104
|
+
On the official reference servers, run on Windows with Node 24:
|
|
105
|
+
|
|
106
|
+
- `@modelcontextprotocol/server-everything` 2.0.0 over stdio: 16 tools, 16 passed.
|
|
107
|
+
- The same server over Streamable HTTP (`--url`): 16 tools, 16 passed.
|
|
108
|
+
- `@modelcontextprotocol/server-filesystem` 2.0.0 on a scratch folder: 14 tools, 8 passed, 3 warned (`read_file` on a path that doesn't exist), 3 skipped (`write_file`, `edit_file` and `move_file` are marked destructive).
|
|
109
|
+
|
|
110
|
+
### How arguments are generated
|
|
111
|
+
|
|
112
|
+
For each tool, the probe builds the dull, valid input a careful user would type: required properties only, the lowest allowed number (1 when the schema allows it), the first `enum` value, and a short string that matches `pattern`, `format`, and the length limits. It validates every generated value against the schema before sending it. If no valid value exists, the tool is reported as `args-unsatisfiable` and not called.
|
|
113
|
+
|
|
114
|
+
Pass `--include-optional` to fill optional properties too, and `-v` to print the arguments sent to each tool.
|
|
115
|
+
|
|
116
|
+
### Real arguments with `--args`
|
|
117
|
+
|
|
118
|
+
Generated arguments can't guess the id of a note that exists. Give the probe real values in a JSON file, one object per tool (or a list of objects to call a tool more than once):
|
|
119
|
+
|
|
120
|
+
```json
|
|
121
|
+
{"get_note": {"id": "note-001"}}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
$ amagi-mcp-probe --strict --args args.json -- python examples/servers/notes_server.py
|
|
126
|
+
...
|
|
127
|
+
3 tools: 3 passed. PASS
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Arguments from the file are checked against the tool's `inputSchema` too. A mismatch is reported as `args-override-invalid`.
|
|
131
|
+
|
|
132
|
+
### Probe the servers in your client config
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
amagi-mcp-probe --config .mcp.json
|
|
136
|
+
amagi-mcp-probe --config claude_desktop_config.json --server notes
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
It reads the `mcpServers` format used by Claude Code and Claude Desktop, and the `servers` format in `.vscode/mcp.json`. `${VAR}` and `${VAR:-default}` in a config are expanded from your environment. Legacy SSE servers in a config are reported as unsupported.
|
|
140
|
+
|
|
141
|
+
### What it checks
|
|
142
|
+
|
|
143
|
+
`amagi-mcp-probe --list-checks` prints all 28 check ids. The main ones:
|
|
144
|
+
|
|
145
|
+
- Startup: the server starts, answers `initialize` within `--startup-timeout`, and returns `protocolVersion`, `capabilities`, and `serverInfo`.
|
|
146
|
+
- stdout: anything on stdout that isn't JSON-RPC (a stray `print`, a startup banner) is an error, because it breaks real clients. The probe tells you which tool was running when it appeared.
|
|
147
|
+
- Tool definitions: names, duplicates, missing descriptions, `inputSchema` that isn't a valid object schema, `required` properties that aren't defined, and invalid `outputSchema`.
|
|
148
|
+
- Calls: crashes, timeouts, JSON-RPC errors on schema-valid arguments, malformed results, `isError: true`, and `structuredContent` that is missing or doesn't match `outputSchema`.
|
|
149
|
+
|
|
150
|
+
Use `--ignore CHECK` to drop a check you don't care about.
|
|
151
|
+
|
|
152
|
+
### Options
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
amagi-mcp-probe --timeout 10 -- ... # seconds per tools/list and tools/call (default 30)
|
|
156
|
+
amagi-mcp-probe --startup-timeout 60 -- ... # seconds for the initialize handshake (default 30)
|
|
157
|
+
amagi-mcp-probe --only 'get_*' -- ... # call only matching tools (repeatable)
|
|
158
|
+
amagi-mcp-probe --skip send_email -- ... # never call these tools (repeatable)
|
|
159
|
+
amagi-mcp-probe --no-call -- ... # check schemas only, call nothing
|
|
160
|
+
amagi-mcp-probe --allow-destructive -- ... # also call tools marked destructiveHint: true
|
|
161
|
+
amagi-mcp-probe --env API_KEY=test -- ... # environment for the server process (repeatable)
|
|
162
|
+
amagi-mcp-probe --cwd sandbox -- ... # working directory for the server process
|
|
163
|
+
amagi-mcp-probe --url URL --header 'Authorization: Bearer x' # Streamable HTTP server
|
|
164
|
+
amagi-mcp-probe --format json -- ... # also: text (default), markdown, github
|
|
165
|
+
amagi-mcp-probe --junit probe.xml -- ... # also write a JUnit XML report
|
|
166
|
+
amagi-mcp-probe --strict -- ... # warnings fail the run too
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Exit codes: 0 means no errors, 1 means errors were found (or warnings, with `--strict`), 2 means a usage or config error.
|
|
170
|
+
|
|
171
|
+
## Run it in CI
|
|
172
|
+
|
|
173
|
+
```yaml
|
|
174
|
+
# .github/workflows/mcp-probe.yml
|
|
175
|
+
name: mcp-probe
|
|
176
|
+
on: [push, pull_request]
|
|
177
|
+
jobs:
|
|
178
|
+
probe:
|
|
179
|
+
runs-on: ubuntu-latest
|
|
180
|
+
steps:
|
|
181
|
+
- uses: actions/checkout@v4
|
|
182
|
+
- uses: actions/setup-python@v5
|
|
183
|
+
with:
|
|
184
|
+
python-version: "3.12"
|
|
185
|
+
- run: pip install amagi-mcp-probe
|
|
186
|
+
- run: pip install -e . # your server
|
|
187
|
+
- run: >
|
|
188
|
+
amagi-mcp-probe --format github --junit probe.xml
|
|
189
|
+
--summary "$GITHUB_STEP_SUMMARY"
|
|
190
|
+
-- python -m my_server
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
`--format github` turns each finding into an error or warning annotation, `--summary` appends a Markdown table of every tool to the job summary page, and `--junit` writes a report that test-report actions can read. A Node server works the same way: add `actions/setup-node`, then put `node dist/index.js` after `--`.
|
|
194
|
+
|
|
195
|
+
The flags in this workflow are covered by the test suite. The workflow file itself hasn't been run on GitHub yet.
|
|
196
|
+
|
|
197
|
+
## Use it from Claude Code
|
|
198
|
+
|
|
199
|
+
This repo ships a skill at `.claude/skills/amagi-mcp-probe/SKILL.md`. Copy that folder into your own `.claude/skills/` and Claude Code can probe the server you're building, read the JSON report, and fix what failed.
|
|
200
|
+
|
|
201
|
+
## Limits
|
|
202
|
+
|
|
203
|
+
- It calls your tools for real. A tool that sends email, charges a card, or deletes files will do that, so point the server at a sandbox. Tools marked `destructiveHint: true` are skipped by default, but only if the server marks them.
|
|
204
|
+
- Transports: stdio and Streamable HTTP. The legacy HTTP+SSE transport is not supported.
|
|
205
|
+
- Generated arguments are minimal and know nothing about your data, so expect some `tool-error` warnings for lookups like `get_note`. Use `--args` for those.
|
|
206
|
+
- The probe declares no client capabilities, so tools that need sampling, elicitation, or roots from the client aren't fully exercised. Tools that require task-based execution are skipped.
|
|
207
|
+
- `pattern` regexes are checked with Python's `re`. A pattern that uses ECMA-262 syntax Python lacks (such as `\p{L}`) gets a `schema-note` warning instead of a false error.
|
|
208
|
+
- It makes one call per tool (more with `--args`). That catches crashes and schema mismatches, but edge cases deep inside a handler need your own tests or a fuzzer.
|
|
209
|
+
|
|
210
|
+
## If this saved you time
|
|
211
|
+
|
|
212
|
+
A star helps other people find it.
|
|
213
|
+
|
|
214
|
+
## More from AmagiLabs
|
|
215
|
+
|
|
216
|
+
- [amagilabs](https://github.com/amagilabs): other small open source dev tools from the same lab.
|
|
217
|
+
- [amagi-watch](https://github.com/amagilabs/amagi-watch): turn a video URL into a Claude-readable transcript and frame index.
|
|
218
|
+
- [amagi-legal-review-skills](https://github.com/amagilabs/amagi-legal-review-skills): Claude Code skills for contract and lease review.
|
|
219
|
+
|
|
220
|
+
## License
|
|
221
|
+
|
|
222
|
+
MIT, see [LICENSE](LICENSE). Copyright (c) 2026 Amagisolutionsgroup, Inc. (AmagiLabs)
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# amagi-mcp-probe
|
|
2
|
+
|
|
3
|
+
Smoke-test any MCP server in one command: it starts the server, checks every tool schema, calls every tool with valid generated arguments, and fails your CI build on crashes, hangs, and schema violations.
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pipx install amagi-mcp-probe
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Python 3.9 or newer. The only dependency is jsonschema.
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
Put the command that starts your server after `--`:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
amagi-mcp-probe -- python my_server.py
|
|
21
|
+
amagi-mcp-probe -- npx -y @modelcontextprotocol/server-everything
|
|
22
|
+
amagi-mcp-probe --url http://localhost:3001/mcp
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The probe runs the `initialize` handshake, pages through `tools/list`, validates each `inputSchema` and `outputSchema`, then calls each tool once with arguments built from its schema. Every result is checked as a `CallToolResult`, and `structuredContent` is checked against the tool's `outputSchema`.
|
|
26
|
+
|
|
27
|
+
Real output from `examples/servers/buggy_server.py`, a small server in this repo with one bug per tool:
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
$ amagi-mcp-probe --timeout 3 -- python examples/servers/buggy_server.py
|
|
31
|
+
buggy_server.py: python examples/servers/buggy_server.py
|
|
32
|
+
buggy 0.3.0, protocol 2025-11-25, ready in 1312 ms
|
|
33
|
+
|
|
34
|
+
PASS echo <1 ms
|
|
35
|
+
FAIL resize_image 156 ms
|
|
36
|
+
error server-crash: server process exited (code 1) while running this tool
|
|
37
|
+
serve("buggy", "0.3.0", TOOLS)
|
|
38
|
+
~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
39
|
+
File "examples\servers\minimcp.py", line 65, in serve
|
|
40
|
+
result = tool["handler"](params.get("arguments") or {})
|
|
41
|
+
File "examples\servers\buggy_server.py", line 22, in resize_image
|
|
42
|
+
scale = 100 / (args["width"] - 1)
|
|
43
|
+
~~~~^~~~~~~~~~~~~~~~~~~~~
|
|
44
|
+
ZeroDivisionError: division by zero
|
|
45
|
+
FAIL fetch_page <1 ms
|
|
46
|
+
error stdout-noise: 1 line(s) on stdout (log to stderr instead) were not JSON-RPC while this tool ran
|
|
47
|
+
fetching https://example.com/
|
|
48
|
+
FAIL weather <1 ms
|
|
49
|
+
error structured-content-invalid: structuredContent (root): 'temp_c' is a required property
|
|
50
|
+
FAIL lookup_user <1 ms
|
|
51
|
+
error call-rpc-error: JSON-RPC error -32602: user_id must be a string (args match inputSchema: schema and handler disagree)
|
|
52
|
+
FAIL slow_report 3014 ms
|
|
53
|
+
warn tool-description: no description; models pick tools by their description
|
|
54
|
+
error call-timeout: no answer within 3s
|
|
55
|
+
FAIL count_rows
|
|
56
|
+
error input-schema-invalid: properties/limit/type: 'int' is not valid under any of the given schemas
|
|
57
|
+
SKIP drop_table destructiveHint: true (use --allow-destructive to call it)
|
|
58
|
+
server restarted 2 times after a crash or hang
|
|
59
|
+
|
|
60
|
+
8 tools: 1 passed, 6 failed, 1 skipped. FAIL
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The exit code is 1. After the crash and the hang, the probe restarted the server and kept going, so one broken tool doesn't hide the rest. The crash detail is the tail of the server's stderr.
|
|
64
|
+
|
|
65
|
+
A well-behaved server looks like this:
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
$ amagi-mcp-probe -- python examples/servers/notes_server.py
|
|
69
|
+
notes_server.py: python examples/servers/notes_server.py
|
|
70
|
+
notes 1.0.0, protocol 2025-11-25, ready in 2235 ms
|
|
71
|
+
|
|
72
|
+
PASS add_note <1 ms
|
|
73
|
+
WARN get_note <1 ms
|
|
74
|
+
warn tool-error: isError: true: no note with id note-000
|
|
75
|
+
PASS search_notes <1 ms
|
|
76
|
+
|
|
77
|
+
3 tools: 2 passed, 1 warned. PASS
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`get_note` got a schema-valid id that doesn't exist, so the tool answered with `isError: true`. The probe counts that as a warning and the run still passes, because the server handled bad input the way MCP intends. Give it a real id with `--args` (below) and all three pass.
|
|
81
|
+
|
|
82
|
+
On the official reference servers, run on Windows with Node 24:
|
|
83
|
+
|
|
84
|
+
- `@modelcontextprotocol/server-everything` 2.0.0 over stdio: 16 tools, 16 passed.
|
|
85
|
+
- The same server over Streamable HTTP (`--url`): 16 tools, 16 passed.
|
|
86
|
+
- `@modelcontextprotocol/server-filesystem` 2.0.0 on a scratch folder: 14 tools, 8 passed, 3 warned (`read_file` on a path that doesn't exist), 3 skipped (`write_file`, `edit_file` and `move_file` are marked destructive).
|
|
87
|
+
|
|
88
|
+
### How arguments are generated
|
|
89
|
+
|
|
90
|
+
For each tool, the probe builds the dull, valid input a careful user would type: required properties only, the lowest allowed number (1 when the schema allows it), the first `enum` value, and a short string that matches `pattern`, `format`, and the length limits. It validates every generated value against the schema before sending it. If no valid value exists, the tool is reported as `args-unsatisfiable` and not called.
|
|
91
|
+
|
|
92
|
+
Pass `--include-optional` to fill optional properties too, and `-v` to print the arguments sent to each tool.
|
|
93
|
+
|
|
94
|
+
### Real arguments with `--args`
|
|
95
|
+
|
|
96
|
+
Generated arguments can't guess the id of a note that exists. Give the probe real values in a JSON file, one object per tool (or a list of objects to call a tool more than once):
|
|
97
|
+
|
|
98
|
+
```json
|
|
99
|
+
{"get_note": {"id": "note-001"}}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
$ amagi-mcp-probe --strict --args args.json -- python examples/servers/notes_server.py
|
|
104
|
+
...
|
|
105
|
+
3 tools: 3 passed. PASS
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Arguments from the file are checked against the tool's `inputSchema` too. A mismatch is reported as `args-override-invalid`.
|
|
109
|
+
|
|
110
|
+
### Probe the servers in your client config
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
amagi-mcp-probe --config .mcp.json
|
|
114
|
+
amagi-mcp-probe --config claude_desktop_config.json --server notes
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
It reads the `mcpServers` format used by Claude Code and Claude Desktop, and the `servers` format in `.vscode/mcp.json`. `${VAR}` and `${VAR:-default}` in a config are expanded from your environment. Legacy SSE servers in a config are reported as unsupported.
|
|
118
|
+
|
|
119
|
+
### What it checks
|
|
120
|
+
|
|
121
|
+
`amagi-mcp-probe --list-checks` prints all 28 check ids. The main ones:
|
|
122
|
+
|
|
123
|
+
- Startup: the server starts, answers `initialize` within `--startup-timeout`, and returns `protocolVersion`, `capabilities`, and `serverInfo`.
|
|
124
|
+
- stdout: anything on stdout that isn't JSON-RPC (a stray `print`, a startup banner) is an error, because it breaks real clients. The probe tells you which tool was running when it appeared.
|
|
125
|
+
- Tool definitions: names, duplicates, missing descriptions, `inputSchema` that isn't a valid object schema, `required` properties that aren't defined, and invalid `outputSchema`.
|
|
126
|
+
- Calls: crashes, timeouts, JSON-RPC errors on schema-valid arguments, malformed results, `isError: true`, and `structuredContent` that is missing or doesn't match `outputSchema`.
|
|
127
|
+
|
|
128
|
+
Use `--ignore CHECK` to drop a check you don't care about.
|
|
129
|
+
|
|
130
|
+
### Options
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
amagi-mcp-probe --timeout 10 -- ... # seconds per tools/list and tools/call (default 30)
|
|
134
|
+
amagi-mcp-probe --startup-timeout 60 -- ... # seconds for the initialize handshake (default 30)
|
|
135
|
+
amagi-mcp-probe --only 'get_*' -- ... # call only matching tools (repeatable)
|
|
136
|
+
amagi-mcp-probe --skip send_email -- ... # never call these tools (repeatable)
|
|
137
|
+
amagi-mcp-probe --no-call -- ... # check schemas only, call nothing
|
|
138
|
+
amagi-mcp-probe --allow-destructive -- ... # also call tools marked destructiveHint: true
|
|
139
|
+
amagi-mcp-probe --env API_KEY=test -- ... # environment for the server process (repeatable)
|
|
140
|
+
amagi-mcp-probe --cwd sandbox -- ... # working directory for the server process
|
|
141
|
+
amagi-mcp-probe --url URL --header 'Authorization: Bearer x' # Streamable HTTP server
|
|
142
|
+
amagi-mcp-probe --format json -- ... # also: text (default), markdown, github
|
|
143
|
+
amagi-mcp-probe --junit probe.xml -- ... # also write a JUnit XML report
|
|
144
|
+
amagi-mcp-probe --strict -- ... # warnings fail the run too
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Exit codes: 0 means no errors, 1 means errors were found (or warnings, with `--strict`), 2 means a usage or config error.
|
|
148
|
+
|
|
149
|
+
## Run it in CI
|
|
150
|
+
|
|
151
|
+
```yaml
|
|
152
|
+
# .github/workflows/mcp-probe.yml
|
|
153
|
+
name: mcp-probe
|
|
154
|
+
on: [push, pull_request]
|
|
155
|
+
jobs:
|
|
156
|
+
probe:
|
|
157
|
+
runs-on: ubuntu-latest
|
|
158
|
+
steps:
|
|
159
|
+
- uses: actions/checkout@v4
|
|
160
|
+
- uses: actions/setup-python@v5
|
|
161
|
+
with:
|
|
162
|
+
python-version: "3.12"
|
|
163
|
+
- run: pip install amagi-mcp-probe
|
|
164
|
+
- run: pip install -e . # your server
|
|
165
|
+
- run: >
|
|
166
|
+
amagi-mcp-probe --format github --junit probe.xml
|
|
167
|
+
--summary "$GITHUB_STEP_SUMMARY"
|
|
168
|
+
-- python -m my_server
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
`--format github` turns each finding into an error or warning annotation, `--summary` appends a Markdown table of every tool to the job summary page, and `--junit` writes a report that test-report actions can read. A Node server works the same way: add `actions/setup-node`, then put `node dist/index.js` after `--`.
|
|
172
|
+
|
|
173
|
+
The flags in this workflow are covered by the test suite. The workflow file itself hasn't been run on GitHub yet.
|
|
174
|
+
|
|
175
|
+
## Use it from Claude Code
|
|
176
|
+
|
|
177
|
+
This repo ships a skill at `.claude/skills/amagi-mcp-probe/SKILL.md`. Copy that folder into your own `.claude/skills/` and Claude Code can probe the server you're building, read the JSON report, and fix what failed.
|
|
178
|
+
|
|
179
|
+
## Limits
|
|
180
|
+
|
|
181
|
+
- It calls your tools for real. A tool that sends email, charges a card, or deletes files will do that, so point the server at a sandbox. Tools marked `destructiveHint: true` are skipped by default, but only if the server marks them.
|
|
182
|
+
- Transports: stdio and Streamable HTTP. The legacy HTTP+SSE transport is not supported.
|
|
183
|
+
- Generated arguments are minimal and know nothing about your data, so expect some `tool-error` warnings for lookups like `get_note`. Use `--args` for those.
|
|
184
|
+
- The probe declares no client capabilities, so tools that need sampling, elicitation, or roots from the client aren't fully exercised. Tools that require task-based execution are skipped.
|
|
185
|
+
- `pattern` regexes are checked with Python's `re`. A pattern that uses ECMA-262 syntax Python lacks (such as `\p{L}`) gets a `schema-note` warning instead of a false error.
|
|
186
|
+
- It makes one call per tool (more with `--args`). That catches crashes and schema mismatches, but edge cases deep inside a handler need your own tests or a fuzzer.
|
|
187
|
+
|
|
188
|
+
## If this saved you time
|
|
189
|
+
|
|
190
|
+
A star helps other people find it.
|
|
191
|
+
|
|
192
|
+
## More from AmagiLabs
|
|
193
|
+
|
|
194
|
+
- [amagilabs](https://github.com/amagilabs): other small open source dev tools from the same lab.
|
|
195
|
+
- [amagi-watch](https://github.com/amagilabs/amagi-watch): turn a video URL into a Claude-readable transcript and frame index.
|
|
196
|
+
- [amagi-legal-review-skills](https://github.com/amagilabs/amagi-legal-review-skills): Claude Code skills for contract and lease review.
|
|
197
|
+
|
|
198
|
+
## License
|
|
199
|
+
|
|
200
|
+
MIT, see [LICENSE](LICENSE). Copyright (c) 2026 Amagisolutionsgroup, Inc. (AmagiLabs)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "amagi-mcp-probe"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Headless smoke test for MCP servers: starts the server, checks every tool schema, calls every tool with generated arguments, and fails CI on crashes, hangs and schema violations."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [{ name = "AmagiLabs (Amagisolutionsgroup, Inc.)", email = "dev@amagisolutionsgroup.com" }]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
dependencies = [
|
|
14
|
+
"jsonschema>=4.18",
|
|
15
|
+
]
|
|
16
|
+
keywords = ["mcp", "model-context-protocol", "testing", "smoke-test", "ci", "github-actions", "json-schema", "claude"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Environment :: Console",
|
|
22
|
+
"Topic :: Software Development :: Testing",
|
|
23
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/amagilabs/amagi-mcp-probe"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
amagi-mcp-probe = "amagi_mcp_probe.cli:main"
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
dev = ["pytest>=7.0"]
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.packages.find]
|
|
36
|
+
where = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
testpaths = ["tests"]
|