amagi-mcp-probe 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. amagi_mcp_probe-0.1.0/LICENSE +21 -0
  2. amagi_mcp_probe-0.1.0/PKG-INFO +222 -0
  3. amagi_mcp_probe-0.1.0/README.md +200 -0
  4. amagi_mcp_probe-0.1.0/pyproject.toml +39 -0
  5. amagi_mcp_probe-0.1.0/setup.cfg +4 -0
  6. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/__init__.py +3 -0
  7. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/__main__.py +5 -0
  8. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/cli.py +231 -0
  9. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/config.py +105 -0
  10. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/model.py +106 -0
  11. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/probe.py +512 -0
  12. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/report.py +230 -0
  13. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/samples.py +578 -0
  14. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/schemas.py +126 -0
  15. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe/transport.py +443 -0
  16. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/PKG-INFO +222 -0
  17. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/SOURCES.txt +24 -0
  18. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/dependency_links.txt +1 -0
  19. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/entry_points.txt +2 -0
  20. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/requires.txt +4 -0
  21. amagi_mcp_probe-0.1.0/src/amagi_mcp_probe.egg-info/top_level.txt +1 -0
  22. amagi_mcp_probe-0.1.0/tests/test_cli.py +214 -0
  23. amagi_mcp_probe-0.1.0/tests/test_http.py +59 -0
  24. amagi_mcp_probe-0.1.0/tests/test_probe.py +181 -0
  25. amagi_mcp_probe-0.1.0/tests/test_samples.py +118 -0
  26. amagi_mcp_probe-0.1.0/tests/test_schemas_results.py +67 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Amagisolutionsgroup, Inc. (AmagiLabs)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,222 @@
1
+ Metadata-Version: 2.4
2
+ Name: amagi-mcp-probe
3
+ Version: 0.1.0
4
+ Summary: Headless smoke test for MCP servers: starts the server, checks every tool schema, calls every tool with generated arguments, and fails CI on crashes, hangs and schema violations.
5
+ Author-email: "AmagiLabs (Amagisolutionsgroup, Inc.)" <dev@amagisolutionsgroup.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/amagilabs/amagi-mcp-probe
8
+ Keywords: mcp,model-context-protocol,testing,smoke-test,ci,github-actions,json-schema,claude
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Environment :: Console
13
+ Classifier: Topic :: Software Development :: Testing
14
+ Classifier: Topic :: Software Development :: Quality Assurance
15
+ Requires-Python: >=3.9
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: jsonschema>=4.18
19
+ Provides-Extra: dev
20
+ Requires-Dist: pytest>=7.0; extra == "dev"
21
+ Dynamic: license-file
22
+
23
+ # amagi-mcp-probe
24
+
25
+ Smoke-test any MCP server in one command: it starts the server, checks every tool schema, calls every tool with valid generated arguments, and fails your CI build on crashes, hangs, and schema violations.
26
+
27
+ ![demo](assets/demo.gif)
28
+
29
+ ## Install
30
+
31
+ ```bash
32
+ pipx install amagi-mcp-probe
33
+ ```
34
+
35
+ Python 3.9 or newer. The only dependency is jsonschema.
36
+
37
+ ## Usage
38
+
39
+ Put the command that starts your server after `--`:
40
+
41
+ ```bash
42
+ amagi-mcp-probe -- python my_server.py
43
+ amagi-mcp-probe -- npx -y @modelcontextprotocol/server-everything
44
+ amagi-mcp-probe --url http://localhost:3001/mcp
45
+ ```
46
+
47
+ The probe runs the `initialize` handshake, pages through `tools/list`, validates each `inputSchema` and `outputSchema`, then calls each tool once with arguments built from its schema. Every result is checked as a `CallToolResult`, and `structuredContent` is checked against the tool's `outputSchema`.
48
+
49
+ Real output from `examples/servers/buggy_server.py`, a small server in this repo with one bug per tool:
50
+
51
+ ```
52
+ $ amagi-mcp-probe --timeout 3 -- python examples/servers/buggy_server.py
53
+ buggy_server.py: python examples/servers/buggy_server.py
54
+ buggy 0.3.0, protocol 2025-11-25, ready in 1312 ms
55
+
56
+ PASS echo <1 ms
57
+ FAIL resize_image 156 ms
58
+ error server-crash: server process exited (code 1) while running this tool
59
+ serve("buggy", "0.3.0", TOOLS)
60
+ ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
61
+ File "examples\servers\minimcp.py", line 65, in serve
62
+ result = tool["handler"](params.get("arguments") or {})
63
+ File "examples\servers\buggy_server.py", line 22, in resize_image
64
+ scale = 100 / (args["width"] - 1)
65
+ ~~~~^~~~~~~~~~~~~~~~~~~~~
66
+ ZeroDivisionError: division by zero
67
+ FAIL fetch_page <1 ms
68
+ error stdout-noise: 1 line(s) on stdout (log to stderr instead) were not JSON-RPC while this tool ran
69
+ fetching https://example.com/
70
+ FAIL weather <1 ms
71
+ error structured-content-invalid: structuredContent (root): 'temp_c' is a required property
72
+ FAIL lookup_user <1 ms
73
+ error call-rpc-error: JSON-RPC error -32602: user_id must be a string (args match inputSchema: schema and handler disagree)
74
+ FAIL slow_report 3014 ms
75
+ warn tool-description: no description; models pick tools by their description
76
+ error call-timeout: no answer within 3s
77
+ FAIL count_rows
78
+ error input-schema-invalid: properties/limit/type: 'int' is not valid under any of the given schemas
79
+ SKIP drop_table destructiveHint: true (use --allow-destructive to call it)
80
+ server restarted 2 times after a crash or hang
81
+
82
+ 8 tools: 1 passed, 6 failed, 1 skipped. FAIL
83
+ ```
84
+
85
+ The exit code is 1. After the crash and the hang, the probe restarted the server and kept going, so one broken tool doesn't hide the rest. The crash detail is the tail of the server's stderr.
86
+
87
+ A well-behaved server looks like this:
88
+
89
+ ```
90
+ $ amagi-mcp-probe -- python examples/servers/notes_server.py
91
+ notes_server.py: python examples/servers/notes_server.py
92
+ notes 1.0.0, protocol 2025-11-25, ready in 2235 ms
93
+
94
+ PASS add_note <1 ms
95
+ WARN get_note <1 ms
96
+ warn tool-error: isError: true: no note with id note-000
97
+ PASS search_notes <1 ms
98
+
99
+ 3 tools: 2 passed, 1 warned. PASS
100
+ ```
101
+
102
+ `get_note` got a schema-valid id that doesn't exist, so the tool answered with `isError: true`. The probe counts that as a warning and the run still passes, because the server handled bad input the way MCP intends. Give it a real id with `--args` (below) and all three pass.
103
+
104
+ On the official reference servers, run on Windows with Node 24:
105
+
106
+ - `@modelcontextprotocol/server-everything` 2.0.0 over stdio: 16 tools, 16 passed.
107
+ - The same server over Streamable HTTP (`--url`): 16 tools, 16 passed.
108
+ - `@modelcontextprotocol/server-filesystem` 2.0.0 on a scratch folder: 14 tools, 8 passed, 3 warned (`read_file` on a path that doesn't exist), 3 skipped (`write_file`, `edit_file` and `move_file` are marked destructive).
109
+
110
+ ### How arguments are generated
111
+
112
+ For each tool, the probe builds the dull, valid input a careful user would type: required properties only, the lowest allowed number (1 when the schema allows it), the first `enum` value, and a short string that matches `pattern`, `format`, and the length limits. It validates every generated value against the schema before sending it. If no valid value exists, the tool is reported as `args-unsatisfiable` and not called.
113
+
114
+ Pass `--include-optional` to fill optional properties too, and `-v` to print the arguments sent to each tool.
115
+
116
+ ### Real arguments with `--args`
117
+
118
+ Generated arguments can't guess the id of a note that exists. Give the probe real values in a JSON file, one object per tool (or a list of objects to call a tool more than once):
119
+
120
+ ```json
121
+ {"get_note": {"id": "note-001"}}
122
+ ```
123
+
124
+ ```
125
+ $ amagi-mcp-probe --strict --args args.json -- python examples/servers/notes_server.py
126
+ ...
127
+ 3 tools: 3 passed. PASS
128
+ ```
129
+
130
+ Arguments from the file are checked against the tool's `inputSchema` too. A mismatch is reported as `args-override-invalid`.
131
+
132
+ ### Probe the servers in your client config
133
+
134
+ ```bash
135
+ amagi-mcp-probe --config .mcp.json
136
+ amagi-mcp-probe --config claude_desktop_config.json --server notes
137
+ ```
138
+
139
+ It reads the `mcpServers` format used by Claude Code and Claude Desktop, and the `servers` format in `.vscode/mcp.json`. `${VAR}` and `${VAR:-default}` in a config are expanded from your environment. Legacy SSE servers in a config are reported as unsupported.
140
+
141
+ ### What it checks
142
+
143
+ `amagi-mcp-probe --list-checks` prints all 28 check ids. The main ones:
144
+
145
+ - Startup: the server starts, answers `initialize` within `--startup-timeout`, and returns `protocolVersion`, `capabilities`, and `serverInfo`.
146
+ - stdout: anything on stdout that isn't JSON-RPC (a stray `print`, a startup banner) is an error, because it breaks real clients. The probe tells you which tool was running when it appeared.
147
+ - Tool definitions: names, duplicates, missing descriptions, `inputSchema` that isn't a valid object schema, `required` properties that aren't defined, and invalid `outputSchema`.
148
+ - Calls: crashes, timeouts, JSON-RPC errors on schema-valid arguments, malformed results, `isError: true`, and `structuredContent` that is missing or doesn't match `outputSchema`.
149
+
150
+ Use `--ignore CHECK` to drop a check you don't care about.
151
+
152
+ ### Options
153
+
154
+ ```bash
155
+ amagi-mcp-probe --timeout 10 -- ... # seconds per tools/list and tools/call (default 30)
156
+ amagi-mcp-probe --startup-timeout 60 -- ... # seconds for the initialize handshake (default 30)
157
+ amagi-mcp-probe --only 'get_*' -- ... # call only matching tools (repeatable)
158
+ amagi-mcp-probe --skip send_email -- ... # never call these tools (repeatable)
159
+ amagi-mcp-probe --no-call -- ... # check schemas only, call nothing
160
+ amagi-mcp-probe --allow-destructive -- ... # also call tools marked destructiveHint: true
161
+ amagi-mcp-probe --env API_KEY=test -- ... # environment for the server process (repeatable)
162
+ amagi-mcp-probe --cwd sandbox -- ... # working directory for the server process
163
+ amagi-mcp-probe --url URL --header 'Authorization: Bearer x' # Streamable HTTP server
164
+ amagi-mcp-probe --format json -- ... # also: text (default), markdown, github
165
+ amagi-mcp-probe --junit probe.xml -- ... # also write a JUnit XML report
166
+ amagi-mcp-probe --strict -- ... # warnings fail the run too
167
+ ```
168
+
169
+ Exit codes: 0 means no errors, 1 means errors were found (or warnings, with `--strict`), 2 means a usage or config error.
170
+
171
+ ## Run it in CI
172
+
173
+ ```yaml
174
+ # .github/workflows/mcp-probe.yml
175
+ name: mcp-probe
176
+ on: [push, pull_request]
177
+ jobs:
178
+ probe:
179
+ runs-on: ubuntu-latest
180
+ steps:
181
+ - uses: actions/checkout@v4
182
+ - uses: actions/setup-python@v5
183
+ with:
184
+ python-version: "3.12"
185
+ - run: pip install amagi-mcp-probe
186
+ - run: pip install -e . # your server
187
+ - run: >
188
+ amagi-mcp-probe --format github --junit probe.xml
189
+ --summary "$GITHUB_STEP_SUMMARY"
190
+ -- python -m my_server
191
+ ```
192
+
193
+ `--format github` turns each finding into an error or warning annotation, `--summary` appends a Markdown table of every tool to the job summary page, and `--junit` writes a report that test-report actions can read. A Node server works the same way: add `actions/setup-node`, then put `node dist/index.js` after `--`.
194
+
195
+ The flags in this workflow are covered by the test suite. The workflow file itself hasn't been run on GitHub yet.
196
+
197
+ ## Use it from Claude Code
198
+
199
+ This repo ships a skill at `.claude/skills/amagi-mcp-probe/SKILL.md`. Copy that folder into your own `.claude/skills/` and Claude Code can probe the server you're building, read the JSON report, and fix what failed.
200
+
201
+ ## Limits
202
+
203
+ - It calls your tools for real. A tool that sends email, charges a card, or deletes files will do that, so point the server at a sandbox. Tools marked `destructiveHint: true` are skipped by default, but only if the server marks them.
204
+ - Transports: stdio and Streamable HTTP. The legacy HTTP+SSE transport is not supported.
205
+ - Generated arguments are minimal and know nothing about your data, so expect some `tool-error` warnings for lookups like `get_note`. Use `--args` for those.
206
+ - The probe declares no client capabilities, so tools that need sampling, elicitation, or roots from the client aren't fully exercised. Tools that require task-based execution are skipped.
207
+ - `pattern` regexes are checked with Python's `re`. A pattern that uses ECMA-262 syntax Python lacks (such as `\p{L}`) gets a `schema-note` warning instead of a false error.
208
+ - It makes one call per tool (more with `--args`). That catches crashes and schema mismatches, but edge cases deep inside a handler need your own tests or a fuzzer.
209
+
210
+ ## If this saved you time
211
+
212
+ A star helps other people find it.
213
+
214
+ ## More from AmagiLabs
215
+
216
+ - [amagilabs](https://github.com/amagilabs): other small open source dev tools from the same lab.
217
+ - [amagi-watch](https://github.com/amagilabs/amagi-watch): turn a video URL into a Claude-readable transcript and frame index.
218
+ - [amagi-legal-review-skills](https://github.com/amagilabs/amagi-legal-review-skills): Claude Code skills for contract and lease review.
219
+
220
+ ## License
221
+
222
+ MIT, see [LICENSE](LICENSE). Copyright (c) 2026 Amagisolutionsgroup, Inc. (AmagiLabs)
@@ -0,0 +1,200 @@
1
+ # amagi-mcp-probe
2
+
3
+ Smoke-test any MCP server in one command: it starts the server, checks every tool schema, calls every tool with valid generated arguments, and fails your CI build on crashes, hangs, and schema violations.
4
+
5
+ ![demo](assets/demo.gif)
6
+
7
+ ## Install
8
+
9
+ ```bash
10
+ pipx install amagi-mcp-probe
11
+ ```
12
+
13
+ Python 3.9 or newer. The only dependency is jsonschema.
14
+
15
+ ## Usage
16
+
17
+ Put the command that starts your server after `--`:
18
+
19
+ ```bash
20
+ amagi-mcp-probe -- python my_server.py
21
+ amagi-mcp-probe -- npx -y @modelcontextprotocol/server-everything
22
+ amagi-mcp-probe --url http://localhost:3001/mcp
23
+ ```
24
+
25
+ The probe runs the `initialize` handshake, pages through `tools/list`, validates each `inputSchema` and `outputSchema`, then calls each tool once with arguments built from its schema. Every result is checked as a `CallToolResult`, and `structuredContent` is checked against the tool's `outputSchema`.
26
+
27
+ Real output from `examples/servers/buggy_server.py`, a small server in this repo with one bug per tool:
28
+
29
+ ```
30
+ $ amagi-mcp-probe --timeout 3 -- python examples/servers/buggy_server.py
31
+ buggy_server.py: python examples/servers/buggy_server.py
32
+ buggy 0.3.0, protocol 2025-11-25, ready in 1312 ms
33
+
34
+ PASS echo <1 ms
35
+ FAIL resize_image 156 ms
36
+ error server-crash: server process exited (code 1) while running this tool
37
+ serve("buggy", "0.3.0", TOOLS)
38
+ ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
39
+ File "examples\servers\minimcp.py", line 65, in serve
40
+ result = tool["handler"](params.get("arguments") or {})
41
+ File "examples\servers\buggy_server.py", line 22, in resize_image
42
+ scale = 100 / (args["width"] - 1)
43
+ ~~~~^~~~~~~~~~~~~~~~~~~~~
44
+ ZeroDivisionError: division by zero
45
+ FAIL fetch_page <1 ms
46
+ error stdout-noise: 1 line(s) on stdout (log to stderr instead) were not JSON-RPC while this tool ran
47
+ fetching https://example.com/
48
+ FAIL weather <1 ms
49
+ error structured-content-invalid: structuredContent (root): 'temp_c' is a required property
50
+ FAIL lookup_user <1 ms
51
+ error call-rpc-error: JSON-RPC error -32602: user_id must be a string (args match inputSchema: schema and handler disagree)
52
+ FAIL slow_report 3014 ms
53
+ warn tool-description: no description; models pick tools by their description
54
+ error call-timeout: no answer within 3s
55
+ FAIL count_rows
56
+ error input-schema-invalid: properties/limit/type: 'int' is not valid under any of the given schemas
57
+ SKIP drop_table destructiveHint: true (use --allow-destructive to call it)
58
+ server restarted 2 times after a crash or hang
59
+
60
+ 8 tools: 1 passed, 6 failed, 1 skipped. FAIL
61
+ ```
62
+
63
+ The exit code is 1. After the crash and the hang, the probe restarted the server and kept going, so one broken tool doesn't hide the rest. The crash detail is the tail of the server's stderr.
64
+
65
+ A well-behaved server looks like this:
66
+
67
+ ```
68
+ $ amagi-mcp-probe -- python examples/servers/notes_server.py
69
+ notes_server.py: python examples/servers/notes_server.py
70
+ notes 1.0.0, protocol 2025-11-25, ready in 2235 ms
71
+
72
+ PASS add_note <1 ms
73
+ WARN get_note <1 ms
74
+ warn tool-error: isError: true: no note with id note-000
75
+ PASS search_notes <1 ms
76
+
77
+ 3 tools: 2 passed, 1 warned. PASS
78
+ ```
79
+
80
+ `get_note` got a schema-valid id that doesn't exist, so the tool answered with `isError: true`. The probe counts that as a warning and the run still passes, because the server handled bad input the way MCP intends. Give it a real id with `--args` (below) and all three pass.
81
+
82
+ On the official reference servers, run on Windows with Node 24:
83
+
84
+ - `@modelcontextprotocol/server-everything` 2.0.0 over stdio: 16 tools, 16 passed.
85
+ - The same server over Streamable HTTP (`--url`): 16 tools, 16 passed.
86
+ - `@modelcontextprotocol/server-filesystem` 2.0.0 on a scratch folder: 14 tools, 8 passed, 3 warned (`read_file` on a path that doesn't exist), 3 skipped (`write_file`, `edit_file` and `move_file` are marked destructive).
87
+
88
+ ### How arguments are generated
89
+
90
+ For each tool, the probe builds the dull, valid input a careful user would type: required properties only, the lowest allowed number (1 when the schema allows it), the first `enum` value, and a short string that matches `pattern`, `format`, and the length limits. It validates every generated value against the schema before sending it. If no valid value exists, the tool is reported as `args-unsatisfiable` and not called.
91
+
92
+ Pass `--include-optional` to fill optional properties too, and `-v` to print the arguments sent to each tool.
93
+
94
+ ### Real arguments with `--args`
95
+
96
+ Generated arguments can't guess the id of a note that exists. Give the probe real values in a JSON file, one object per tool (or a list of objects to call a tool more than once):
97
+
98
+ ```json
99
+ {"get_note": {"id": "note-001"}}
100
+ ```
101
+
102
+ ```
103
+ $ amagi-mcp-probe --strict --args args.json -- python examples/servers/notes_server.py
104
+ ...
105
+ 3 tools: 3 passed. PASS
106
+ ```
107
+
108
+ Arguments from the file are checked against the tool's `inputSchema` too. A mismatch is reported as `args-override-invalid`.
109
+
110
+ ### Probe the servers in your client config
111
+
112
+ ```bash
113
+ amagi-mcp-probe --config .mcp.json
114
+ amagi-mcp-probe --config claude_desktop_config.json --server notes
115
+ ```
116
+
117
+ It reads the `mcpServers` format used by Claude Code and Claude Desktop, and the `servers` format in `.vscode/mcp.json`. `${VAR}` and `${VAR:-default}` in a config are expanded from your environment. Legacy SSE servers in a config are reported as unsupported.
118
+
119
+ ### What it checks
120
+
121
+ `amagi-mcp-probe --list-checks` prints all 28 check ids. The main ones:
122
+
123
+ - Startup: the server starts, answers `initialize` within `--startup-timeout`, and returns `protocolVersion`, `capabilities`, and `serverInfo`.
124
+ - stdout: anything on stdout that isn't JSON-RPC (a stray `print`, a startup banner) is an error, because it breaks real clients. The probe tells you which tool was running when it appeared.
125
+ - Tool definitions: names, duplicates, missing descriptions, `inputSchema` that isn't a valid object schema, `required` properties that aren't defined, and invalid `outputSchema`.
126
+ - Calls: crashes, timeouts, JSON-RPC errors on schema-valid arguments, malformed results, `isError: true`, and `structuredContent` that is missing or doesn't match `outputSchema`.
127
+
128
+ Use `--ignore CHECK` to drop a check you don't care about.
129
+
130
+ ### Options
131
+
132
+ ```bash
133
+ amagi-mcp-probe --timeout 10 -- ... # seconds per tools/list and tools/call (default 30)
134
+ amagi-mcp-probe --startup-timeout 60 -- ... # seconds for the initialize handshake (default 30)
135
+ amagi-mcp-probe --only 'get_*' -- ... # call only matching tools (repeatable)
136
+ amagi-mcp-probe --skip send_email -- ... # never call these tools (repeatable)
137
+ amagi-mcp-probe --no-call -- ... # check schemas only, call nothing
138
+ amagi-mcp-probe --allow-destructive -- ... # also call tools marked destructiveHint: true
139
+ amagi-mcp-probe --env API_KEY=test -- ... # environment for the server process (repeatable)
140
+ amagi-mcp-probe --cwd sandbox -- ... # working directory for the server process
141
+ amagi-mcp-probe --url URL --header 'Authorization: Bearer x' # Streamable HTTP server
142
+ amagi-mcp-probe --format json -- ... # also: text (default), markdown, github
143
+ amagi-mcp-probe --junit probe.xml -- ... # also write a JUnit XML report
144
+ amagi-mcp-probe --strict -- ... # warnings fail the run too
145
+ ```
146
+
147
+ Exit codes: 0 means no errors, 1 means errors were found (or warnings, with `--strict`), 2 means a usage or config error.
148
+
149
+ ## Run it in CI
150
+
151
+ ```yaml
152
+ # .github/workflows/mcp-probe.yml
153
+ name: mcp-probe
154
+ on: [push, pull_request]
155
+ jobs:
156
+ probe:
157
+ runs-on: ubuntu-latest
158
+ steps:
159
+ - uses: actions/checkout@v4
160
+ - uses: actions/setup-python@v5
161
+ with:
162
+ python-version: "3.12"
163
+ - run: pip install amagi-mcp-probe
164
+ - run: pip install -e . # your server
165
+ - run: >
166
+ amagi-mcp-probe --format github --junit probe.xml
167
+ --summary "$GITHUB_STEP_SUMMARY"
168
+ -- python -m my_server
169
+ ```
170
+
171
+ `--format github` turns each finding into an error or warning annotation, `--summary` appends a Markdown table of every tool to the job summary page, and `--junit` writes a report that test-report actions can read. A Node server works the same way: add `actions/setup-node`, then put `node dist/index.js` after `--`.
172
+
173
+ The flags in this workflow are covered by the test suite. The workflow file itself hasn't been run on GitHub yet.
174
+
175
+ ## Use it from Claude Code
176
+
177
+ This repo ships a skill at `.claude/skills/amagi-mcp-probe/SKILL.md`. Copy that folder into your own `.claude/skills/` and Claude Code can probe the server you're building, read the JSON report, and fix what failed.
178
+
179
+ ## Limits
180
+
181
+ - It calls your tools for real. A tool that sends email, charges a card, or deletes files will do that, so point the server at a sandbox. Tools marked `destructiveHint: true` are skipped by default, but only if the server marks them.
182
+ - Transports: stdio and Streamable HTTP. The legacy HTTP+SSE transport is not supported.
183
+ - Generated arguments are minimal and know nothing about your data, so expect some `tool-error` warnings for lookups like `get_note`. Use `--args` for those.
184
+ - The probe declares no client capabilities, so tools that need sampling, elicitation, or roots from the client aren't fully exercised. Tools that require task-based execution are skipped.
185
+ - `pattern` regexes are checked with Python's `re`. A pattern that uses ECMA-262 syntax Python lacks (such as `\p{L}`) gets a `schema-note` warning instead of a false error.
186
+ - It makes one call per tool (more with `--args`). That catches crashes and schema mismatches, but edge cases deep inside a handler need your own tests or a fuzzer.
187
+
188
+ ## If this saved you time
189
+
190
+ A star helps other people find it.
191
+
192
+ ## More from AmagiLabs
193
+
194
+ - [amagilabs](https://github.com/amagilabs): other small open source dev tools from the same lab.
195
+ - [amagi-watch](https://github.com/amagilabs/amagi-watch): turn a video URL into a Claude-readable transcript and frame index.
196
+ - [amagi-legal-review-skills](https://github.com/amagilabs/amagi-legal-review-skills): Claude Code skills for contract and lease review.
197
+
198
+ ## License
199
+
200
+ MIT, see [LICENSE](LICENSE). Copyright (c) 2026 Amagisolutionsgroup, Inc. (AmagiLabs)
@@ -0,0 +1,39 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "amagi-mcp-probe"
7
+ version = "0.1.0"
8
+ description = "Headless smoke test for MCP servers: starts the server, checks every tool schema, calls every tool with generated arguments, and fails CI on crashes, hangs and schema violations."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ authors = [{ name = "AmagiLabs (Amagisolutionsgroup, Inc.)", email = "dev@amagisolutionsgroup.com" }]
12
+ requires-python = ">=3.9"
13
+ dependencies = [
14
+ "jsonschema>=4.18",
15
+ ]
16
+ keywords = ["mcp", "model-context-protocol", "testing", "smoke-test", "ci", "github-actions", "json-schema", "claude"]
17
+ classifiers = [
18
+ "Programming Language :: Python :: 3",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Operating System :: OS Independent",
21
+ "Environment :: Console",
22
+ "Topic :: Software Development :: Testing",
23
+ "Topic :: Software Development :: Quality Assurance",
24
+ ]
25
+
26
+ [project.urls]
27
+ Homepage = "https://github.com/amagilabs/amagi-mcp-probe"
28
+
29
+ [project.scripts]
30
+ amagi-mcp-probe = "amagi_mcp_probe.cli:main"
31
+
32
+ [project.optional-dependencies]
33
+ dev = ["pytest>=7.0"]
34
+
35
+ [tool.setuptools.packages.find]
36
+ where = ["src"]
37
+
38
+ [tool.pytest.ini_options]
39
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """amagi-mcp-probe: a headless, CI-first smoke test for MCP servers."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())