pytest-mcp-server 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_mcp_server-0.1.0/.gitignore +12 -0
- pytest_mcp_server-0.1.0/CHANGELOG.md +16 -0
- pytest_mcp_server-0.1.0/LICENSE +21 -0
- pytest_mcp_server-0.1.0/PKG-INFO +239 -0
- pytest_mcp_server-0.1.0/README.md +214 -0
- pytest_mcp_server-0.1.0/pyproject.toml +46 -0
- pytest_mcp_server-0.1.0/src/mcptest/__init__.py +45 -0
- pytest_mcp_server-0.1.0/src/mcptest/__main__.py +5 -0
- pytest_mcp_server-0.1.0/src/mcptest/cli.py +180 -0
- pytest_mcp_server-0.1.0/src/mcptest/client.py +553 -0
- pytest_mcp_server-0.1.0/src/mcptest/lint.py +214 -0
- pytest_mcp_server-0.1.0/src/mcptest/plugin.py +161 -0
- pytest_mcp_server-0.1.0/src/mcptest/snapshot.py +200 -0
- pytest_mcp_server-0.1.0/tests/conftest.py +88 -0
- pytest_mcp_server-0.1.0/tests/servers/fake_server.py +182 -0
- pytest_mcp_server-0.1.0/tests/test_cli.py +70 -0
- pytest_mcp_server-0.1.0/tests/test_client.py +112 -0
- pytest_mcp_server-0.1.0/tests/test_lint.py +81 -0
- pytest_mcp_server-0.1.0/tests/test_plugin.py +93 -0
- pytest_mcp_server-0.1.0/tests/test_sdk_interop.py +88 -0
- pytest_mcp_server-0.1.0/tests/test_snapshot.py +88 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 (2026-11-06)
|
|
4
|
+
|
|
5
|
+
First release. Published on PyPI as `pytest-mcp-server` (the name `mcptest` was refused as too
|
|
6
|
+
similar to `mcp-test`); the command and import are `mcptest`.
|
|
7
|
+
|
|
8
|
+
- Dependency-free MCP client for stdio and Streamable HTTP: handshake, paginated tool,
|
|
9
|
+
resource and prompt listing, tool calls, server ping replies, and clear errors that include
|
|
10
|
+
the server's stderr and any non-protocol output on stdout.
|
|
11
|
+
- Linter with 15 rules for tool names, descriptions and schemas, including tool-poisoning
|
|
12
|
+
phrases and hidden characters.
|
|
13
|
+
- Snapshot format and diff that classifies each change as breaking or compatible.
|
|
14
|
+
- pytest plugin: `mcp`, `mcp_server` and `mcp_snapshot` fixtures, `mcp_server` marker,
|
|
15
|
+
`mcp_command` / `mcp_url` settings, `--mcp-update-snapshots`.
|
|
16
|
+
- CLI: `mcptest lint`, `tools`, `call`, `snapshot`, `diff`.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sri Krishna Lingamaneni
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pytest-mcp-server
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Automated tests, linting and breaking-change checks for MCP servers, as a pytest plugin and CLI
|
|
5
|
+
Project-URL: Homepage, https://github.com/srikrishnaL/kitforge/tree/main/packages/mcptest
|
|
6
|
+
Project-URL: Source, https://github.com/srikrishnaL/kitforge
|
|
7
|
+
Project-URL: Issues, https://github.com/srikrishnaL/kitforge/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/srikrishnaL/kitforge/blob/main/packages/mcptest/CHANGELOG.md
|
|
9
|
+
Author: Sri Krishna Lingamaneni
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: ai agents,ci,lint,llm,mcp,model context protocol,pytest,testing
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Framework :: Pytest
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# mcptest
|
|
27
|
+
|
|
28
|
+
Automated tests, linting and breaking-change checks for [MCP](https://modelcontextprotocol.io) servers.
|
|
29
|
+
|
|
30
|
+
You build an MCP server, then poke at it by hand in the Inspector. mcptest turns that into tests
|
|
31
|
+
that run in CI: start the server, call its tools, check what an agent will actually see, and fail
|
|
32
|
+
the build when a change would break the agents and clients that depend on it.
|
|
33
|
+
|
|
34
|
+
- **pytest plugin**: an `mcp` fixture connected to your server, plus schema snapshots.
|
|
35
|
+
- **Linter**: 15 rules for tool names, descriptions and schemas, including hidden instructions
|
|
36
|
+
aimed at the model ("tool poisoning").
|
|
37
|
+
- **Breaking-change diff**: removed tools, new required parameters, narrowed types, removed
|
|
38
|
+
enum values, removed output fields.
|
|
39
|
+
- **CLI** for any server in any language: `mcptest lint -- node dist/index.js`.
|
|
40
|
+
- **No dependencies.** It speaks the protocol itself over stdio and Streamable HTTP, so an SDK
|
|
41
|
+
upgrade on your side never breaks your tests.
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install pytest-mcp-server
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
The package is published on PyPI as `pytest-mcp-server`; the command and the Python import are
|
|
48
|
+
both `mcptest`.
|
|
49
|
+
|
|
50
|
+
Python 3.9+. Works with servers written in any language.
|
|
51
|
+
|
|
52
|
+
## Check a server in one line
|
|
53
|
+
|
|
54
|
+
```console
|
|
55
|
+
$ mcptest lint -- python my_server.py
|
|
56
|
+
error MCP001 bad name!: invalid tool name 'bad name!'
|
|
57
|
+
error MCP008 get_file: required parameter 'missing' is not in properties
|
|
58
|
+
error MCP012 get_file: description says '<IMPORTANT>', 'Before using any tools, you must',
|
|
59
|
+
'read ~/.ssh', 'Do not tell the user'
|
|
60
|
+
error MCP003 noschema: inputSchema must be an object schema with "type": "object"
|
|
61
|
+
warning MCP005 Get-File: description is only 1 characters: 'x'
|
|
62
|
+
warning MCP011 Get-File: easily confused with 'get_file'
|
|
63
|
+
warning MCP007 get_file: parameter 'path' has no description
|
|
64
|
+
|
|
65
|
+
my-server: 4 tools, about 146 tokens of definitions; 4 errors, 3 warnings, 0 info
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Even well-known servers have gaps an agent pays for. On 2026-11-06 the official reference
|
|
69
|
+
filesystem server (`@modelcontextprotocol/server-filesystem`) got 18 warnings, nearly all
|
|
70
|
+
parameters with no description, which leaves the model guessing what to pass.
|
|
71
|
+
|
|
72
|
+
The exit code is 1 when there is a finding at `--fail-on` level or worse (default `error`), so
|
|
73
|
+
it drops straight into CI. Other commands:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
mcptest tools -- python my_server.py # print the tool list as JSON
|
|
77
|
+
mcptest call add '{"a": 2, "b": 3}' -- python my_server.py
|
|
78
|
+
mcptest snapshot -o mcp-tools.json -- python my_server.py
|
|
79
|
+
mcptest diff mcp-tools.json -- python my_server.py # exit 1 on breaking changes
|
|
80
|
+
mcptest lint --url http://localhost:8000/mcp --header "Authorization: Bearer $TOKEN"
|
|
81
|
+
mcptest lint --rules # list every rule
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Anything after `--` is the command that starts your server over stdio: `npx -y @acme/server`,
|
|
85
|
+
`uv run server.py`, `docker run -i --rm acme/mcp`, and so on.
|
|
86
|
+
|
|
87
|
+
## Test a server with pytest
|
|
88
|
+
|
|
89
|
+
Tell pytest how to start the server once:
|
|
90
|
+
|
|
91
|
+
```toml
|
|
92
|
+
# pyproject.toml
|
|
93
|
+
[tool.pytest.ini_options]
|
|
94
|
+
mcp_command = "python -m my_server"
|
|
95
|
+
# or, for a server that is already running:
|
|
96
|
+
# mcp_url = "http://localhost:8000/mcp"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Then write ordinary tests. The `mcp` fixture is a connected client, and the server is stopped
|
|
100
|
+
when the test ends:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from mcptest import assert_lint_clean
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_add(mcp):
|
|
107
|
+
result = mcp.call_tool("add", {"a": 2, "b": 3})
|
|
108
|
+
assert not result.is_error
|
|
109
|
+
assert result.text == "5"
|
|
110
|
+
assert result.structured == {"result": 5}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_bad_input_is_a_tool_error(mcp):
|
|
114
|
+
assert mcp.call_tool("add", {"a": "two"}).is_error
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_tools_are_well_described(mcp):
|
|
118
|
+
assert_lint_clean(mcp) # fails on warnings and errors
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_no_surprise_schema_changes(mcp, mcp_snapshot):
|
|
122
|
+
mcp_snapshot(mcp)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
A test can pick a different server with a marker, and `mcp_server` starts as many as you need:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
import pytest
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@pytest.mark.mcp_server(command=["node", "dist/index.js"], env={"API_KEY": "test"})
|
|
132
|
+
def test_node_server(mcp):
|
|
133
|
+
assert "search" in [t["name"] for t in mcp.list_tools()]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_two_versions(mcp_server):
|
|
137
|
+
old = mcp_server(["python", "server_v1.py"])
|
|
138
|
+
new = mcp_server(url="http://localhost:9000/mcp")
|
|
139
|
+
...
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Schema snapshots
|
|
143
|
+
|
|
144
|
+
`mcp_snapshot(mcp)` stores the tool list in `__mcp_snapshots__/<test file>/tools.json` on the
|
|
145
|
+
first run and compares against it afterwards. Commit the file. When the tools change, the test
|
|
146
|
+
fails and says which changes are breaking:
|
|
147
|
+
|
|
148
|
+
```text
|
|
149
|
+
MCP tools differ from snapshot tools.json (3 breaking, 4 compatible):
|
|
150
|
+
BREAKING add: new required parameter 'c'
|
|
151
|
+
BREAKING add: outputSchema was removed
|
|
152
|
+
BREAKING echo: tool was removed
|
|
153
|
+
ok add: parameter 'a' type changed integer -> number
|
|
154
|
+
...
|
|
155
|
+
Run pytest --mcp-update-snapshots to accept these changes.
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Pass `allow_compatible=True` to fail only on breaking changes.
|
|
159
|
+
|
|
160
|
+
## Lint rules
|
|
161
|
+
|
|
162
|
+
| Code | Level | What it catches |
|
|
163
|
+
| --- | --- | --- |
|
|
164
|
+
| MCP001 | error | Tool name is not 1-128 characters of `A-Z a-z 0-9 _ - .` |
|
|
165
|
+
| MCP002 | error | Two tools share a name |
|
|
166
|
+
| MCP003 | error | `inputSchema` missing or not `"type": "object"` |
|
|
167
|
+
| MCP004 | warning | No description |
|
|
168
|
+
| MCP005 | warning | Description under 20 characters |
|
|
169
|
+
| MCP006 | warning | Description over 1,024 characters |
|
|
170
|
+
| MCP007 | warning | Parameter has no description |
|
|
171
|
+
| MCP008 | error | `required` names a parameter that isn't defined |
|
|
172
|
+
| MCP009 | warning | Parameter has no type |
|
|
173
|
+
| MCP010 | warning | All definitions together exceed 32,000 characters (about 8k tokens per turn) |
|
|
174
|
+
| MCP011 | warning | Two names differ only by case or separators (`get_file`, `Get-File`) |
|
|
175
|
+
| MCP012 | error | Text that tries to instruct the model: "ignore previous instructions", "do not tell the user", `<IMPORTANT>` blocks, reading `~/.ssh` |
|
|
176
|
+
| MCP013 | error | Invisible or bidirectional control characters |
|
|
177
|
+
| MCP014 | error | `outputSchema` is not `"type": "object"` |
|
|
178
|
+
| MCP015 | info | Parameter description just repeats its name |
|
|
179
|
+
|
|
180
|
+
Skip rules with `--ignore MCP005,MCP015` or `assert_lint_clean(mcp, ignore=["MCP005"])`.
|
|
181
|
+
MCP012 is a heuristic: it catches common tool-poisoning phrasing, not every attack.
|
|
182
|
+
|
|
183
|
+
## Use the client directly
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
from mcptest import MCPClient
|
|
187
|
+
|
|
188
|
+
with MCPClient.stdio(["python", "server.py"], env={"DEBUG": "1"}, timeout=10) as client:
|
|
189
|
+
print(client.server_info, client.protocol_version)
|
|
190
|
+
for tool in client.list_tools(): # follows pagination
|
|
191
|
+
print(tool["name"])
|
|
192
|
+
result = client.call_tool("search", {"q": "mcp"})
|
|
193
|
+
print(result.text, result.is_error)
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
When a server dies, mcptest shows its last stderr lines, and it points out anything printed to
|
|
197
|
+
stdout that isn't protocol (the most common stdio bug):
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
MCPTransportError: server closed stdout (exit code 3)
|
|
201
|
+
server stderr (last lines):
|
|
202
|
+
fatal: database not configured
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
## In CI
|
|
206
|
+
|
|
207
|
+
```yaml
|
|
208
|
+
- run: pip install pytest-mcp-server
|
|
209
|
+
- run: mcptest lint --fail-on warning -- python -m my_server
|
|
210
|
+
- run: mcptest diff mcp-tools.json -- python -m my_server
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
## Tested against real servers
|
|
214
|
+
|
|
215
|
+
Checked on 2026-11-06 with mcptest 0.1.0: connect, list tools, call a tool, snapshot twice.
|
|
216
|
+
|
|
217
|
+
| Server | Transport | Tools | Tool call | Lint findings |
|
|
218
|
+
| --- | --- | --- | --- | --- |
|
|
219
|
+
| @modelcontextprotocol/server-everything | stdio, Streamable HTTP | 14 | ok | 1 warning |
|
|
220
|
+
| @modelcontextprotocol/server-filesystem | stdio | 14 | ok | 18 warnings |
|
|
221
|
+
| @modelcontextprotocol/server-memory | stdio | 9 | ok | 4 warnings |
|
|
222
|
+
| @modelcontextprotocol/server-sequential-thinking | stdio | 1 | ok | 1 warning |
|
|
223
|
+
| @playwright/mcp | stdio, Streamable HTTP | 25 | returned as a tool error (no browser in the test machine) | 3 warnings |
|
|
224
|
+
| mcp-server-time | stdio | 3 | ok | none |
|
|
225
|
+
| mcp-server-git | stdio | 14 | ok | 24 warnings |
|
|
226
|
+
| mcp-server-fetch | stdio | 1 | not called (needs network) | none |
|
|
227
|
+
| Official Python SDK 2.3 (`MCPServer`) | stdio, Streamable HTTP | 2 | ok | 3 warnings |
|
|
228
|
+
|
|
229
|
+
Almost all warnings are parameters with no description (MCP007) or very short descriptions (MCP005).
|
|
230
|
+
|
|
231
|
+
## Limits
|
|
232
|
+
|
|
233
|
+
- Covers tools fully, resources and prompts through the client only (no lint rules for them yet).
|
|
234
|
+
- No OAuth flow: for protected HTTP servers, pass a token with `--header` or `headers=`.
|
|
235
|
+
- The legacy HTTP+SSE transport (two endpoints) is not supported; Streamable HTTP and stdio are.
|
|
236
|
+
|
|
237
|
+
## License
|
|
238
|
+
|
|
239
|
+
MIT
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# mcptest
|
|
2
|
+
|
|
3
|
+
Automated tests, linting and breaking-change checks for [MCP](https://modelcontextprotocol.io) servers.
|
|
4
|
+
|
|
5
|
+
You build an MCP server, then poke at it by hand in the Inspector. mcptest turns that into tests
|
|
6
|
+
that run in CI: start the server, call its tools, check what an agent will actually see, and fail
|
|
7
|
+
the build when a change would break the agents and clients that depend on it.
|
|
8
|
+
|
|
9
|
+
- **pytest plugin**: an `mcp` fixture connected to your server, plus schema snapshots.
|
|
10
|
+
- **Linter**: 15 rules for tool names, descriptions and schemas, including hidden instructions
|
|
11
|
+
aimed at the model ("tool poisoning").
|
|
12
|
+
- **Breaking-change diff**: removed tools, new required parameters, narrowed types, removed
|
|
13
|
+
enum values, removed output fields.
|
|
14
|
+
- **CLI** for any server in any language: `mcptest lint -- node dist/index.js`.
|
|
15
|
+
- **No dependencies.** It speaks the protocol itself over stdio and Streamable HTTP, so an SDK
|
|
16
|
+
upgrade on your side never breaks your tests.
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install pytest-mcp-server
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The package is published on PyPI as `pytest-mcp-server`; the command and the Python import are
|
|
23
|
+
both `mcptest`.
|
|
24
|
+
|
|
25
|
+
Python 3.9+. Works with servers written in any language.
|
|
26
|
+
|
|
27
|
+
## Check a server in one line
|
|
28
|
+
|
|
29
|
+
```console
|
|
30
|
+
$ mcptest lint -- python my_server.py
|
|
31
|
+
error MCP001 bad name!: invalid tool name 'bad name!'
|
|
32
|
+
error MCP008 get_file: required parameter 'missing' is not in properties
|
|
33
|
+
error MCP012 get_file: description says '<IMPORTANT>', 'Before using any tools, you must',
|
|
34
|
+
'read ~/.ssh', 'Do not tell the user'
|
|
35
|
+
error MCP003 noschema: inputSchema must be an object schema with "type": "object"
|
|
36
|
+
warning MCP005 Get-File: description is only 1 characters: 'x'
|
|
37
|
+
warning MCP011 Get-File: easily confused with 'get_file'
|
|
38
|
+
warning MCP007 get_file: parameter 'path' has no description
|
|
39
|
+
|
|
40
|
+
my-server: 4 tools, about 146 tokens of definitions; 4 errors, 3 warnings, 0 info
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Even well-known servers have gaps an agent pays for. On 2026-11-06 the official reference
|
|
44
|
+
filesystem server (`@modelcontextprotocol/server-filesystem`) got 18 warnings, nearly all
|
|
45
|
+
parameters with no description, which leaves the model guessing what to pass.
|
|
46
|
+
|
|
47
|
+
The exit code is 1 when there is a finding at `--fail-on` level or worse (default `error`), so
|
|
48
|
+
it drops straight into CI. Other commands:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
mcptest tools -- python my_server.py # print the tool list as JSON
|
|
52
|
+
mcptest call add '{"a": 2, "b": 3}' -- python my_server.py
|
|
53
|
+
mcptest snapshot -o mcp-tools.json -- python my_server.py
|
|
54
|
+
mcptest diff mcp-tools.json -- python my_server.py # exit 1 on breaking changes
|
|
55
|
+
mcptest lint --url http://localhost:8000/mcp --header "Authorization: Bearer $TOKEN"
|
|
56
|
+
mcptest lint --rules # list every rule
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Anything after `--` is the command that starts your server over stdio: `npx -y @acme/server`,
|
|
60
|
+
`uv run server.py`, `docker run -i --rm acme/mcp`, and so on.
|
|
61
|
+
|
|
62
|
+
## Test a server with pytest
|
|
63
|
+
|
|
64
|
+
Tell pytest how to start the server once:
|
|
65
|
+
|
|
66
|
+
```toml
|
|
67
|
+
# pyproject.toml
|
|
68
|
+
[tool.pytest.ini_options]
|
|
69
|
+
mcp_command = "python -m my_server"
|
|
70
|
+
# or, for a server that is already running:
|
|
71
|
+
# mcp_url = "http://localhost:8000/mcp"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Then write ordinary tests. The `mcp` fixture is a connected client, and the server is stopped
|
|
75
|
+
when the test ends:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from mcptest import assert_lint_clean
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_add(mcp):
|
|
82
|
+
result = mcp.call_tool("add", {"a": 2, "b": 3})
|
|
83
|
+
assert not result.is_error
|
|
84
|
+
assert result.text == "5"
|
|
85
|
+
assert result.structured == {"result": 5}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_bad_input_is_a_tool_error(mcp):
|
|
89
|
+
assert mcp.call_tool("add", {"a": "two"}).is_error
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_tools_are_well_described(mcp):
|
|
93
|
+
assert_lint_clean(mcp) # fails on warnings and errors
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_no_surprise_schema_changes(mcp, mcp_snapshot):
|
|
97
|
+
mcp_snapshot(mcp)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
A test can pick a different server with a marker, and `mcp_server` starts as many as you need:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
import pytest
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@pytest.mark.mcp_server(command=["node", "dist/index.js"], env={"API_KEY": "test"})
|
|
107
|
+
def test_node_server(mcp):
|
|
108
|
+
assert "search" in [t["name"] for t in mcp.list_tools()]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_two_versions(mcp_server):
|
|
112
|
+
old = mcp_server(["python", "server_v1.py"])
|
|
113
|
+
new = mcp_server(url="http://localhost:9000/mcp")
|
|
114
|
+
...
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## Schema snapshots
|
|
118
|
+
|
|
119
|
+
`mcp_snapshot(mcp)` stores the tool list in `__mcp_snapshots__/<test file>/tools.json` on the
|
|
120
|
+
first run and compares against it afterwards. Commit the file. When the tools change, the test
|
|
121
|
+
fails and says which changes are breaking:
|
|
122
|
+
|
|
123
|
+
```text
|
|
124
|
+
MCP tools differ from snapshot tools.json (3 breaking, 4 compatible):
|
|
125
|
+
BREAKING add: new required parameter 'c'
|
|
126
|
+
BREAKING add: outputSchema was removed
|
|
127
|
+
BREAKING echo: tool was removed
|
|
128
|
+
ok add: parameter 'a' type changed integer -> number
|
|
129
|
+
...
|
|
130
|
+
Run pytest --mcp-update-snapshots to accept these changes.
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Pass `allow_compatible=True` to fail only on breaking changes.
|
|
134
|
+
|
|
135
|
+
## Lint rules
|
|
136
|
+
|
|
137
|
+
| Code | Level | What it catches |
|
|
138
|
+
| --- | --- | --- |
|
|
139
|
+
| MCP001 | error | Tool name is not 1-128 characters of `A-Z a-z 0-9 _ - .` |
|
|
140
|
+
| MCP002 | error | Two tools share a name |
|
|
141
|
+
| MCP003 | error | `inputSchema` missing or not `"type": "object"` |
|
|
142
|
+
| MCP004 | warning | No description |
|
|
143
|
+
| MCP005 | warning | Description under 20 characters |
|
|
144
|
+
| MCP006 | warning | Description over 1,024 characters |
|
|
145
|
+
| MCP007 | warning | Parameter has no description |
|
|
146
|
+
| MCP008 | error | `required` names a parameter that isn't defined |
|
|
147
|
+
| MCP009 | warning | Parameter has no type |
|
|
148
|
+
| MCP010 | warning | All definitions together exceed 32,000 characters (about 8k tokens per turn) |
|
|
149
|
+
| MCP011 | warning | Two names differ only by case or separators (`get_file`, `Get-File`) |
|
|
150
|
+
| MCP012 | error | Text that tries to instruct the model: "ignore previous instructions", "do not tell the user", `<IMPORTANT>` blocks, reading `~/.ssh` |
|
|
151
|
+
| MCP013 | error | Invisible or bidirectional control characters |
|
|
152
|
+
| MCP014 | error | `outputSchema` is not `"type": "object"` |
|
|
153
|
+
| MCP015 | info | Parameter description just repeats its name |
|
|
154
|
+
|
|
155
|
+
Skip rules with `--ignore MCP005,MCP015` or `assert_lint_clean(mcp, ignore=["MCP005"])`.
|
|
156
|
+
MCP012 is a heuristic: it catches common tool-poisoning phrasing, not every attack.
|
|
157
|
+
|
|
158
|
+
## Use the client directly
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
from mcptest import MCPClient
|
|
162
|
+
|
|
163
|
+
with MCPClient.stdio(["python", "server.py"], env={"DEBUG": "1"}, timeout=10) as client:
|
|
164
|
+
print(client.server_info, client.protocol_version)
|
|
165
|
+
for tool in client.list_tools(): # follows pagination
|
|
166
|
+
print(tool["name"])
|
|
167
|
+
result = client.call_tool("search", {"q": "mcp"})
|
|
168
|
+
print(result.text, result.is_error)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
When a server dies, mcptest shows its last stderr lines, and it points out anything printed to
|
|
172
|
+
stdout that isn't protocol (the most common stdio bug):
|
|
173
|
+
|
|
174
|
+
```text
|
|
175
|
+
MCPTransportError: server closed stdout (exit code 3)
|
|
176
|
+
server stderr (last lines):
|
|
177
|
+
fatal: database not configured
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## In CI
|
|
181
|
+
|
|
182
|
+
```yaml
|
|
183
|
+
- run: pip install pytest-mcp-server
|
|
184
|
+
- run: mcptest lint --fail-on warning -- python -m my_server
|
|
185
|
+
- run: mcptest diff mcp-tools.json -- python -m my_server
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
## Tested against real servers
|
|
189
|
+
|
|
190
|
+
Checked on 2026-11-06 with mcptest 0.1.0: connect, list tools, call a tool, snapshot twice.
|
|
191
|
+
|
|
192
|
+
| Server | Transport | Tools | Tool call | Lint findings |
|
|
193
|
+
| --- | --- | --- | --- | --- |
|
|
194
|
+
| @modelcontextprotocol/server-everything | stdio, Streamable HTTP | 14 | ok | 1 warning |
|
|
195
|
+
| @modelcontextprotocol/server-filesystem | stdio | 14 | ok | 18 warnings |
|
|
196
|
+
| @modelcontextprotocol/server-memory | stdio | 9 | ok | 4 warnings |
|
|
197
|
+
| @modelcontextprotocol/server-sequential-thinking | stdio | 1 | ok | 1 warning |
|
|
198
|
+
| @playwright/mcp | stdio, Streamable HTTP | 25 | returned as a tool error (no browser in the test machine) | 3 warnings |
|
|
199
|
+
| mcp-server-time | stdio | 3 | ok | none |
|
|
200
|
+
| mcp-server-git | stdio | 14 | ok | 24 warnings |
|
|
201
|
+
| mcp-server-fetch | stdio | 1 | not called (needs network) | none |
|
|
202
|
+
| Official Python SDK 2.3 (`MCPServer`) | stdio, Streamable HTTP | 2 | ok | 3 warnings |
|
|
203
|
+
|
|
204
|
+
Almost all warnings are parameters with no description (MCP007) or very short descriptions (MCP005).
|
|
205
|
+
|
|
206
|
+
## Limits
|
|
207
|
+
|
|
208
|
+
- Covers tools fully, resources and prompts through the client only (no lint rules for them yet).
|
|
209
|
+
- No OAuth flow: for protected HTTP servers, pass a token with `--header` or `headers=`.
|
|
210
|
+
- The legacy HTTP+SSE transport (two endpoints) is not supported; Streamable HTTP and stdio are.
|
|
211
|
+
|
|
212
|
+
## License
|
|
213
|
+
|
|
214
|
+
MIT
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pytest-mcp-server"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Automated tests, linting and breaking-change checks for MCP servers, as a pytest plugin and CLI"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
authors = [{ name = "Sri Krishna Lingamaneni" }]
|
|
14
|
+
keywords = ["mcp", "model context protocol", "pytest", "testing", "lint", "ai agents", "llm", "ci"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Framework :: Pytest",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Topic :: Software Development :: Testing",
|
|
23
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
24
|
+
]
|
|
25
|
+
dependencies = []
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
test = ["pytest>=7"]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/srikrishnaL/kitforge/tree/main/packages/mcptest"
|
|
32
|
+
Source = "https://github.com/srikrishnaL/kitforge"
|
|
33
|
+
Issues = "https://github.com/srikrishnaL/kitforge/issues"
|
|
34
|
+
Changelog = "https://github.com/srikrishnaL/kitforge/blob/main/packages/mcptest/CHANGELOG.md"
|
|
35
|
+
|
|
36
|
+
[project.scripts]
|
|
37
|
+
mcptest = "mcptest.cli:main"
|
|
38
|
+
|
|
39
|
+
[project.entry-points.pytest11]
|
|
40
|
+
mcptest = "mcptest.plugin"
|
|
41
|
+
|
|
42
|
+
[tool.hatch.build.targets.wheel]
|
|
43
|
+
packages = ["src/mcptest"]
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""mcptest: automated tests and CI checks for MCP servers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Iterable
|
|
6
|
+
|
|
7
|
+
from .client import (
|
|
8
|
+
MCPClient,
|
|
9
|
+
MCPError,
|
|
10
|
+
MCPTimeout,
|
|
11
|
+
MCPTransportError,
|
|
12
|
+
ToolResult,
|
|
13
|
+
)
|
|
14
|
+
from .lint import RULES, Finding, at_or_above, lint_tools
|
|
15
|
+
from .snapshot import Change, diff, normalize
|
|
16
|
+
|
|
17
|
+
__version__ = "0.1.0"
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"MCPClient",
|
|
21
|
+
"MCPError",
|
|
22
|
+
"MCPTimeout",
|
|
23
|
+
"MCPTransportError",
|
|
24
|
+
"ToolResult",
|
|
25
|
+
"Finding",
|
|
26
|
+
"RULES",
|
|
27
|
+
"lint_tools",
|
|
28
|
+
"assert_lint_clean",
|
|
29
|
+
"Change",
|
|
30
|
+
"diff",
|
|
31
|
+
"normalize",
|
|
32
|
+
"__version__",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def assert_lint_clean(source: Any, *, fail_on: str = "warning", ignore: Iterable[str] = ()) -> None:
|
|
37
|
+
"""Raise ``AssertionError`` if the tools have findings at ``fail_on`` or worse.
|
|
38
|
+
|
|
39
|
+
``source`` is an :class:`MCPClient` or a ``tools/list`` result.
|
|
40
|
+
"""
|
|
41
|
+
tools = source.list_tools() if isinstance(source, MCPClient) else source
|
|
42
|
+
findings = at_or_above(lint_tools(tools, ignore=ignore), fail_on)
|
|
43
|
+
if findings:
|
|
44
|
+
body = "\n ".join(str(f) for f in findings)
|
|
45
|
+
raise AssertionError(f"{len(findings)} MCP lint finding(s):\n {body}")
|