audit-mcp 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- audit_mcp-0.5.0/LICENSE +21 -0
- audit_mcp-0.5.0/PKG-INFO +140 -0
- audit_mcp-0.5.0/README.md +111 -0
- audit_mcp-0.5.0/audit_mcp.egg-info/PKG-INFO +140 -0
- audit_mcp-0.5.0/audit_mcp.egg-info/SOURCES.txt +32 -0
- audit_mcp-0.5.0/audit_mcp.egg-info/dependency_links.txt +1 -0
- audit_mcp-0.5.0/audit_mcp.egg-info/entry_points.txt +3 -0
- audit_mcp-0.5.0/audit_mcp.egg-info/requires.txt +6 -0
- audit_mcp-0.5.0/audit_mcp.egg-info/top_level.txt +1 -0
- audit_mcp-0.5.0/checkmcp/__init__.py +2 -0
- audit_mcp-0.5.0/checkmcp/badge.py +38 -0
- audit_mcp-0.5.0/checkmcp/calibrate.py +198 -0
- audit_mcp-0.5.0/checkmcp/cli.py +215 -0
- audit_mcp-0.5.0/checkmcp/evals.py +200 -0
- audit_mcp-0.5.0/checkmcp/gateway.py +159 -0
- audit_mcp-0.5.0/checkmcp/mcp_server.py +202 -0
- audit_mcp-0.5.0/checkmcp/monitor.py +59 -0
- audit_mcp-0.5.0/checkmcp/optimize.py +123 -0
- audit_mcp-0.5.0/checkmcp/page.py +84 -0
- audit_mcp-0.5.0/checkmcp/plans.py +11 -0
- audit_mcp-0.5.0/checkmcp/probe.py +383 -0
- audit_mcp-0.5.0/checkmcp/repo.py +147 -0
- audit_mcp-0.5.0/checkmcp/scanners.py +92 -0
- audit_mcp-0.5.0/checkmcp/score.py +178 -0
- audit_mcp-0.5.0/checkmcp/security.py +86 -0
- audit_mcp-0.5.0/checkmcp/store.py +298 -0
- audit_mcp-0.5.0/pyproject.toml +45 -0
- audit_mcp-0.5.0/setup.cfg +4 -0
- audit_mcp-0.5.0/tests/test_calibrate.py +45 -0
- audit_mcp-0.5.0/tests/test_evals.py +68 -0
- audit_mcp-0.5.0/tests/test_mcp_server.py +148 -0
- audit_mcp-0.5.0/tests/test_misc.py +12 -0
- audit_mcp-0.5.0/tests/test_monitor.py +45 -0
- audit_mcp-0.5.0/tests/test_score.py +54 -0
audit_mcp-0.5.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 CheckMCP
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
audit_mcp-0.5.0/PKG-INFO
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: audit-mcp
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: CheckMCP — vendor-neutral quality/security/context-cost audit & score for any MCP server.
|
|
5
|
+
Author: Hugo
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://checkmcp.dev
|
|
8
|
+
Project-URL: Repository, https://github.com/H129hj/checkmcp
|
|
9
|
+
Keywords: mcp,model-context-protocol,audit,score,lint,security,gateway,guardrail
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Security
|
|
19
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: exact-tokens
|
|
25
|
+
Requires-Dist: tiktoken>=0.7; extra == "exact-tokens"
|
|
26
|
+
Provides-Extra: test
|
|
27
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
<!-- mcp-name: io.github.H129hj/checkmcp -->
|
|
31
|
+
|
|
32
|
+
# CheckMCP
|
|
33
|
+
|
|
34
|
+
[](https://pypi.org/project/audit-mcp/) [](LICENSE) [](action.yml) [](https://checkmcp.dev)
|
|
35
|
+
|
|
36
|
+
**Vendor-neutral quality / security / context-cost audit & score for any MCP server.**
|
|
37
|
+
One `uvx`/`pipx` command → an **MCP Score /100** + **causal opportunities** (why the score), Lighthouse-style.
|
|
38
|
+
|
|
39
|
+
> Installed from PyPI as **`audit-mcp`** (the name `checkmcp` was already taken); the command is `audit-mcp`. Brand, site and repo remain **CheckMCP** / checkmcp.dev.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
uvx audit-mcp https://mcp.deepwiki.com/mcp
|
|
43
|
+
# or
|
|
44
|
+
pipx run audit-mcp https://mcp.context7.com/mcp --json
|
|
45
|
+
audit-mcp https://my-mcp.example.com/mcp --token "$TOKEN"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Use it as an MCP server
|
|
49
|
+
|
|
50
|
+
`audit-mcp mcp` turns the auditor itself into an MCP server (stdio) exposing one tool,
|
|
51
|
+
`audit_mcp_server` — so your agent can answer *"is this MCP server safe?"* mid-conversation.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
# Claude Code
|
|
55
|
+
claude mcp add audit-mcp -- uvx audit-mcp mcp
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
```json
|
|
59
|
+
// Cursor (.cursor/mcp.json) / Claude Desktop (claude_desktop_config.json)
|
|
60
|
+
{ "mcpServers": { "audit-mcp": { "command": "uvx", "args": ["audit-mcp", "mcp"] } } }
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Registry name: `io.github.H129hj/checkmcp` <!-- mcp-name: io.github.H129hj/checkmcp -->
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
No dependencies (stdlib only). `tiktoken` optional for exact token counts.
|
|
67
|
+
|
|
68
|
+
## What it measures (7 pillars)
|
|
69
|
+
- **Security** — OWASP MCP Top 10 (tool poisoning, hardcoded secrets, command injection), lethal-trifecta.
|
|
70
|
+
- **Tool design** — sprawl/consolidation (percentile-calibrated on real servers: median ~7 tools, p95 ~42).
|
|
71
|
+
- **Schemas / desc** — descriptions + `inputSchema`/`outputSchema` completeness.
|
|
72
|
+
- **Context-cost** — tokens spent on `tools/list`, paid on every request (the #1 pain of 2026).
|
|
73
|
+
- **Compliance** — protocol-version gap, annotations, JSON-RPC error conformance, OAuth discovery.
|
|
74
|
+
- **Reliability** — single-shot today (not credited; continuous T3 monitoring on checkmcp.dev).
|
|
75
|
+
- **Coverage** — the 3 primitives (tools **+ resources + prompts**).
|
|
76
|
+
|
|
77
|
+
Hard floors: secret-in-schema → cap D, failed handshake → cap F. Every penalty is attributed: `measure → mechanism → effect → Δscore`.
|
|
78
|
+
|
|
79
|
+
## CLI flags
|
|
80
|
+
| flag | what |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `--json` | machine-readable report |
|
|
83
|
+
| `--badge` | SVG badge + README embed snippets |
|
|
84
|
+
| `--html` | standalone SEO/GEO page (JSON-LD `SoftwareApplication` + FAQ) |
|
|
85
|
+
| `--repo owner/name` | add maintenance/license/provenance signal from GitHub |
|
|
86
|
+
| `--token <bearer>` | audit an OAuth-protected server |
|
|
87
|
+
| `--min-score N` | CI: exit 1 if MCP Score < N |
|
|
88
|
+
| `--baseline file` | CI: pin tool definitions; fail on regression (rug-pull) |
|
|
89
|
+
| `--gh-summary` | CI: write a Markdown summary to `$GITHUB_STEP_SUMMARY` |
|
|
90
|
+
| `--deep` | runtime depth via an external scanner (mcp-scan/snyk) if present |
|
|
91
|
+
| `--evals` | behavioral sandbox: actually invokes read-only tools with canary inputs to catch tool-output prompt-injection, exfiltration vectors, secret/PII leakage and context bombs (sends real traffic; CI-fails on a malicious verdict) |
|
|
92
|
+
|
|
93
|
+
## GitHub Action
|
|
94
|
+
|
|
95
|
+
```yaml
|
|
96
|
+
# .github/workflows/mcp-audit.yml
|
|
97
|
+
name: MCP audit
|
|
98
|
+
on: [push, pull_request]
|
|
99
|
+
jobs:
|
|
100
|
+
checkmcp:
|
|
101
|
+
runs-on: ubuntu-latest
|
|
102
|
+
steps:
|
|
103
|
+
- uses: actions/checkout@v4
|
|
104
|
+
- uses: H129hj/checkmcp@v1
|
|
105
|
+
with:
|
|
106
|
+
url: https://my-mcp.example.com/mcp
|
|
107
|
+
min-score: "70"
|
|
108
|
+
baseline: .checkmcp-baseline.json # commit it → fails on rug-pull
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Behavioral evals (`--evals`)
|
|
112
|
+
Static analysis catches *declared* danger; `--evals` catches *runtime* danger by actually invoking
|
|
113
|
+
read-only tools with canary inputs and inspecting the **responses** for tool-output prompt-injection,
|
|
114
|
+
exfiltration vectors and secret/PII leakage (multilingual; optional callback-canary confirms exfil).
|
|
115
|
+
CI-fails on a malicious verdict.
|
|
116
|
+
|
|
117
|
+
## Self-hosted security gateway
|
|
118
|
+
Beyond auditing, CheckMCP ships an **in-band MCP gateway** — a proxy you put between your agent and an
|
|
119
|
+
MCP server. It inspects every call, and in *active* mode **blocks/strips** tool-poisoning & exfiltration
|
|
120
|
+
before they reach the agent. Run it in your own infra (tool traffic never leaves your network):
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
docker pull ghcr.io/h129hj/checkmcp-gateway:latest # or build from source
|
|
124
|
+
docker run -p 8080:8080 -e GATEWAY_BACKEND_URL=https://mcp.example.com/mcp \
|
|
125
|
+
-e GATEWAY_MODE=active -e GATEWAY_SECRET=$(openssl rand -hex 16) \
|
|
126
|
+
ghcr.io/h129hj/checkmcp-gateway:latest
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
See **[GATEWAY.md](GATEWAY.md)** for config (passive/active, OAuth backends, policy, logs).
|
|
130
|
+
|
|
131
|
+
## Hosted
|
|
132
|
+
Full reports, public directory, live badges, continuous drift monitoring, a governance policy API and a
|
|
133
|
+
hosted gateway at **[checkmcp.dev](https://checkmcp.dev)**.
|
|
134
|
+
|
|
135
|
+
## Honest limitations
|
|
136
|
+
- Percentile bands come from a growing corpus (one+ registries) — widening over time.
|
|
137
|
+
- Exact tokens with `pipx install "audit-mcp[exact-tokens]"` (cl100k_base); otherwise chars/4 approximation.
|
|
138
|
+
- Pillar weights are expert priors. `python -m checkmcp.calibrate samples.json` validates them against a labeled agent-success sample (per-pillar correlation + OLS-suggested weights + construct-validity R²) — supply real outcomes to close the loop.
|
|
139
|
+
|
|
140
|
+
MIT.
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
<!-- mcp-name: io.github.H129hj/checkmcp -->
|
|
2
|
+
|
|
3
|
+
# CheckMCP
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/audit-mcp/) [](LICENSE) [](action.yml) [](https://checkmcp.dev)
|
|
6
|
+
|
|
7
|
+
**Vendor-neutral quality / security / context-cost audit & score for any MCP server.**
|
|
8
|
+
One `uvx`/`pipx` command → an **MCP Score /100** + **causal opportunities** (why the score), Lighthouse-style.
|
|
9
|
+
|
|
10
|
+
> Installed from PyPI as **`audit-mcp`** (the name `checkmcp` was already taken); the command is `audit-mcp`. Brand, site and repo remain **CheckMCP** / checkmcp.dev.
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
uvx audit-mcp https://mcp.deepwiki.com/mcp
|
|
14
|
+
# or
|
|
15
|
+
pipx run audit-mcp https://mcp.context7.com/mcp --json
|
|
16
|
+
audit-mcp https://my-mcp.example.com/mcp --token "$TOKEN"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Use it as an MCP server
|
|
20
|
+
|
|
21
|
+
`audit-mcp mcp` turns the auditor itself into an MCP server (stdio) exposing one tool,
|
|
22
|
+
`audit_mcp_server` — so your agent can answer *"is this MCP server safe?"* mid-conversation.
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
# Claude Code
|
|
26
|
+
claude mcp add audit-mcp -- uvx audit-mcp mcp
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```json
|
|
30
|
+
// Cursor (.cursor/mcp.json) / Claude Desktop (claude_desktop_config.json)
|
|
31
|
+
{ "mcpServers": { "audit-mcp": { "command": "uvx", "args": ["audit-mcp", "mcp"] } } }
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Registry name: `io.github.H129hj/checkmcp` <!-- mcp-name: io.github.H129hj/checkmcp -->
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
No dependencies (stdlib only). `tiktoken` optional for exact token counts.
|
|
38
|
+
|
|
39
|
+
## What it measures (7 pillars)
|
|
40
|
+
- **Security** — OWASP MCP Top 10 (tool poisoning, hardcoded secrets, command injection), lethal-trifecta.
|
|
41
|
+
- **Tool design** — sprawl/consolidation (percentile-calibrated on real servers: median ~7 tools, p95 ~42).
|
|
42
|
+
- **Schemas / desc** — descriptions + `inputSchema`/`outputSchema` completeness.
|
|
43
|
+
- **Context-cost** — tokens spent on `tools/list`, paid on every request (the #1 pain of 2026).
|
|
44
|
+
- **Compliance** — protocol-version gap, annotations, JSON-RPC error conformance, OAuth discovery.
|
|
45
|
+
- **Reliability** — single-shot today (not credited; continuous T3 monitoring on checkmcp.dev).
|
|
46
|
+
- **Coverage** — the 3 primitives (tools **+ resources + prompts**).
|
|
47
|
+
|
|
48
|
+
Hard floors: secret-in-schema → cap D, failed handshake → cap F. Every penalty is attributed: `measure → mechanism → effect → Δscore`.
|
|
49
|
+
|
|
50
|
+
## CLI flags
|
|
51
|
+
| flag | what |
|
|
52
|
+
|---|---|
|
|
53
|
+
| `--json` | machine-readable report |
|
|
54
|
+
| `--badge` | SVG badge + README embed snippets |
|
|
55
|
+
| `--html` | standalone SEO/GEO page (JSON-LD `SoftwareApplication` + FAQ) |
|
|
56
|
+
| `--repo owner/name` | add maintenance/license/provenance signal from GitHub |
|
|
57
|
+
| `--token <bearer>` | audit an OAuth-protected server |
|
|
58
|
+
| `--min-score N` | CI: exit 1 if MCP Score < N |
|
|
59
|
+
| `--baseline file` | CI: pin tool definitions; fail on regression (rug-pull) |
|
|
60
|
+
| `--gh-summary` | CI: write a Markdown summary to `$GITHUB_STEP_SUMMARY` |
|
|
61
|
+
| `--deep` | runtime depth via an external scanner (mcp-scan/snyk) if present |
|
|
62
|
+
| `--evals` | behavioral sandbox: actually invokes read-only tools with canary inputs to catch tool-output prompt-injection, exfiltration vectors, secret/PII leakage and context bombs (sends real traffic; CI-fails on a malicious verdict) |
|
|
63
|
+
|
|
64
|
+
## GitHub Action
|
|
65
|
+
|
|
66
|
+
```yaml
|
|
67
|
+
# .github/workflows/mcp-audit.yml
|
|
68
|
+
name: MCP audit
|
|
69
|
+
on: [push, pull_request]
|
|
70
|
+
jobs:
|
|
71
|
+
checkmcp:
|
|
72
|
+
runs-on: ubuntu-latest
|
|
73
|
+
steps:
|
|
74
|
+
- uses: actions/checkout@v4
|
|
75
|
+
- uses: H129hj/checkmcp@v1
|
|
76
|
+
with:
|
|
77
|
+
url: https://my-mcp.example.com/mcp
|
|
78
|
+
min-score: "70"
|
|
79
|
+
baseline: .checkmcp-baseline.json # commit it → fails on rug-pull
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Behavioral evals (`--evals`)
|
|
83
|
+
Static analysis catches *declared* danger; `--evals` catches *runtime* danger by actually invoking
|
|
84
|
+
read-only tools with canary inputs and inspecting the **responses** for tool-output prompt-injection,
|
|
85
|
+
exfiltration vectors and secret/PII leakage (multilingual; optional callback-canary confirms exfil).
|
|
86
|
+
CI-fails on a malicious verdict.
|
|
87
|
+
|
|
88
|
+
## Self-hosted security gateway
|
|
89
|
+
Beyond auditing, CheckMCP ships an **in-band MCP gateway** — a proxy you put between your agent and an
|
|
90
|
+
MCP server. It inspects every call, and in *active* mode **blocks/strips** tool-poisoning & exfiltration
|
|
91
|
+
before they reach the agent. Run it in your own infra (tool traffic never leaves your network):
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
docker pull ghcr.io/h129hj/checkmcp-gateway:latest # or build from source
|
|
95
|
+
docker run -p 8080:8080 -e GATEWAY_BACKEND_URL=https://mcp.example.com/mcp \
|
|
96
|
+
-e GATEWAY_MODE=active -e GATEWAY_SECRET=$(openssl rand -hex 16) \
|
|
97
|
+
ghcr.io/h129hj/checkmcp-gateway:latest
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
See **[GATEWAY.md](GATEWAY.md)** for config (passive/active, OAuth backends, policy, logs).
|
|
101
|
+
|
|
102
|
+
## Hosted
|
|
103
|
+
Full reports, public directory, live badges, continuous drift monitoring, a governance policy API and a
|
|
104
|
+
hosted gateway at **[checkmcp.dev](https://checkmcp.dev)**.
|
|
105
|
+
|
|
106
|
+
## Honest limitations
|
|
107
|
+
- Percentile bands come from a growing corpus (one+ registries) — widening over time.
|
|
108
|
+
- Exact tokens with `pipx install "audit-mcp[exact-tokens]"` (cl100k_base); otherwise chars/4 approximation.
|
|
109
|
+
- Pillar weights are expert priors. `python -m checkmcp.calibrate samples.json` validates them against a labeled agent-success sample (per-pillar correlation + OLS-suggested weights + construct-validity R²) — supply real outcomes to close the loop.
|
|
110
|
+
|
|
111
|
+
MIT.
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: audit-mcp
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: CheckMCP — vendor-neutral quality/security/context-cost audit & score for any MCP server.
|
|
5
|
+
Author: Hugo
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://checkmcp.dev
|
|
8
|
+
Project-URL: Repository, https://github.com/H129hj/checkmcp
|
|
9
|
+
Keywords: mcp,model-context-protocol,audit,score,lint,security,gateway,guardrail
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Security
|
|
19
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: exact-tokens
|
|
25
|
+
Requires-Dist: tiktoken>=0.7; extra == "exact-tokens"
|
|
26
|
+
Provides-Extra: test
|
|
27
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
<!-- mcp-name: io.github.H129hj/checkmcp -->
|
|
31
|
+
|
|
32
|
+
# CheckMCP
|
|
33
|
+
|
|
34
|
+
[](https://pypi.org/project/audit-mcp/) [](LICENSE) [](action.yml) [](https://checkmcp.dev)
|
|
35
|
+
|
|
36
|
+
**Vendor-neutral quality / security / context-cost audit & score for any MCP server.**
|
|
37
|
+
One `uvx`/`pipx` command → an **MCP Score /100** + **causal opportunities** (why the score), Lighthouse-style.
|
|
38
|
+
|
|
39
|
+
> Installed from PyPI as **`audit-mcp`** (the name `checkmcp` was already taken); the command is `audit-mcp`. Brand, site and repo remain **CheckMCP** / checkmcp.dev.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
uvx audit-mcp https://mcp.deepwiki.com/mcp
|
|
43
|
+
# or
|
|
44
|
+
pipx run audit-mcp https://mcp.context7.com/mcp --json
|
|
45
|
+
audit-mcp https://my-mcp.example.com/mcp --token "$TOKEN"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Use it as an MCP server
|
|
49
|
+
|
|
50
|
+
`audit-mcp mcp` turns the auditor itself into an MCP server (stdio) exposing one tool,
|
|
51
|
+
`audit_mcp_server` — so your agent can answer *"is this MCP server safe?"* mid-conversation.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
# Claude Code
|
|
55
|
+
claude mcp add audit-mcp -- uvx audit-mcp mcp
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
```json
|
|
59
|
+
// Cursor (.cursor/mcp.json) / Claude Desktop (claude_desktop_config.json)
|
|
60
|
+
{ "mcpServers": { "audit-mcp": { "command": "uvx", "args": ["audit-mcp", "mcp"] } } }
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Registry name: `io.github.H129hj/checkmcp` <!-- mcp-name: io.github.H129hj/checkmcp -->
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
No dependencies (stdlib only). `tiktoken` optional for exact token counts.
|
|
67
|
+
|
|
68
|
+
## What it measures (7 pillars)
|
|
69
|
+
- **Security** — OWASP MCP Top 10 (tool poisoning, hardcoded secrets, command injection), lethal-trifecta.
|
|
70
|
+
- **Tool design** — sprawl/consolidation (percentile-calibrated on real servers: median ~7 tools, p95 ~42).
|
|
71
|
+
- **Schemas / desc** — descriptions + `inputSchema`/`outputSchema` completeness.
|
|
72
|
+
- **Context-cost** — tokens spent on `tools/list`, paid on every request (the #1 pain of 2026).
|
|
73
|
+
- **Compliance** — protocol-version gap, annotations, JSON-RPC error conformance, OAuth discovery.
|
|
74
|
+
- **Reliability** — single-shot today (not credited; continuous T3 monitoring on checkmcp.dev).
|
|
75
|
+
- **Coverage** — the 3 primitives (tools **+ resources + prompts**).
|
|
76
|
+
|
|
77
|
+
Hard floors: secret-in-schema → cap D, failed handshake → cap F. Every penalty is attributed: `measure → mechanism → effect → Δscore`.
|
|
78
|
+
|
|
79
|
+
## CLI flags
|
|
80
|
+
| flag | what |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `--json` | machine-readable report |
|
|
83
|
+
| `--badge` | SVG badge + README embed snippets |
|
|
84
|
+
| `--html` | standalone SEO/GEO page (JSON-LD `SoftwareApplication` + FAQ) |
|
|
85
|
+
| `--repo owner/name` | add maintenance/license/provenance signal from GitHub |
|
|
86
|
+
| `--token <bearer>` | audit an OAuth-protected server |
|
|
87
|
+
| `--min-score N` | CI: exit 1 if MCP Score < N |
|
|
88
|
+
| `--baseline file` | CI: pin tool definitions; fail on regression (rug-pull) |
|
|
89
|
+
| `--gh-summary` | CI: write a Markdown summary to `$GITHUB_STEP_SUMMARY` |
|
|
90
|
+
| `--deep` | runtime depth via an external scanner (mcp-scan/snyk) if present |
|
|
91
|
+
| `--evals` | behavioral sandbox: actually invokes read-only tools with canary inputs to catch tool-output prompt-injection, exfiltration vectors, secret/PII leakage and context bombs (sends real traffic; CI-fails on a malicious verdict) |
|
|
92
|
+
|
|
93
|
+
## GitHub Action
|
|
94
|
+
|
|
95
|
+
```yaml
|
|
96
|
+
# .github/workflows/mcp-audit.yml
|
|
97
|
+
name: MCP audit
|
|
98
|
+
on: [push, pull_request]
|
|
99
|
+
jobs:
|
|
100
|
+
checkmcp:
|
|
101
|
+
runs-on: ubuntu-latest
|
|
102
|
+
steps:
|
|
103
|
+
- uses: actions/checkout@v4
|
|
104
|
+
- uses: H129hj/checkmcp@v1
|
|
105
|
+
with:
|
|
106
|
+
url: https://my-mcp.example.com/mcp
|
|
107
|
+
min-score: "70"
|
|
108
|
+
baseline: .checkmcp-baseline.json # commit it → fails on rug-pull
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Behavioral evals (`--evals`)
|
|
112
|
+
Static analysis catches *declared* danger; `--evals` catches *runtime* danger by actually invoking
|
|
113
|
+
read-only tools with canary inputs and inspecting the **responses** for tool-output prompt-injection,
|
|
114
|
+
exfiltration vectors and secret/PII leakage (multilingual; optional callback-canary confirms exfil).
|
|
115
|
+
CI-fails on a malicious verdict.
|
|
116
|
+
|
|
117
|
+
## Self-hosted security gateway
|
|
118
|
+
Beyond auditing, CheckMCP ships an **in-band MCP gateway** — a proxy you put between your agent and an
|
|
119
|
+
MCP server. It inspects every call, and in *active* mode **blocks/strips** tool-poisoning & exfiltration
|
|
120
|
+
before they reach the agent. Run it in your own infra (tool traffic never leaves your network):
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
docker pull ghcr.io/h129hj/checkmcp-gateway:latest # or build from source
|
|
124
|
+
docker run -p 8080:8080 -e GATEWAY_BACKEND_URL=https://mcp.example.com/mcp \
|
|
125
|
+
-e GATEWAY_MODE=active -e GATEWAY_SECRET=$(openssl rand -hex 16) \
|
|
126
|
+
ghcr.io/h129hj/checkmcp-gateway:latest
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
See **[GATEWAY.md](GATEWAY.md)** for config (passive/active, OAuth backends, policy, logs).
|
|
130
|
+
|
|
131
|
+
## Hosted
|
|
132
|
+
Full reports, public directory, live badges, continuous drift monitoring, a governance policy API and a
|
|
133
|
+
hosted gateway at **[checkmcp.dev](https://checkmcp.dev)**.
|
|
134
|
+
|
|
135
|
+
## Honest limitations
|
|
136
|
+
- Percentile bands come from a growing corpus (one+ registries) — widening over time.
|
|
137
|
+
- Exact tokens with `pipx install "audit-mcp[exact-tokens]"` (cl100k_base); otherwise chars/4 approximation.
|
|
138
|
+
- Pillar weights are expert priors. `python -m checkmcp.calibrate samples.json` validates them against a labeled agent-success sample (per-pillar correlation + OLS-suggested weights + construct-validity R²) — supply real outcomes to close the loop.
|
|
139
|
+
|
|
140
|
+
MIT.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
audit_mcp.egg-info/PKG-INFO
|
|
5
|
+
audit_mcp.egg-info/SOURCES.txt
|
|
6
|
+
audit_mcp.egg-info/dependency_links.txt
|
|
7
|
+
audit_mcp.egg-info/entry_points.txt
|
|
8
|
+
audit_mcp.egg-info/requires.txt
|
|
9
|
+
audit_mcp.egg-info/top_level.txt
|
|
10
|
+
checkmcp/__init__.py
|
|
11
|
+
checkmcp/badge.py
|
|
12
|
+
checkmcp/calibrate.py
|
|
13
|
+
checkmcp/cli.py
|
|
14
|
+
checkmcp/evals.py
|
|
15
|
+
checkmcp/gateway.py
|
|
16
|
+
checkmcp/mcp_server.py
|
|
17
|
+
checkmcp/monitor.py
|
|
18
|
+
checkmcp/optimize.py
|
|
19
|
+
checkmcp/page.py
|
|
20
|
+
checkmcp/plans.py
|
|
21
|
+
checkmcp/probe.py
|
|
22
|
+
checkmcp/repo.py
|
|
23
|
+
checkmcp/scanners.py
|
|
24
|
+
checkmcp/score.py
|
|
25
|
+
checkmcp/security.py
|
|
26
|
+
checkmcp/store.py
|
|
27
|
+
tests/test_calibrate.py
|
|
28
|
+
tests/test_evals.py
|
|
29
|
+
tests/test_mcp_server.py
|
|
30
|
+
tests/test_misc.py
|
|
31
|
+
tests/test_monitor.py
|
|
32
|
+
tests/test_score.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
checkmcp
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Badge SVG "MCP Score" (style shields) + snippet d'embed. Mécanisme viral de CheckMCP."""
|
|
2
|
+
|
|
3
|
+
GRADE_COLOR = {"A": "#2ea44f", "B": "#7fbf3f", "C": "#dfb317", "D": "#fe7d37", "F": "#e05d44"}
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _w(text, px=7):
|
|
7
|
+
return max(20, int(len(text) * px) + 10)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def badge_svg(score, grade, label="MCP Score"):
|
|
11
|
+
val = f"{score} · {grade}"
|
|
12
|
+
color = GRADE_COLOR.get(grade, "#9f9f9f")
|
|
13
|
+
lw, vw = _w(label), _w(val)
|
|
14
|
+
total = lw + vw
|
|
15
|
+
return f'''<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" role="img" aria-label="{label}: {val}">
|
|
16
|
+
<linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient>
|
|
17
|
+
<clipPath id="r"><rect width="{total}" height="20" rx="3" fill="#fff"/></clipPath>
|
|
18
|
+
<g clip-path="url(#r)">
|
|
19
|
+
<rect width="{lw}" height="20" fill="#555"/>
|
|
20
|
+
<rect x="{lw}" width="{vw}" height="20" fill="{color}"/>
|
|
21
|
+
<rect width="{total}" height="20" fill="url(#s)"/>
|
|
22
|
+
</g>
|
|
23
|
+
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">
|
|
24
|
+
<text x="{lw/2*10}" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="{(lw-10)*10}">{label}</text>
|
|
25
|
+
<text x="{lw/2*10}" y="140" transform="scale(.1)" textLength="{(lw-10)*10}">{label}</text>
|
|
26
|
+
<text x="{(lw+vw/2)*10}" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="{(vw-10)*10}">{val}</text>
|
|
27
|
+
<text x="{(lw+vw/2)*10}" y="140" transform="scale(.1)" textLength="{(vw-10)*10}">{val}</text>
|
|
28
|
+
</g></svg>'''
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def embed_snippets(slug, score, grade):
|
|
32
|
+
base = f"https://checkmcp.com/badge/{slug}.svg"
|
|
33
|
+
page = f"https://checkmcp.com/mcp/{slug}"
|
|
34
|
+
return {
|
|
35
|
+
"markdown": f"[]({page})",
|
|
36
|
+
"html": f'<a href="{page}"><img src="{base}" alt="MCP Score {score} {grade}"></a>',
|
|
37
|
+
"url": base,
|
|
38
|
+
}
|