audit-mcp 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. audit_mcp-0.5.0/LICENSE +21 -0
  2. audit_mcp-0.5.0/PKG-INFO +140 -0
  3. audit_mcp-0.5.0/README.md +111 -0
  4. audit_mcp-0.5.0/audit_mcp.egg-info/PKG-INFO +140 -0
  5. audit_mcp-0.5.0/audit_mcp.egg-info/SOURCES.txt +32 -0
  6. audit_mcp-0.5.0/audit_mcp.egg-info/dependency_links.txt +1 -0
  7. audit_mcp-0.5.0/audit_mcp.egg-info/entry_points.txt +3 -0
  8. audit_mcp-0.5.0/audit_mcp.egg-info/requires.txt +6 -0
  9. audit_mcp-0.5.0/audit_mcp.egg-info/top_level.txt +1 -0
  10. audit_mcp-0.5.0/checkmcp/__init__.py +2 -0
  11. audit_mcp-0.5.0/checkmcp/badge.py +38 -0
  12. audit_mcp-0.5.0/checkmcp/calibrate.py +198 -0
  13. audit_mcp-0.5.0/checkmcp/cli.py +215 -0
  14. audit_mcp-0.5.0/checkmcp/evals.py +200 -0
  15. audit_mcp-0.5.0/checkmcp/gateway.py +159 -0
  16. audit_mcp-0.5.0/checkmcp/mcp_server.py +202 -0
  17. audit_mcp-0.5.0/checkmcp/monitor.py +59 -0
  18. audit_mcp-0.5.0/checkmcp/optimize.py +123 -0
  19. audit_mcp-0.5.0/checkmcp/page.py +84 -0
  20. audit_mcp-0.5.0/checkmcp/plans.py +11 -0
  21. audit_mcp-0.5.0/checkmcp/probe.py +383 -0
  22. audit_mcp-0.5.0/checkmcp/repo.py +147 -0
  23. audit_mcp-0.5.0/checkmcp/scanners.py +92 -0
  24. audit_mcp-0.5.0/checkmcp/score.py +178 -0
  25. audit_mcp-0.5.0/checkmcp/security.py +86 -0
  26. audit_mcp-0.5.0/checkmcp/store.py +298 -0
  27. audit_mcp-0.5.0/pyproject.toml +45 -0
  28. audit_mcp-0.5.0/setup.cfg +4 -0
  29. audit_mcp-0.5.0/tests/test_calibrate.py +45 -0
  30. audit_mcp-0.5.0/tests/test_evals.py +68 -0
  31. audit_mcp-0.5.0/tests/test_mcp_server.py +148 -0
  32. audit_mcp-0.5.0/tests/test_misc.py +12 -0
  33. audit_mcp-0.5.0/tests/test_monitor.py +45 -0
  34. audit_mcp-0.5.0/tests/test_score.py +54 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 CheckMCP
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,140 @@
1
+ Metadata-Version: 2.4
2
+ Name: audit-mcp
3
+ Version: 0.5.0
4
+ Summary: CheckMCP — vendor-neutral quality/security/context-cost audit & score for any MCP server.
5
+ Author: Hugo
6
+ License: MIT
7
+ Project-URL: Homepage, https://checkmcp.dev
8
+ Project-URL: Repository, https://github.com/H129hj/checkmcp
9
+ Keywords: mcp,model-context-protocol,audit,score,lint,security,gateway,guardrail
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Topic :: Security
19
+ Classifier: Topic :: Software Development :: Quality Assurance
20
+ Classifier: Topic :: Software Development :: Testing
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Provides-Extra: exact-tokens
25
+ Requires-Dist: tiktoken>=0.7; extra == "exact-tokens"
26
+ Provides-Extra: test
27
+ Requires-Dist: pytest>=7; extra == "test"
28
+ Dynamic: license-file
29
+
30
+ <!-- mcp-name: io.github.H129hj/checkmcp -->
31
+
32
+ # CheckMCP
33
+
34
+ [![PyPI](https://img.shields.io/pypi/v/audit-mcp)](https://pypi.org/project/audit-mcp/) [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) [![GitHub Action](https://img.shields.io/badge/CI-GitHub%20Action-2088FF?logo=githubactions&logoColor=white)](action.yml) [![Web audit](https://img.shields.io/badge/web-checkmcp.dev-blue)](https://checkmcp.dev)
35
+
36
+ **Vendor-neutral quality / security / context-cost audit & score for any MCP server.**
37
+ One `uvx`/`pipx` command → an **MCP Score /100** + **causal opportunities** (why the score), Lighthouse-style.
38
+
39
+ > Installed from PyPI as **`audit-mcp`** (the name `checkmcp` was already taken); the command is `audit-mcp`. Brand, site and repo remain **CheckMCP** / checkmcp.dev.
40
+
41
+ ```bash
42
+ uvx audit-mcp https://mcp.deepwiki.com/mcp
43
+ # or
44
+ pipx run audit-mcp https://mcp.context7.com/mcp --json
45
+ audit-mcp https://my-mcp.example.com/mcp --token "$TOKEN"
46
+ ```
47
+
48
+ ## Use it as an MCP server
49
+
50
+ `audit-mcp mcp` turns the auditor itself into an MCP server (stdio) exposing one tool,
51
+ `audit_mcp_server` — so your agent can answer *"is this MCP server safe?"* mid-conversation.
52
+
53
+ ```bash
54
+ # Claude Code
55
+ claude mcp add audit-mcp -- uvx audit-mcp mcp
56
+ ```
57
+
58
+ ```json
59
+ // Cursor (.cursor/mcp.json) / Claude Desktop (claude_desktop_config.json)
60
+ { "mcpServers": { "audit-mcp": { "command": "uvx", "args": ["audit-mcp", "mcp"] } } }
61
+ ```
62
+
63
+ Registry name: `io.github.H129hj/checkmcp` <!-- mcp-name: io.github.H129hj/checkmcp -->
64
+
65
+
66
+ No dependencies (stdlib only). `tiktoken` optional for exact token counts.
67
+
68
+ ## What it measures (7 pillars)
69
+ - **Security** — OWASP MCP Top 10 (tool poisoning, hardcoded secrets, command injection), lethal-trifecta.
70
+ - **Tool design** — sprawl/consolidation (percentile-calibrated on real servers: median ~7 tools, p95 ~42).
71
+ - **Schemas / desc** — descriptions + `inputSchema`/`outputSchema` completeness.
72
+ - **Context-cost** — tokens spent on `tools/list`, paid on every request (the #1 pain of 2026).
73
+ - **Compliance** — protocol-version gap, annotations, JSON-RPC error conformance, OAuth discovery.
74
+ - **Reliability** — single-shot today (not credited; continuous T3 monitoring on checkmcp.dev).
75
+ - **Coverage** — the 3 primitives (tools **+ resources + prompts**).
76
+
77
+ Hard floors: secret-in-schema → cap D, failed handshake → cap F. Every penalty is attributed: `measure → mechanism → effect → Δscore`.
78
+
79
+ ## CLI flags
80
+ | flag | what |
81
+ |---|---|
82
+ | `--json` | machine-readable report |
83
+ | `--badge` | SVG badge + README embed snippets |
84
+ | `--html` | standalone SEO/GEO page (JSON-LD `SoftwareApplication` + FAQ) |
85
+ | `--repo owner/name` | add maintenance/license/provenance signal from GitHub |
86
+ | `--token <bearer>` | audit an OAuth-protected server |
87
+ | `--min-score N` | CI: exit 1 if MCP Score < N |
88
+ | `--baseline file` | CI: pin tool definitions; fail on regression (rug-pull) |
89
+ | `--gh-summary` | CI: write a Markdown summary to `$GITHUB_STEP_SUMMARY` |
90
+ | `--deep` | runtime depth via an external scanner (mcp-scan/snyk) if present |
91
+ | `--evals` | behavioral sandbox: actually invokes read-only tools with canary inputs to catch tool-output prompt-injection, exfiltration vectors, secret/PII leakage and context bombs (sends real traffic; CI-fails on a malicious verdict) |
92
+
93
+ ## GitHub Action
94
+
95
+ ```yaml
96
+ # .github/workflows/mcp-audit.yml
97
+ name: MCP audit
98
+ on: [push, pull_request]
99
+ jobs:
100
+ checkmcp:
101
+ runs-on: ubuntu-latest
102
+ steps:
103
+ - uses: actions/checkout@v4
104
+ - uses: H129hj/checkmcp@v1
105
+ with:
106
+ url: https://my-mcp.example.com/mcp
107
+ min-score: "70"
108
+ baseline: .checkmcp-baseline.json # commit it → fails on rug-pull
109
+ ```
110
+
111
+ ## Behavioral evals (`--evals`)
112
+ Static analysis catches *declared* danger; `--evals` catches *runtime* danger by actually invoking
113
+ read-only tools with canary inputs and inspecting the **responses** for tool-output prompt-injection,
114
+ exfiltration vectors and secret/PII leakage (multilingual; optional callback-canary confirms exfil).
115
+ CI-fails on a malicious verdict.
116
+
117
+ ## Self-hosted security gateway
118
+ Beyond auditing, CheckMCP ships an **in-band MCP gateway** — a proxy you put between your agent and an
119
+ MCP server. It inspects every call, and in *active* mode **blocks/strips** tool-poisoning & exfiltration
120
+ before they reach the agent. Run it in your own infra (tool traffic never leaves your network):
121
+
122
+ ```bash
123
+ docker pull ghcr.io/h129hj/checkmcp-gateway:latest # or build from source
124
+ docker run -p 8080:8080 -e GATEWAY_BACKEND_URL=https://mcp.example.com/mcp \
125
+ -e GATEWAY_MODE=active -e GATEWAY_SECRET=$(openssl rand -hex 16) \
126
+ ghcr.io/h129hj/checkmcp-gateway:latest
127
+ ```
128
+
129
+ See **[GATEWAY.md](GATEWAY.md)** for config (passive/active, OAuth backends, policy, logs).
130
+
131
+ ## Hosted
132
+ Full reports, public directory, live badges, continuous drift monitoring, a governance policy API and a
133
+ hosted gateway at **[checkmcp.dev](https://checkmcp.dev)**.
134
+
135
+ ## Honest limitations
136
+ - Percentile bands come from a growing corpus (one+ registries) — widening over time.
137
+ - Exact tokens with `pipx install "audit-mcp[exact-tokens]"` (cl100k_base); otherwise chars/4 approximation.
138
+ - Pillar weights are expert priors. `python -m checkmcp.calibrate samples.json` validates them against a labeled agent-success sample (per-pillar correlation + OLS-suggested weights + construct-validity R²) — supply real outcomes to close the loop.
139
+
140
+ MIT.
@@ -0,0 +1,111 @@
1
+ <!-- mcp-name: io.github.H129hj/checkmcp -->
2
+
3
+ # CheckMCP
4
+
5
+ [![PyPI](https://img.shields.io/pypi/v/audit-mcp)](https://pypi.org/project/audit-mcp/) [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) [![GitHub Action](https://img.shields.io/badge/CI-GitHub%20Action-2088FF?logo=githubactions&logoColor=white)](action.yml) [![Web audit](https://img.shields.io/badge/web-checkmcp.dev-blue)](https://checkmcp.dev)
6
+
7
+ **Vendor-neutral quality / security / context-cost audit & score for any MCP server.**
8
+ One `uvx`/`pipx` command → an **MCP Score /100** + **causal opportunities** (why the score), Lighthouse-style.
9
+
10
+ > Installed from PyPI as **`audit-mcp`** (the name `checkmcp` was already taken); the command is `audit-mcp`. Brand, site and repo remain **CheckMCP** / checkmcp.dev.
11
+
12
+ ```bash
13
+ uvx audit-mcp https://mcp.deepwiki.com/mcp
14
+ # or
15
+ pipx run audit-mcp https://mcp.context7.com/mcp --json
16
+ audit-mcp https://my-mcp.example.com/mcp --token "$TOKEN"
17
+ ```
18
+
19
+ ## Use it as an MCP server
20
+
21
+ `audit-mcp mcp` turns the auditor itself into an MCP server (stdio) exposing one tool,
22
+ `audit_mcp_server` — so your agent can answer *"is this MCP server safe?"* mid-conversation.
23
+
24
+ ```bash
25
+ # Claude Code
26
+ claude mcp add audit-mcp -- uvx audit-mcp mcp
27
+ ```
28
+
29
+ ```json
30
+ // Cursor (.cursor/mcp.json) / Claude Desktop (claude_desktop_config.json)
31
+ { "mcpServers": { "audit-mcp": { "command": "uvx", "args": ["audit-mcp", "mcp"] } } }
32
+ ```
33
+
34
+ Registry name: `io.github.H129hj/checkmcp` <!-- mcp-name: io.github.H129hj/checkmcp -->
35
+
36
+
37
+ No dependencies (stdlib only). `tiktoken` optional for exact token counts.
38
+
39
+ ## What it measures (7 pillars)
40
+ - **Security** — OWASP MCP Top 10 (tool poisoning, hardcoded secrets, command injection), lethal-trifecta.
41
+ - **Tool design** — sprawl/consolidation (percentile-calibrated on real servers: median ~7 tools, p95 ~42).
42
+ - **Schemas / desc** — descriptions + `inputSchema`/`outputSchema` completeness.
43
+ - **Context-cost** — tokens spent on `tools/list`, paid on every request (the #1 pain of 2026).
44
+ - **Compliance** — protocol-version gap, annotations, JSON-RPC error conformance, OAuth discovery.
45
+ - **Reliability** — single-shot today (not credited; continuous T3 monitoring on checkmcp.dev).
46
+ - **Coverage** — the 3 primitives (tools **+ resources + prompts**).
47
+
48
+ Hard floors: secret-in-schema → cap D, failed handshake → cap F. Every penalty is attributed: `measure → mechanism → effect → Δscore`.
49
+
50
+ ## CLI flags
51
+ | flag | what |
52
+ |---|---|
53
+ | `--json` | machine-readable report |
54
+ | `--badge` | SVG badge + README embed snippets |
55
+ | `--html` | standalone SEO/GEO page (JSON-LD `SoftwareApplication` + FAQ) |
56
+ | `--repo owner/name` | add maintenance/license/provenance signal from GitHub |
57
+ | `--token <bearer>` | audit an OAuth-protected server |
58
+ | `--min-score N` | CI: exit 1 if MCP Score < N |
59
+ | `--baseline file` | CI: pin tool definitions; fail on regression (rug-pull) |
60
+ | `--gh-summary` | CI: write a Markdown summary to `$GITHUB_STEP_SUMMARY` |
61
+ | `--deep` | runtime depth via an external scanner (mcp-scan/snyk) if present |
62
+ | `--evals` | behavioral sandbox: actually invokes read-only tools with canary inputs to catch tool-output prompt-injection, exfiltration vectors, secret/PII leakage and context bombs (sends real traffic; CI-fails on a malicious verdict) |
63
+
64
+ ## GitHub Action
65
+
66
+ ```yaml
67
+ # .github/workflows/mcp-audit.yml
68
+ name: MCP audit
69
+ on: [push, pull_request]
70
+ jobs:
71
+ checkmcp:
72
+ runs-on: ubuntu-latest
73
+ steps:
74
+ - uses: actions/checkout@v4
75
+ - uses: H129hj/checkmcp@v1
76
+ with:
77
+ url: https://my-mcp.example.com/mcp
78
+ min-score: "70"
79
+ baseline: .checkmcp-baseline.json # commit it → fails on rug-pull
80
+ ```
81
+
82
+ ## Behavioral evals (`--evals`)
83
+ Static analysis catches *declared* danger; `--evals` catches *runtime* danger by actually invoking
84
+ read-only tools with canary inputs and inspecting the **responses** for tool-output prompt-injection,
85
+ exfiltration vectors and secret/PII leakage (multilingual; optional callback-canary confirms exfil).
86
+ CI-fails on a malicious verdict.
87
+
88
+ ## Self-hosted security gateway
89
+ Beyond auditing, CheckMCP ships an **in-band MCP gateway** — a proxy you put between your agent and an
90
+ MCP server. It inspects every call, and in *active* mode **blocks/strips** tool-poisoning & exfiltration
91
+ before they reach the agent. Run it in your own infra (tool traffic never leaves your network):
92
+
93
+ ```bash
94
+ docker pull ghcr.io/h129hj/checkmcp-gateway:latest # or build from source
95
+ docker run -p 8080:8080 -e GATEWAY_BACKEND_URL=https://mcp.example.com/mcp \
96
+ -e GATEWAY_MODE=active -e GATEWAY_SECRET=$(openssl rand -hex 16) \
97
+ ghcr.io/h129hj/checkmcp-gateway:latest
98
+ ```
99
+
100
+ See **[GATEWAY.md](GATEWAY.md)** for config (passive/active, OAuth backends, policy, logs).
101
+
102
+ ## Hosted
103
+ Full reports, public directory, live badges, continuous drift monitoring, a governance policy API and a
104
+ hosted gateway at **[checkmcp.dev](https://checkmcp.dev)**.
105
+
106
+ ## Honest limitations
107
+ - Percentile bands come from a growing corpus (one+ registries) — widening over time.
108
+ - Exact tokens with `pipx install "audit-mcp[exact-tokens]"` (cl100k_base); otherwise chars/4 approximation.
109
+ - Pillar weights are expert priors. `python -m checkmcp.calibrate samples.json` validates them against a labeled agent-success sample (per-pillar correlation + OLS-suggested weights + construct-validity R²) — supply real outcomes to close the loop.
110
+
111
+ MIT.
@@ -0,0 +1,140 @@
1
+ Metadata-Version: 2.4
2
+ Name: audit-mcp
3
+ Version: 0.5.0
4
+ Summary: CheckMCP — vendor-neutral quality/security/context-cost audit & score for any MCP server.
5
+ Author: Hugo
6
+ License: MIT
7
+ Project-URL: Homepage, https://checkmcp.dev
8
+ Project-URL: Repository, https://github.com/H129hj/checkmcp
9
+ Keywords: mcp,model-context-protocol,audit,score,lint,security,gateway,guardrail
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Topic :: Security
19
+ Classifier: Topic :: Software Development :: Quality Assurance
20
+ Classifier: Topic :: Software Development :: Testing
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Provides-Extra: exact-tokens
25
+ Requires-Dist: tiktoken>=0.7; extra == "exact-tokens"
26
+ Provides-Extra: test
27
+ Requires-Dist: pytest>=7; extra == "test"
28
+ Dynamic: license-file
29
+
30
+ <!-- mcp-name: io.github.H129hj/checkmcp -->
31
+
32
+ # CheckMCP
33
+
34
+ [![PyPI](https://img.shields.io/pypi/v/audit-mcp)](https://pypi.org/project/audit-mcp/) [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) [![GitHub Action](https://img.shields.io/badge/CI-GitHub%20Action-2088FF?logo=githubactions&logoColor=white)](action.yml) [![Web audit](https://img.shields.io/badge/web-checkmcp.dev-blue)](https://checkmcp.dev)
35
+
36
+ **Vendor-neutral quality / security / context-cost audit & score for any MCP server.**
37
+ One `uvx`/`pipx` command → an **MCP Score /100** + **causal opportunities** (why the score), Lighthouse-style.
38
+
39
+ > Installed from PyPI as **`audit-mcp`** (the name `checkmcp` was already taken); the command is `audit-mcp`. Brand, site and repo remain **CheckMCP** / checkmcp.dev.
40
+
41
+ ```bash
42
+ uvx audit-mcp https://mcp.deepwiki.com/mcp
43
+ # or
44
+ pipx run audit-mcp https://mcp.context7.com/mcp --json
45
+ audit-mcp https://my-mcp.example.com/mcp --token "$TOKEN"
46
+ ```
47
+
48
+ ## Use it as an MCP server
49
+
50
+ `audit-mcp mcp` turns the auditor itself into an MCP server (stdio) exposing one tool,
51
+ `audit_mcp_server` — so your agent can answer *"is this MCP server safe?"* mid-conversation.
52
+
53
+ ```bash
54
+ # Claude Code
55
+ claude mcp add audit-mcp -- uvx audit-mcp mcp
56
+ ```
57
+
58
+ ```json
59
+ // Cursor (.cursor/mcp.json) / Claude Desktop (claude_desktop_config.json)
60
+ { "mcpServers": { "audit-mcp": { "command": "uvx", "args": ["audit-mcp", "mcp"] } } }
61
+ ```
62
+
63
+ Registry name: `io.github.H129hj/checkmcp` <!-- mcp-name: io.github.H129hj/checkmcp -->
64
+
65
+
66
+ No dependencies (stdlib only). `tiktoken` optional for exact token counts.
67
+
68
+ ## What it measures (7 pillars)
69
+ - **Security** — OWASP MCP Top 10 (tool poisoning, hardcoded secrets, command injection), lethal-trifecta.
70
+ - **Tool design** — sprawl/consolidation (percentile-calibrated on real servers: median ~7 tools, p95 ~42).
71
+ - **Schemas / desc** — descriptions + `inputSchema`/`outputSchema` completeness.
72
+ - **Context-cost** — tokens spent on `tools/list`, paid on every request (the #1 pain of 2026).
73
+ - **Compliance** — protocol-version gap, annotations, JSON-RPC error conformance, OAuth discovery.
74
+ - **Reliability** — single-shot today (not credited; continuous T3 monitoring on checkmcp.dev).
75
+ - **Coverage** — the 3 primitives (tools **+ resources + prompts**).
76
+
77
+ Hard floors: secret-in-schema → cap D, failed handshake → cap F. Every penalty is attributed: `measure → mechanism → effect → Δscore`.
78
+
79
+ ## CLI flags
80
+ | flag | what |
81
+ |---|---|
82
+ | `--json` | machine-readable report |
83
+ | `--badge` | SVG badge + README embed snippets |
84
+ | `--html` | standalone SEO/GEO page (JSON-LD `SoftwareApplication` + FAQ) |
85
+ | `--repo owner/name` | add maintenance/license/provenance signal from GitHub |
86
+ | `--token <bearer>` | audit an OAuth-protected server |
87
+ | `--min-score N` | CI: exit 1 if MCP Score < N |
88
+ | `--baseline file` | CI: pin tool definitions; fail on regression (rug-pull) |
89
+ | `--gh-summary` | CI: write a Markdown summary to `$GITHUB_STEP_SUMMARY` |
90
+ | `--deep` | runtime depth via an external scanner (mcp-scan/snyk) if present |
91
+ | `--evals` | behavioral sandbox: actually invokes read-only tools with canary inputs to catch tool-output prompt-injection, exfiltration vectors, secret/PII leakage and context bombs (sends real traffic; CI-fails on a malicious verdict) |
92
+
93
+ ## GitHub Action
94
+
95
+ ```yaml
96
+ # .github/workflows/mcp-audit.yml
97
+ name: MCP audit
98
+ on: [push, pull_request]
99
+ jobs:
100
+ checkmcp:
101
+ runs-on: ubuntu-latest
102
+ steps:
103
+ - uses: actions/checkout@v4
104
+ - uses: H129hj/checkmcp@v1
105
+ with:
106
+ url: https://my-mcp.example.com/mcp
107
+ min-score: "70"
108
+ baseline: .checkmcp-baseline.json # commit it → fails on rug-pull
109
+ ```
110
+
111
+ ## Behavioral evals (`--evals`)
112
+ Static analysis catches *declared* danger; `--evals` catches *runtime* danger by actually invoking
113
+ read-only tools with canary inputs and inspecting the **responses** for tool-output prompt-injection,
114
+ exfiltration vectors and secret/PII leakage (multilingual; optional callback-canary confirms exfil).
115
+ CI-fails on a malicious verdict.
116
+
117
+ ## Self-hosted security gateway
118
+ Beyond auditing, CheckMCP ships an **in-band MCP gateway** — a proxy you put between your agent and an
119
+ MCP server. It inspects every call, and in *active* mode **blocks/strips** tool-poisoning & exfiltration
120
+ before they reach the agent. Run it in your own infra (tool traffic never leaves your network):
121
+
122
+ ```bash
123
+ docker pull ghcr.io/h129hj/checkmcp-gateway:latest # or build from source
124
+ docker run -p 8080:8080 -e GATEWAY_BACKEND_URL=https://mcp.example.com/mcp \
125
+ -e GATEWAY_MODE=active -e GATEWAY_SECRET=$(openssl rand -hex 16) \
126
+ ghcr.io/h129hj/checkmcp-gateway:latest
127
+ ```
128
+
129
+ See **[GATEWAY.md](GATEWAY.md)** for config (passive/active, OAuth backends, policy, logs).
130
+
131
+ ## Hosted
132
+ Full reports, public directory, live badges, continuous drift monitoring, a governance policy API and a
133
+ hosted gateway at **[checkmcp.dev](https://checkmcp.dev)**.
134
+
135
+ ## Honest limitations
136
+ - Percentile bands come from a growing corpus (one+ registries) — widening over time.
137
+ - Exact tokens with `pipx install "audit-mcp[exact-tokens]"` (cl100k_base); otherwise chars/4 approximation.
138
+ - Pillar weights are expert priors. `python -m checkmcp.calibrate samples.json` validates them against a labeled agent-success sample (per-pillar correlation + OLS-suggested weights + construct-validity R²) — supply real outcomes to close the loop.
139
+
140
+ MIT.
@@ -0,0 +1,32 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ audit_mcp.egg-info/PKG-INFO
5
+ audit_mcp.egg-info/SOURCES.txt
6
+ audit_mcp.egg-info/dependency_links.txt
7
+ audit_mcp.egg-info/entry_points.txt
8
+ audit_mcp.egg-info/requires.txt
9
+ audit_mcp.egg-info/top_level.txt
10
+ checkmcp/__init__.py
11
+ checkmcp/badge.py
12
+ checkmcp/calibrate.py
13
+ checkmcp/cli.py
14
+ checkmcp/evals.py
15
+ checkmcp/gateway.py
16
+ checkmcp/mcp_server.py
17
+ checkmcp/monitor.py
18
+ checkmcp/optimize.py
19
+ checkmcp/page.py
20
+ checkmcp/plans.py
21
+ checkmcp/probe.py
22
+ checkmcp/repo.py
23
+ checkmcp/scanners.py
24
+ checkmcp/score.py
25
+ checkmcp/security.py
26
+ checkmcp/store.py
27
+ tests/test_calibrate.py
28
+ tests/test_evals.py
29
+ tests/test_mcp_server.py
30
+ tests/test_misc.py
31
+ tests/test_monitor.py
32
+ tests/test_score.py
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ audit-mcp = checkmcp.cli:main
3
+ audit-mcp-mcp = checkmcp.mcp_server:main
@@ -0,0 +1,6 @@
1
+
2
+ [exact-tokens]
3
+ tiktoken>=0.7
4
+
5
+ [test]
6
+ pytest>=7
@@ -0,0 +1 @@
1
+ checkmcp
@@ -0,0 +1,2 @@
1
+ """CheckMCP — audit & score de qualité/sécurité/context-cost pour serveurs MCP."""
2
+ __version__ = "0.5.0"
@@ -0,0 +1,38 @@
1
+ """Badge SVG "MCP Score" (style shields) + snippet d'embed. Mécanisme viral de CheckMCP."""
2
+
3
+ GRADE_COLOR = {"A": "#2ea44f", "B": "#7fbf3f", "C": "#dfb317", "D": "#fe7d37", "F": "#e05d44"}
4
+
5
+
6
+ def _w(text, px=7):
7
+ return max(20, int(len(text) * px) + 10)
8
+
9
+
10
+ def badge_svg(score, grade, label="MCP Score"):
11
+ val = f"{score} · {grade}"
12
+ color = GRADE_COLOR.get(grade, "#9f9f9f")
13
+ lw, vw = _w(label), _w(val)
14
+ total = lw + vw
15
+ return f'''<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" role="img" aria-label="{label}: {val}">
16
+ <linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient>
17
+ <clipPath id="r"><rect width="{total}" height="20" rx="3" fill="#fff"/></clipPath>
18
+ <g clip-path="url(#r)">
19
+ <rect width="{lw}" height="20" fill="#555"/>
20
+ <rect x="{lw}" width="{vw}" height="20" fill="{color}"/>
21
+ <rect width="{total}" height="20" fill="url(#s)"/>
22
+ </g>
23
+ <g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">
24
+ <text x="{lw/2*10}" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="{(lw-10)*10}">{label}</text>
25
+ <text x="{lw/2*10}" y="140" transform="scale(.1)" textLength="{(lw-10)*10}">{label}</text>
26
+ <text x="{(lw+vw/2)*10}" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="{(vw-10)*10}">{val}</text>
27
+ <text x="{(lw+vw/2)*10}" y="140" transform="scale(.1)" textLength="{(vw-10)*10}">{val}</text>
28
+ </g></svg>'''
29
+
30
+
31
+ def embed_snippets(slug, score, grade):
32
+ base = f"https://checkmcp.com/badge/{slug}.svg"
33
+ page = f"https://checkmcp.com/mcp/{slug}"
34
+ return {
35
+ "markdown": f"[![MCP Score {score} {grade}]({base})]({page})",
36
+ "html": f'<a href="{page}"><img src="{base}" alt="MCP Score {score} {grade}"></a>',
37
+ "url": base,
38
+ }