agentcheckup 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentcheckup/__init__.py +27 -0
- agentcheckup/__main__.py +6 -0
- agentcheckup/cli.py +242 -0
- agentcheckup/conflicts.py +253 -0
- agentcheckup/discovery.py +206 -0
- agentcheckup/engine.py +146 -0
- agentcheckup/loader.py +214 -0
- agentcheckup/models.py +166 -0
- agentcheckup/report.py +274 -0
- agentcheckup/rules/__init__.py +12 -0
- agentcheckup/rules/base.py +91 -0
- agentcheckup/rules/config_rules.py +180 -0
- agentcheckup/rules/docs.py +124 -0
- agentcheckup/rules/injection.py +127 -0
- agentcheckup/rules/naming.py +100 -0
- agentcheckup/rules/safety.py +131 -0
- agentcheckup/rules/schema.py +143 -0
- agentcheckup/tokens.py +69 -0
- agentcheckup-0.4.0.dist-info/METADATA +261 -0
- agentcheckup-0.4.0.dist-info/RECORD +23 -0
- agentcheckup-0.4.0.dist-info/WHEEL +4 -0
- agentcheckup-0.4.0.dist-info/entry_points.txt +2 -0
- agentcheckup-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Client-config rules.
|
|
2
|
+
|
|
3
|
+
Most MCP incidents start here rather than in the server code: an unpinned
|
|
4
|
+
``npx -y`` package that silently updates, a shell wrapper that turns every arg
|
|
5
|
+
into a command, a filesystem server rooted at ``/``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
from ..models import Finding, ServerConfig, Severity
|
|
13
|
+
from .base import config_rule
|
|
14
|
+
|
|
15
|
+
SHELL_WRAPPERS = ("sh", "bash", "zsh", "cmd", "cmd.exe", "powershell", "pwsh")
|
|
16
|
+
|
|
17
|
+
DANGEROUS_FLAGS = (
|
|
18
|
+
"--dangerously-skip-permissions",
|
|
19
|
+
"--yolo",
|
|
20
|
+
"--allow-all",
|
|
21
|
+
"--no-sandbox",
|
|
22
|
+
"--disable-web-security",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
SECRET_KEY = re.compile(r"(api[_-]?key|token|secret|password|passwd|credential|private[_-]?key)", re.I)
|
|
26
|
+
|
|
27
|
+
DRIVE_ROOT = re.compile(r"^[A-Za-z]:[\\/]?$")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _package_arg(args: list[str]) -> str | None:
|
|
31
|
+
"""First positional argument, skipping flags and their values."""
|
|
32
|
+
skip_next = False
|
|
33
|
+
for arg in args:
|
|
34
|
+
if skip_next:
|
|
35
|
+
skip_next = False
|
|
36
|
+
continue
|
|
37
|
+
if arg in ("-y", "--yes", "-q", "--quiet"):
|
|
38
|
+
continue
|
|
39
|
+
if arg.startswith("-"):
|
|
40
|
+
# Flags like `--registry` take a value; treat conservatively.
|
|
41
|
+
if "=" not in arg and arg in ("--registry", "--cache", "--prefix", "--cwd"):
|
|
42
|
+
skip_next = True
|
|
43
|
+
continue
|
|
44
|
+
return arg
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _is_pinned(spec: str) -> bool:
|
|
49
|
+
"""`pkg@1.2.3` is pinned; `pkg`, `pkg@latest`, `pkg@^1` are not."""
|
|
50
|
+
if "@" not in spec:
|
|
51
|
+
return False
|
|
52
|
+
version = spec.rsplit("@", 1)[1]
|
|
53
|
+
if not version or version in ("latest", "next", "beta", "canary"):
|
|
54
|
+
return False
|
|
55
|
+
return version[0].isdigit()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@config_rule("AGT101", Severity.WARNING, "npx package is not version-pinned")
|
|
59
|
+
def unpinned_npx(server: ServerConfig):
|
|
60
|
+
"""`npx -y foo` re-resolves to the latest release on every launch.
|
|
61
|
+
|
|
62
|
+
That is a silent supply-chain dependency: the code you audited last week is
|
|
63
|
+
not necessarily the code running today.
|
|
64
|
+
"""
|
|
65
|
+
if not server.command or "npx" not in server.command.lower():
|
|
66
|
+
return None
|
|
67
|
+
spec = _package_arg(server.args)
|
|
68
|
+
if spec and not _is_pinned(spec):
|
|
69
|
+
return Finding(
|
|
70
|
+
rule="AGT101",
|
|
71
|
+
severity=Severity.WARNING,
|
|
72
|
+
message=f"Server {server.name!r} runs unpinned npx package {spec!r}.",
|
|
73
|
+
tool=server.name,
|
|
74
|
+
hint=f"Pin it: npx -y {spec}@1.2.3",
|
|
75
|
+
)
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@config_rule("AGT102", Severity.WARNING, "uvx package is not version-pinned")
|
|
80
|
+
def unpinned_uvx(server: ServerConfig):
|
|
81
|
+
"""Same failure mode as AGT101, on the Python side."""
|
|
82
|
+
if not server.command or "uvx" not in server.command.lower():
|
|
83
|
+
return None
|
|
84
|
+
spec = _package_arg(server.args)
|
|
85
|
+
if spec and not _is_pinned(spec):
|
|
86
|
+
return Finding(
|
|
87
|
+
rule="AGT102",
|
|
88
|
+
severity=Severity.WARNING,
|
|
89
|
+
message=f"Server {server.name!r} runs unpinned uvx package {spec!r}.",
|
|
90
|
+
tool=server.name,
|
|
91
|
+
hint=f"Pin it: uvx {spec}==1.2.3",
|
|
92
|
+
)
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@config_rule("AGT103", Severity.WARNING, "Config wraps the server in a shell")
|
|
97
|
+
def shell_wrapper(server: ServerConfig):
|
|
98
|
+
"""`sh -c "..."` re-parses every argument, so any string can become a command.
|
|
99
|
+
|
|
100
|
+
It also defeats argument escaping that the client already did for you.
|
|
101
|
+
"""
|
|
102
|
+
if not server.command:
|
|
103
|
+
return None
|
|
104
|
+
base = server.command.replace("\\", "/").rsplit("/", 1)[-1].lower()
|
|
105
|
+
if base in SHELL_WRAPPERS and any(a in ("-c", "/c", "/C", "-Command") for a in server.args):
|
|
106
|
+
return Finding(
|
|
107
|
+
rule="AGT103",
|
|
108
|
+
severity=Severity.WARNING,
|
|
109
|
+
message=f"Server {server.name!r} is launched through a shell wrapper ({server.command}).",
|
|
110
|
+
tool=server.name,
|
|
111
|
+
hint="Call the executable directly and pass args as a list.",
|
|
112
|
+
)
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@config_rule("AGT104", Severity.ERROR, "Filesystem server is rooted at the volume root")
|
|
117
|
+
def broad_fs_root(server: ServerConfig):
|
|
118
|
+
"""Granting the filesystem root gives the agent read-write access to everything.
|
|
119
|
+
|
|
120
|
+
One prompt injection is then enough to reach SSH keys, browser profiles and
|
|
121
|
+
every other project on the machine.
|
|
122
|
+
"""
|
|
123
|
+
for arg in server.args:
|
|
124
|
+
if arg == "/" or DRIVE_ROOT.match(arg):
|
|
125
|
+
return Finding(
|
|
126
|
+
rule="AGT104",
|
|
127
|
+
severity=Severity.ERROR,
|
|
128
|
+
message=f"Server {server.name!r} is granted the whole filesystem ({arg}).",
|
|
129
|
+
tool=server.name,
|
|
130
|
+
hint="Point it at a project directory instead of the volume root.",
|
|
131
|
+
)
|
|
132
|
+
return None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@config_rule("AGT105", Severity.WARNING, "Remote server uses plain HTTP")
|
|
136
|
+
def insecure_url(server: ServerConfig):
|
|
137
|
+
"""Tool calls carry your data and often your credentials."""
|
|
138
|
+
if server.url and server.url.lower().startswith("http://"):
|
|
139
|
+
return Finding(
|
|
140
|
+
rule="AGT105",
|
|
141
|
+
severity=Severity.WARNING,
|
|
142
|
+
message=f"Server {server.name!r} uses an unencrypted http:// endpoint.",
|
|
143
|
+
tool=server.name,
|
|
144
|
+
hint="Use https://, or a localhost port you control.",
|
|
145
|
+
)
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@config_rule("AGT106", Severity.ERROR, "Config passes a permission-bypass flag")
|
|
150
|
+
def dangerous_flag(server: ServerConfig):
|
|
151
|
+
"""These flags disable the only confirmation step between the model and your disk."""
|
|
152
|
+
joined = " ".join([server.command or "", *server.args])
|
|
153
|
+
for flag in DANGEROUS_FLAGS:
|
|
154
|
+
if flag in joined:
|
|
155
|
+
return Finding(
|
|
156
|
+
rule="AGT106",
|
|
157
|
+
severity=Severity.ERROR,
|
|
158
|
+
message=f"Server {server.name!r} is launched with {flag}.",
|
|
159
|
+
tool=server.name,
|
|
160
|
+
hint="Remove the flag and grant the narrowest permission that still works.",
|
|
161
|
+
)
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@config_rule("AGT107", Severity.WARNING, "Credentials are stored in plaintext in the config")
|
|
166
|
+
def plaintext_secret(server: ServerConfig):
|
|
167
|
+
"""Config files get committed, screenshotted and shared in issues.
|
|
168
|
+
|
|
169
|
+
Reading from the environment keeps the secret out of the file.
|
|
170
|
+
"""
|
|
171
|
+
hits = [k for k in server.env if SECRET_KEY.search(k)]
|
|
172
|
+
if hits:
|
|
173
|
+
return Finding(
|
|
174
|
+
rule="AGT107",
|
|
175
|
+
severity=Severity.WARNING,
|
|
176
|
+
message=f"Server {server.name!r} stores credential(s) inline: {', '.join(sorted(hits)[:5])}.",
|
|
177
|
+
tool=server.name,
|
|
178
|
+
hint="Reference an env var (e.g. ${GITHUB_TOKEN}) instead of pasting the value.",
|
|
179
|
+
)
|
|
180
|
+
return None
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Documentation rules.
|
|
2
|
+
|
|
3
|
+
Tool descriptions are not documentation for humans - they are the *prompt* the
|
|
4
|
+
model reads to decide whether and how to call the tool. Treating them as an
|
|
5
|
+
afterthought is the single most common reason agents pick the wrong tool.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from ..models import Finding, Severity, Tool
|
|
11
|
+
from .base import tool_rule
|
|
12
|
+
|
|
13
|
+
#: Phrases that indicate the description says *when not* to use the tool.
|
|
14
|
+
#: Presence of any one is enough; we do not try to judge quality beyond this.
|
|
15
|
+
NEGATIVE_CUES = (
|
|
16
|
+
"do not",
|
|
17
|
+
"don't",
|
|
18
|
+
"not for",
|
|
19
|
+
"instead",
|
|
20
|
+
"only when",
|
|
21
|
+
"use this when",
|
|
22
|
+
"prefer ",
|
|
23
|
+
"avoid ",
|
|
24
|
+
"not use",
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
MIN_DESCRIPTION = 40
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@tool_rule("AGT010", Severity.ERROR, "Tool has no description")
|
|
31
|
+
def missing_description(tool: Tool):
|
|
32
|
+
"""A tool without a description is invisible to the model.
|
|
33
|
+
|
|
34
|
+
It will still appear in the manifest and can still be called, but selection
|
|
35
|
+
becomes a coin flip against every other tool.
|
|
36
|
+
"""
|
|
37
|
+
if not tool.description.strip():
|
|
38
|
+
return Finding(
|
|
39
|
+
rule="AGT010",
|
|
40
|
+
severity=Severity.ERROR,
|
|
41
|
+
message=f"Tool {tool.name!r} has no description.",
|
|
42
|
+
tool=tool.name,
|
|
43
|
+
hint="One or two sentences: what it does, and what it returns.",
|
|
44
|
+
)
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@tool_rule("AGT011", Severity.INFO, "Tool description is too short to be useful")
|
|
49
|
+
def short_description(tool: Tool):
|
|
50
|
+
"""Under ~40 characters there is no room for the 'when' or the return shape."""
|
|
51
|
+
text = tool.description.strip()
|
|
52
|
+
if text and len(text) < MIN_DESCRIPTION:
|
|
53
|
+
return Finding(
|
|
54
|
+
rule="AGT011",
|
|
55
|
+
severity=Severity.INFO,
|
|
56
|
+
message=f"Tool {tool.name!r} description is {len(text)} chars.",
|
|
57
|
+
tool=tool.name,
|
|
58
|
+
hint="Aim for 80-200 chars: what it does, when to use it, what it returns.",
|
|
59
|
+
)
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@tool_rule("AGT012", Severity.INFO, "Description never says when NOT to use the tool")
|
|
64
|
+
def no_negative_guidance(tool: Tool):
|
|
65
|
+
"""Disambiguation is what separates a reliable tool set from a lucky one.
|
|
66
|
+
|
|
67
|
+
When two tools overlap, the model needs an explicit boundary. One clause is
|
|
68
|
+
usually enough: "Use this only for public repos; use `get_private_repo` for
|
|
69
|
+
authenticated access."
|
|
70
|
+
"""
|
|
71
|
+
text = tool.description.lower()
|
|
72
|
+
if len(text) >= MIN_DESCRIPTION and not any(cue in text for cue in NEGATIVE_CUES):
|
|
73
|
+
return Finding(
|
|
74
|
+
rule="AGT012",
|
|
75
|
+
severity=Severity.INFO,
|
|
76
|
+
message=f"Tool {tool.name!r} description gives no disambiguation clause.",
|
|
77
|
+
tool=tool.name,
|
|
78
|
+
hint='Add e.g. "Use this only when ...; prefer `other_tool` for ...".',
|
|
79
|
+
)
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@tool_rule("AGT013", Severity.WARNING, "Input parameters are undocumented")
|
|
84
|
+
def undocumented_params(tool: Tool):
|
|
85
|
+
"""Each parameter is a decision the model has to make blind."""
|
|
86
|
+
props = tool.properties()
|
|
87
|
+
if not props:
|
|
88
|
+
return None
|
|
89
|
+
missing = [
|
|
90
|
+
name
|
|
91
|
+
for name, spec in props.items()
|
|
92
|
+
if not (isinstance(spec, dict) and str(spec.get("description", "")).strip())
|
|
93
|
+
]
|
|
94
|
+
if missing:
|
|
95
|
+
preview = ", ".join(sorted(missing)[:5])
|
|
96
|
+
more = "" if len(missing) <= 5 else f" (+{len(missing) - 5} more)"
|
|
97
|
+
return Finding(
|
|
98
|
+
rule="AGT013",
|
|
99
|
+
severity=Severity.WARNING,
|
|
100
|
+
message=f"Tool {tool.name!r} has {len(missing)} parameter(s) without a description: {preview}{more}.",
|
|
101
|
+
tool=tool.name,
|
|
102
|
+
hint="Describe format and constraints, not just the type.",
|
|
103
|
+
)
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@tool_rule("AGT014", Severity.INFO, "Description does not state the return shape")
|
|
108
|
+
def no_return_shape(tool: Tool):
|
|
109
|
+
"""The model plans the *next* call from what it expects back.
|
|
110
|
+
|
|
111
|
+
Saying "returns a JSON array of {id, title}" is worth more than another
|
|
112
|
+
sentence about the parameters.
|
|
113
|
+
"""
|
|
114
|
+
text = tool.description.lower()
|
|
115
|
+
cues = ("return", "returns", "output", "responds", "yields", "gives back")
|
|
116
|
+
if len(text) >= MIN_DESCRIPTION and not any(c in text for c in cues):
|
|
117
|
+
return Finding(
|
|
118
|
+
rule="AGT014",
|
|
119
|
+
severity=Severity.INFO,
|
|
120
|
+
message=f"Tool {tool.name!r} description does not describe what it returns.",
|
|
121
|
+
tool=tool.name,
|
|
122
|
+
hint='Add "Returns ..." so the model can plan the next step.',
|
|
123
|
+
)
|
|
124
|
+
return None
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Prompt-injection rules for tool descriptions.
|
|
2
|
+
|
|
3
|
+
Tool descriptions are injected into the model's context verbatim. That makes
|
|
4
|
+
them an *input* channel controlled by whoever wrote the server - a supply-chain
|
|
5
|
+
surface that most MCP setups never audit. These rules flag the shapes that show
|
|
6
|
+
up in real attacks, not hypothetical ones.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from ..models import Finding, Severity, Tool
|
|
14
|
+
from ..tokens import has_invisible_chars
|
|
15
|
+
from .base import tool_rule
|
|
16
|
+
|
|
17
|
+
#: Instructions aimed at the model rather than the reader.
|
|
18
|
+
OVERRIDE_PATTERNS = (
|
|
19
|
+
re.compile(r"ignore\s+(all\s+)?(the\s+)?(previous|prior|above)\s+(instructions|prompts|rules)", re.I),
|
|
20
|
+
re.compile(r"disregard\s+(all\s+)?(the\s+)?(previous|prior|above)", re.I),
|
|
21
|
+
re.compile(r"forget\s+(everything|all)\s+(above|before|you)", re.I),
|
|
22
|
+
re.compile(r"you\s+are\s+now\s+(a|an|in)\b", re.I),
|
|
23
|
+
re.compile(r"new\s+(system\s+)?instructions?\s*:", re.I),
|
|
24
|
+
re.compile(r"system\s*prompt", re.I),
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
#: Requests to keep something hidden from the human.
|
|
28
|
+
CONCEALMENT_PATTERNS = (
|
|
29
|
+
re.compile(r"do\s+not\s+(tell|inform|mention|reveal)\s+(the\s+)?(user|human)", re.I),
|
|
30
|
+
re.compile(r"without\s+(telling|informing|notifying)\s+(the\s+)?user", re.I),
|
|
31
|
+
re.compile(r"(hide|conceal)\s+(this|it)\s+from", re.I),
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
#: Outbound data movement embedded in a description.
|
|
35
|
+
EXFIL_PATTERNS = (
|
|
36
|
+
re.compile(r"(send|post|upload|forward|exfiltrate)\s+.{0,40}\b(to|at)\s+https?://", re.I),
|
|
37
|
+
re.compile(r"https?://[^\s]+\?[^\s]*(key|token|secret|password)=", re.I),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
#: Tool-poisoning via markup the model may read but the reviewer skims past.
|
|
41
|
+
MARKUP_PATTERNS = (
|
|
42
|
+
re.compile(r"<!--.*?-->", re.S),
|
|
43
|
+
re.compile(r"<\|[^|]*\|>"), # chat-template control tokens
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@tool_rule("AGT040", Severity.ERROR, "Description tries to override model instructions")
|
|
48
|
+
def instruction_override(tool: Tool):
|
|
49
|
+
"""A legitimate description never talks to the model in the imperative-about-itself.
|
|
50
|
+
|
|
51
|
+
This is the classic tool-poisoning payload: the server author (or anyone who
|
|
52
|
+
compromised the package) rewrites the agent's priorities at load time.
|
|
53
|
+
"""
|
|
54
|
+
for pattern in OVERRIDE_PATTERNS:
|
|
55
|
+
match = pattern.search(tool.description)
|
|
56
|
+
if match:
|
|
57
|
+
return Finding(
|
|
58
|
+
rule="AGT040",
|
|
59
|
+
severity=Severity.ERROR,
|
|
60
|
+
message=f"Tool {tool.name!r} description contains an instruction-override phrase: {match.group(0)!r}.",
|
|
61
|
+
tool=tool.name,
|
|
62
|
+
hint="Rewrite the description to describe the tool, not to instruct the model.",
|
|
63
|
+
)
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@tool_rule("AGT041", Severity.ERROR, "Description asks the model to hide behaviour from the user")
|
|
68
|
+
def concealment(tool: Tool):
|
|
69
|
+
"""Anything the model is told to hide from its user is, by definition, not auditable."""
|
|
70
|
+
for pattern in CONCEALMENT_PATTERNS:
|
|
71
|
+
match = pattern.search(tool.description)
|
|
72
|
+
if match:
|
|
73
|
+
return Finding(
|
|
74
|
+
rule="AGT041",
|
|
75
|
+
severity=Severity.ERROR,
|
|
76
|
+
message=f"Tool {tool.name!r} description contains a concealment phrase: {match.group(0)!r}.",
|
|
77
|
+
tool=tool.name,
|
|
78
|
+
hint="Remove it. There is no legitimate reason to hide a tool call from the user.",
|
|
79
|
+
)
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@tool_rule("AGT042", Severity.WARNING, "Description embeds an outbound URL with credentials")
|
|
84
|
+
def exfiltration(tool: Tool):
|
|
85
|
+
"""Tool descriptions should not contain endpoints at all, let alone with tokens."""
|
|
86
|
+
for pattern in EXFIL_PATTERNS:
|
|
87
|
+
match = pattern.search(tool.description)
|
|
88
|
+
if match:
|
|
89
|
+
return Finding(
|
|
90
|
+
rule="AGT042",
|
|
91
|
+
severity=Severity.WARNING,
|
|
92
|
+
message=f"Tool {tool.name!r} description contains a suspicious outbound URL: {match.group(0)[:80]!r}.",
|
|
93
|
+
tool=tool.name,
|
|
94
|
+
hint="Endpoints belong in config, never in the model-visible description.",
|
|
95
|
+
)
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@tool_rule("AGT043", Severity.ERROR, "Description hides text with invisible characters")
|
|
100
|
+
def invisible_text(tool: Tool):
|
|
101
|
+
"""Zero-width and bidi characters let a description read one way and behave another."""
|
|
102
|
+
suspects = has_invisible_chars(tool.description)
|
|
103
|
+
if suspects:
|
|
104
|
+
return Finding(
|
|
105
|
+
rule="AGT043",
|
|
106
|
+
severity=Severity.ERROR,
|
|
107
|
+
message=f"Tool {tool.name!r} description contains invisible characters: {', '.join(suspects)}.",
|
|
108
|
+
tool=tool.name,
|
|
109
|
+
hint="Strip zero-width / bidi control characters; re-save the description as plain text.",
|
|
110
|
+
)
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@tool_rule("AGT044", Severity.WARNING, "Description hides instructions in markup")
|
|
115
|
+
def markup_hidden(tool: Tool):
|
|
116
|
+
"""HTML comments and chat-template tokens are invisible in most UIs."""
|
|
117
|
+
for pattern in MARKUP_PATTERNS:
|
|
118
|
+
match = pattern.search(tool.description)
|
|
119
|
+
if match:
|
|
120
|
+
return Finding(
|
|
121
|
+
rule="AGT044",
|
|
122
|
+
severity=Severity.WARNING,
|
|
123
|
+
message=f"Tool {tool.name!r} description contains markup the UI may hide: {match.group(0)[:60]!r}.",
|
|
124
|
+
tool=tool.name,
|
|
125
|
+
hint="Descriptions should be plain prose; no comments or template tokens.",
|
|
126
|
+
)
|
|
127
|
+
return None
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Naming rules: the cheapest class of bug to catch and the easiest to fix."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from ..models import Finding, Severity, Tool
|
|
8
|
+
from .base import tool_rule
|
|
9
|
+
|
|
10
|
+
ALLOWED = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
|
|
11
|
+
SNAKE = re.compile(r"^[a-z][a-z0-9]*(_[a-z0-9]+)*$")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@tool_rule(
|
|
15
|
+
"AGT001",
|
|
16
|
+
Severity.ERROR,
|
|
17
|
+
"Tool name is not a valid MCP identifier",
|
|
18
|
+
)
|
|
19
|
+
def name_charset(tool: Tool):
|
|
20
|
+
"""MCP tool names must be 1-64 chars of [A-Za-z0-9_-].
|
|
21
|
+
|
|
22
|
+
Anything else is rejected by strict clients and silently mangled by others,
|
|
23
|
+
which turns into a "tool not found" bug the model cannot recover from.
|
|
24
|
+
"""
|
|
25
|
+
if not tool.name:
|
|
26
|
+
return Finding(
|
|
27
|
+
rule="AGT001",
|
|
28
|
+
severity=Severity.ERROR,
|
|
29
|
+
message="Tool has an empty name.",
|
|
30
|
+
hint="Every tool needs a non-empty `name`.",
|
|
31
|
+
)
|
|
32
|
+
if not ALLOWED.match(tool.name):
|
|
33
|
+
return Finding(
|
|
34
|
+
rule="AGT001",
|
|
35
|
+
severity=Severity.ERROR,
|
|
36
|
+
message=f"Tool name {tool.name!r} contains characters outside [A-Za-z0-9_-] or exceeds 64 chars.",
|
|
37
|
+
tool=tool.name,
|
|
38
|
+
hint="Rename to something like `read_file` or `search-docs`.",
|
|
39
|
+
)
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@tool_rule(
|
|
44
|
+
"AGT002",
|
|
45
|
+
Severity.INFO,
|
|
46
|
+
"Tool name is not snake_case",
|
|
47
|
+
)
|
|
48
|
+
def name_style(tool: Tool):
|
|
49
|
+
"""Consistent casing measurably helps tool selection.
|
|
50
|
+
|
|
51
|
+
Models see the whole manifest at once; mixed `readFile` / `read_file` /
|
|
52
|
+
`ReadFile` makes near-duplicate names harder to disambiguate.
|
|
53
|
+
"""
|
|
54
|
+
if tool.name and not SNAKE.match(tool.name):
|
|
55
|
+
return Finding(
|
|
56
|
+
rule="AGT002",
|
|
57
|
+
severity=Severity.INFO,
|
|
58
|
+
message=f"Tool name {tool.name!r} is not snake_case.",
|
|
59
|
+
tool=tool.name,
|
|
60
|
+
hint="Prefer `read_file` over `readFile` / `ReadFile`.",
|
|
61
|
+
)
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
#: Prefixes that are almost always a server/domain name rather than a verb.
|
|
66
|
+
#: Deliberately a closed list: a "first segment has >= 3 chars" heuristic fires
|
|
67
|
+
#: on legitimate names like ``search_docs_by_tag`` and the false positives make
|
|
68
|
+
#: the whole report untrustworthy.
|
|
69
|
+
SERVER_PREFIXES = frozenset(
|
|
70
|
+
{
|
|
71
|
+
"github", "gitlab", "jira", "slack", "notion", "linear", "sentry",
|
|
72
|
+
"postgres", "mysql", "sqlite", "redis", "mongo", "elastic",
|
|
73
|
+
"browser", "playwright", "chrome", "puppeteer",
|
|
74
|
+
"filesystem", "files", "fs", "shell", "docker", "k8s", "kubernetes",
|
|
75
|
+
"google", "gmail", "calendar", "drive", "aws", "s3", "azure", "gcp",
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@tool_rule(
|
|
81
|
+
"AGT003",
|
|
82
|
+
Severity.INFO,
|
|
83
|
+
"Tool name encodes the server name as a prefix",
|
|
84
|
+
)
|
|
85
|
+
def name_redundant_prefix(tool: Tool):
|
|
86
|
+
"""`github_get_issue` inside the `github` server wastes context.
|
|
87
|
+
|
|
88
|
+
The client already namespaces by server, so the prefix is pure overhead on
|
|
89
|
+
every single request.
|
|
90
|
+
"""
|
|
91
|
+
head = tool.name.split("_", 1)[0].lower()
|
|
92
|
+
if tool.name.count("_") >= 2 and head in SERVER_PREFIXES:
|
|
93
|
+
return Finding(
|
|
94
|
+
rule="AGT003",
|
|
95
|
+
severity=Severity.INFO,
|
|
96
|
+
message=f"Tool name {tool.name!r} repeats the server/domain name as a prefix.",
|
|
97
|
+
tool=tool.name,
|
|
98
|
+
hint="The client already namespaces by server; drop the prefix to save context.",
|
|
99
|
+
)
|
|
100
|
+
return None
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Safety rules.
|
|
2
|
+
|
|
3
|
+
Annotations exist so a host can gate destructive tools behind a confirmation
|
|
4
|
+
prompt. Missing or contradictory annotations silently disable that gate, which
|
|
5
|
+
is how an agent ends up deleting production data with no dialog box.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from ..models import Finding, Severity, Tool
|
|
11
|
+
from .base import tool_rule
|
|
12
|
+
|
|
13
|
+
DESTRUCTIVE_VERBS = (
|
|
14
|
+
"delete", "remove", "drop", "destroy", "truncate", "purge", "wipe",
|
|
15
|
+
"kill", "terminate", "reset", "overwrite", "revoke",
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
EXEC_VERBS = ("exec", "execute", "shell", "run_command", "spawn", "eval", "system")
|
|
19
|
+
|
|
20
|
+
PATH_PARAMS = ("path", "file", "dir", "directory", "folder", "root", "cwd", "glob")
|
|
21
|
+
|
|
22
|
+
WRITE_VERBS = ("write", "create", "update", "edit", "patch", "put", "post", "append", "insert")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _looks_like(tool: Tool, verbs: tuple[str, ...]) -> str | None:
|
|
26
|
+
name = tool.name.lower()
|
|
27
|
+
for verb in verbs:
|
|
28
|
+
if name == verb or name.startswith(verb + "_") or f"_{verb}" in name:
|
|
29
|
+
return verb
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@tool_rule("AGT030", Severity.WARNING, "Destructive tool does not declare destructiveHint")
|
|
34
|
+
def destructive_without_hint(tool: Tool):
|
|
35
|
+
"""Hosts cannot ask "are you sure?" for a tool that does not admit it is risky."""
|
|
36
|
+
verb = _looks_like(tool, DESTRUCTIVE_VERBS)
|
|
37
|
+
if not verb:
|
|
38
|
+
return None
|
|
39
|
+
if tool.annotations.get("destructiveHint") is True:
|
|
40
|
+
return None
|
|
41
|
+
return Finding(
|
|
42
|
+
rule="AGT030",
|
|
43
|
+
severity=Severity.WARNING,
|
|
44
|
+
message=f"Tool {tool.name!r} looks destructive (verb: {verb}) but sets no destructiveHint.",
|
|
45
|
+
tool=tool.name,
|
|
46
|
+
hint='Add "annotations": {"destructiveHint": true} so hosts can require confirmation.',
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@tool_rule("AGT031", Severity.WARNING, "readOnlyHint contradicts the tool name")
|
|
51
|
+
def readonly_contradiction(tool: Tool):
|
|
52
|
+
"""A wrong ``readOnlyHint: true`` is worse than no hint.
|
|
53
|
+
|
|
54
|
+
Hosts use it to skip confirmation and to allow parallel calls.
|
|
55
|
+
"""
|
|
56
|
+
if tool.annotations.get("readOnlyHint") is not True:
|
|
57
|
+
return None
|
|
58
|
+
verb = _looks_like(tool, DESTRUCTIVE_VERBS + WRITE_VERBS)
|
|
59
|
+
if not verb:
|
|
60
|
+
return None
|
|
61
|
+
return Finding(
|
|
62
|
+
rule="AGT031",
|
|
63
|
+
severity=Severity.WARNING,
|
|
64
|
+
message=f"Tool {tool.name!r} declares readOnlyHint: true but its name implies '{verb}'.",
|
|
65
|
+
tool=tool.name,
|
|
66
|
+
hint="Either the name or the annotation is wrong; hosts trust the annotation.",
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@tool_rule("AGT032", Severity.ERROR, "Tool exposes unconstrained command execution")
|
|
71
|
+
def unconstrained_exec(tool: Tool):
|
|
72
|
+
"""An `exec`-style tool with a free-form string is remote code execution by design.
|
|
73
|
+
|
|
74
|
+
It is sometimes intentional (a local shell server), but it must be a
|
|
75
|
+
deliberate, documented decision rather than a default.
|
|
76
|
+
"""
|
|
77
|
+
verb = _looks_like(tool, EXEC_VERBS)
|
|
78
|
+
if not verb:
|
|
79
|
+
return None
|
|
80
|
+
for name, spec in tool.properties().items():
|
|
81
|
+
if not isinstance(spec, dict):
|
|
82
|
+
continue
|
|
83
|
+
if spec.get("type") != "string":
|
|
84
|
+
continue
|
|
85
|
+
if not any(k in spec for k in ("enum", "pattern", "const")):
|
|
86
|
+
return Finding(
|
|
87
|
+
rule="AGT032",
|
|
88
|
+
severity=Severity.ERROR,
|
|
89
|
+
message=f"Tool {tool.name!r} executes commands from an unconstrained string param {name!r}.",
|
|
90
|
+
tool=tool.name,
|
|
91
|
+
hint="Allow-list commands via `enum`, or drop the param and expose fixed operations.",
|
|
92
|
+
)
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@tool_rule("AGT033", Severity.INFO, "Filesystem parameter has no documented scope")
|
|
97
|
+
def unscoped_path(tool: Tool):
|
|
98
|
+
"""Without a stated root, the model will happily read ``/etc/shadow``.
|
|
99
|
+
|
|
100
|
+
Documenting the sandbox root is cheap and makes the boundary testable.
|
|
101
|
+
"""
|
|
102
|
+
for name, spec in tool.properties().items():
|
|
103
|
+
if not isinstance(spec, dict):
|
|
104
|
+
continue
|
|
105
|
+
if not any(p in name.lower() for p in PATH_PARAMS):
|
|
106
|
+
continue
|
|
107
|
+
desc = str(spec.get("description", "")).lower()
|
|
108
|
+
if any(k in desc for k in ("root", "sandbox", "within", "relative", "base dir", "allowed")):
|
|
109
|
+
continue
|
|
110
|
+
return Finding(
|
|
111
|
+
rule="AGT033",
|
|
112
|
+
severity=Severity.INFO,
|
|
113
|
+
message=f"Tool {tool.name!r} path param {name!r} does not state its allowed root.",
|
|
114
|
+
tool=tool.name,
|
|
115
|
+
hint='Say e.g. "Relative to the sandbox root; absolute paths are rejected."',
|
|
116
|
+
)
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@tool_rule("AGT034", Severity.INFO, "Tool declares no annotations at all")
|
|
121
|
+
def no_annotations(tool: Tool):
|
|
122
|
+
"""No hints means the host must assume the worst case for every call."""
|
|
123
|
+
if tool.annotations:
|
|
124
|
+
return None
|
|
125
|
+
return Finding(
|
|
126
|
+
rule="AGT034",
|
|
127
|
+
severity=Severity.INFO,
|
|
128
|
+
message=f"Tool {tool.name!r} has no annotations.",
|
|
129
|
+
tool=tool.name,
|
|
130
|
+
hint='Declare at least {"readOnlyHint": true/false} and "destructiveHint" where relevant.',
|
|
131
|
+
)
|