ghostrules 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ghostrules/__init__.py +6 -0
- ghostrules/_compat.py +12 -0
- ghostrules/clean.py +41 -0
- ghostrules/cli.py +319 -0
- ghostrules/config.py +93 -0
- ghostrules/jsonloc.py +194 -0
- ghostrules/markdown_utils.py +59 -0
- ghostrules/mcp_config.py +243 -0
- ghostrules/models.py +188 -0
- ghostrules/py.typed +0 -0
- ghostrules/reporters/__init__.py +1 -0
- ghostrules/reporters/common.py +86 -0
- ghostrules/reporters/json_.py +45 -0
- ghostrules/reporters/markdown.py +55 -0
- ghostrules/reporters/sarif.py +137 -0
- ghostrules/reporters/table.py +235 -0
- ghostrules/reveal.py +90 -0
- ghostrules/rules/__init__.py +16 -0
- ghostrules/rules/agent_settings.py +159 -0
- ghostrules/rules/helpers.py +118 -0
- ghostrules/rules/injection.py +317 -0
- ghostrules/rules/mcp.py +445 -0
- ghostrules/rules/patterns.py +187 -0
- ghostrules/rules/registry.py +78 -0
- ghostrules/rules/unicode.py +503 -0
- ghostrules/scanner.py +244 -0
- ghostrules/secrets.py +219 -0
- ghostrules/targets.py +220 -0
- ghostrules/unicode_utils.py +138 -0
- ghostrules-0.1.0.dist-info/METADATA +333 -0
- ghostrules-0.1.0.dist-info/RECORD +34 -0
- ghostrules-0.1.0.dist-info/WHEEL +4 -0
- ghostrules-0.1.0.dist-info/entry_points.txt +2 -0
- ghostrules-0.1.0.dist-info/licenses/LICENSE +21 -0
ghostrules/scanner.py
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""The scan pipeline: discover -> load -> run rules -> suppress -> decode pass -> filter -> sort."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
from dataclasses import replace
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from ghostrules._compat import tomllib
|
|
13
|
+
from ghostrules.config import Config
|
|
14
|
+
from ghostrules.jsonloc import strip_jsonc
|
|
15
|
+
from ghostrules.models import Finding, ScanResult, ScanTarget
|
|
16
|
+
from ghostrules.rules import Rule, all_rules
|
|
17
|
+
from ghostrules.secrets import mask_secrets
|
|
18
|
+
from ghostrules.targets import Candidate, discover, home_targets
|
|
19
|
+
|
|
20
|
+
# Invisible-character findings must never be silenceable from inside the scanned file.
|
|
21
|
+
NEVER_SUPPRESS: frozenset[str] = frozenset({"GR001", "GR002"})
|
|
22
|
+
|
|
23
|
+
_IGNORE_RE = re.compile(r"ghostrules-ignore(?:\s*:\s*((?:GR\d{3}(?:\s*,\s*)?)+))?", re.IGNORECASE)
|
|
24
|
+
_RULE_ID_RE = re.compile(r"GR\d{3}", re.IGNORECASE)
|
|
25
|
+
|
|
26
|
+
# Maps a 1-based line number to the rule ids it ignores; ``None`` means every rule.
|
|
27
|
+
Suppressions = dict[int, "frozenset[str] | None"]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_structured(path: str, text: str) -> tuple[Any, str | None]:
|
|
31
|
+
"""Parse JSON (tolerating JSONC) or TOML by extension. Returns ``(parsed, error)``."""
|
|
32
|
+
lowered = path.lower()
|
|
33
|
+
try:
|
|
34
|
+
if lowered.endswith(".json"):
|
|
35
|
+
body = text.removeprefix("\ufeff")
|
|
36
|
+
try:
|
|
37
|
+
return json.loads(body), None
|
|
38
|
+
except json.JSONDecodeError:
|
|
39
|
+
return json.loads(strip_jsonc(body)), None
|
|
40
|
+
if lowered.endswith(".toml"):
|
|
41
|
+
return tomllib.loads(text.removeprefix("\ufeff")), None
|
|
42
|
+
except (json.JSONDecodeError, tomllib.TOMLDecodeError, RecursionError) as exc:
|
|
43
|
+
return None, str(exc)
|
|
44
|
+
return None, None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def load_target(
|
|
48
|
+
candidate: Candidate, label: str | None = None, allow_domains: tuple[str, ...] = ()
|
|
49
|
+
) -> ScanTarget:
|
|
50
|
+
"""Read a candidate as UTF-8 (``surrogateescape``, so bad bytes never crash) and parse it."""
|
|
51
|
+
data = candidate.abs_path.read_bytes()
|
|
52
|
+
text = data.decode("utf-8", errors="surrogateescape")
|
|
53
|
+
path = label or candidate.rel
|
|
54
|
+
parsed, error = (None, None) if candidate.kind == "other" else parse_structured(path, text)
|
|
55
|
+
return ScanTarget(
|
|
56
|
+
path=path,
|
|
57
|
+
kind=candidate.kind,
|
|
58
|
+
text=text,
|
|
59
|
+
parsed=parsed,
|
|
60
|
+
parse_error=error,
|
|
61
|
+
allow_domains=allow_domains,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def parse_suppressions(lines: Iterable[str]) -> Suppressions:
|
|
66
|
+
"""Find ``ghostrules-ignore[: GR010,GR012]`` markers; each covers its line and the next."""
|
|
67
|
+
table: Suppressions = {}
|
|
68
|
+
for number, line in enumerate(lines, start=1):
|
|
69
|
+
match = _IGNORE_RE.search(line)
|
|
70
|
+
if not match:
|
|
71
|
+
continue
|
|
72
|
+
ids = match.group(1)
|
|
73
|
+
scope = frozenset(m.upper() for m in _RULE_ID_RE.findall(ids)) if ids else None
|
|
74
|
+
for covered in (number, number + 1):
|
|
75
|
+
existing = table.get(covered, frozenset())
|
|
76
|
+
if scope is None or existing is None:
|
|
77
|
+
table[covered] = None
|
|
78
|
+
else:
|
|
79
|
+
table[covered] = existing | scope
|
|
80
|
+
return table
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def is_suppressed(finding: Finding, table: Suppressions) -> bool:
|
|
84
|
+
"""Whether an inline marker silences this finding (never GR001/GR002)."""
|
|
85
|
+
if finding.rule_id in NEVER_SUPPRESS or finding.line not in table:
|
|
86
|
+
return False
|
|
87
|
+
scope = table[finding.line]
|
|
88
|
+
return scope is None or finding.rule_id in scope
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _applicable(rules: Iterable[Rule], config: Config) -> list[Rule]:
|
|
92
|
+
return [r for r in rules if r.id not in config.disable]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _run_rules(target: ScanTarget, rules: Iterable[Rule], errors: list[str]) -> list[Finding]:
|
|
96
|
+
"""Run applicable rules; a crashing rule is recorded in ``errors`` rather than aborting."""
|
|
97
|
+
found: list[Finding] = []
|
|
98
|
+
for r in rules:
|
|
99
|
+
if target.kind not in r.applies_to:
|
|
100
|
+
continue
|
|
101
|
+
try:
|
|
102
|
+
found.extend(r.check(target))
|
|
103
|
+
except Exception as exc: # a buggy rule must not hide every other finding
|
|
104
|
+
detail = mask_secrets(f"{type(exc).__name__}: {exc}")
|
|
105
|
+
errors.append(f"rule {r.id} failed on {target.path}: {detail}")
|
|
106
|
+
return found
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _scan_decoded(
|
|
110
|
+
target: ScanTarget, parents: Iterable[Finding], rules: Iterable[Rule], errors: list[str]
|
|
111
|
+
) -> list[Finding]:
|
|
112
|
+
"""Run ``scans_decoded`` rules over hidden text recovered by other rules.
|
|
113
|
+
|
|
114
|
+
Results are attributed to the parent's location, carry the decoded text, and get +1 severity.
|
|
115
|
+
"""
|
|
116
|
+
decoding_rules = [r for r in rules if r.scans_decoded]
|
|
117
|
+
derived: list[Finding] = []
|
|
118
|
+
for parent in parents:
|
|
119
|
+
if not parent.decoded:
|
|
120
|
+
continue
|
|
121
|
+
virtual = ScanTarget(
|
|
122
|
+
path=target.path,
|
|
123
|
+
kind="instructions",
|
|
124
|
+
text=parent.decoded,
|
|
125
|
+
origin=parent.rule_id,
|
|
126
|
+
allow_domains=target.allow_domains,
|
|
127
|
+
)
|
|
128
|
+
for hit in _run_rules(virtual, decoding_rules, errors):
|
|
129
|
+
derived.append(
|
|
130
|
+
replace(
|
|
131
|
+
hit,
|
|
132
|
+
severity=hit.severity.bumped(),
|
|
133
|
+
line=parent.line,
|
|
134
|
+
column=parent.column,
|
|
135
|
+
end_column=parent.end_column,
|
|
136
|
+
snippet=parent.snippet,
|
|
137
|
+
decoded=parent.decoded,
|
|
138
|
+
message=f"{hit.message} (found in hidden text decoded by {parent.rule_id})",
|
|
139
|
+
)
|
|
140
|
+
)
|
|
141
|
+
return derived
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def scan_target(
|
|
145
|
+
target: ScanTarget, config: Config, rules: Iterable[Rule]
|
|
146
|
+
) -> tuple[list[Finding], int, list[str]]:
|
|
147
|
+
"""Scan one target. Returns ``(findings, suppressed_count, errors)``.
|
|
148
|
+
|
|
149
|
+
Findings derived from decoded hidden text are added after suppression is applied, so an
|
|
150
|
+
inline ignore comment can never silence something that was hidden in invisible characters.
|
|
151
|
+
"""
|
|
152
|
+
errors: list[str] = []
|
|
153
|
+
active = _applicable(rules, config)
|
|
154
|
+
table = parse_suppressions(target.lines)
|
|
155
|
+
kept: list[Finding] = []
|
|
156
|
+
suppressed = 0
|
|
157
|
+
for finding in _run_rules(target, active, errors):
|
|
158
|
+
if is_suppressed(finding, table):
|
|
159
|
+
suppressed += 1
|
|
160
|
+
else:
|
|
161
|
+
kept.append(finding)
|
|
162
|
+
kept.extend(_scan_decoded(target, kept, active, errors))
|
|
163
|
+
return kept, suppressed, errors
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _dedupe(findings: Iterable[Finding]) -> list[Finding]:
|
|
167
|
+
unique: dict[tuple[object, ...], Finding] = {}
|
|
168
|
+
for f in findings:
|
|
169
|
+
key = (f.rule_id, f.path, f.line, f.column, f.end_column, f.title, f.decoded)
|
|
170
|
+
unique.setdefault(key, f)
|
|
171
|
+
return list(unique.values())
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def sort_findings(findings: Iterable[Finding]) -> list[Finding]:
|
|
175
|
+
"""Order by severity (most severe first), hidden-text findings first among equals, then
|
|
176
|
+
files closer to the root, path, line, column and rule id."""
|
|
177
|
+
return sorted(
|
|
178
|
+
findings,
|
|
179
|
+
key=lambda f: (
|
|
180
|
+
-int(f.severity),
|
|
181
|
+
not f.decoded,
|
|
182
|
+
f.path.count("/"),
|
|
183
|
+
f.path,
|
|
184
|
+
f.line,
|
|
185
|
+
f.column,
|
|
186
|
+
f.rule_id,
|
|
187
|
+
),
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def scan_paths(
|
|
192
|
+
paths: Iterable[Path],
|
|
193
|
+
config: Config,
|
|
194
|
+
*,
|
|
195
|
+
all_files: bool = False,
|
|
196
|
+
include_home: bool = False,
|
|
197
|
+
rules: Iterable[Rule] | None = None,
|
|
198
|
+
home_dir: Path | None = None,
|
|
199
|
+
) -> ScanResult:
|
|
200
|
+
"""Scan one or more files/directories (plus user-level configs with ``include_home``).
|
|
201
|
+
|
|
202
|
+
With a single root, reported paths are relative to it; with several, each path is prefixed
|
|
203
|
+
by the root as given so entries stay unambiguous.
|
|
204
|
+
"""
|
|
205
|
+
rule_list = list(all_rules() if rules is None else rules)
|
|
206
|
+
roots = list(paths)
|
|
207
|
+
result = ScanResult(config_source=config.source)
|
|
208
|
+
|
|
209
|
+
candidates: list[tuple[Candidate, str]] = []
|
|
210
|
+
for root in roots:
|
|
211
|
+
if not root.exists():
|
|
212
|
+
result.errors.append(f"path not found: {root}")
|
|
213
|
+
continue
|
|
214
|
+
for cand in discover(
|
|
215
|
+
root, all_files=all_files, exclude=config.exclude, extra_targets=config.extra_targets
|
|
216
|
+
):
|
|
217
|
+
label = cand.rel if len(roots) == 1 or root.is_file() else (root / cand.rel).as_posix()
|
|
218
|
+
candidates.append((cand, label))
|
|
219
|
+
if include_home:
|
|
220
|
+
candidates.extend((cand, cand.rel) for cand in home_targets(home_dir))
|
|
221
|
+
|
|
222
|
+
seen: set[Path] = set()
|
|
223
|
+
findings: list[Finding] = []
|
|
224
|
+
for cand, label in candidates:
|
|
225
|
+
resolved = cand.abs_path.resolve()
|
|
226
|
+
if resolved in seen:
|
|
227
|
+
continue
|
|
228
|
+
seen.add(resolved)
|
|
229
|
+
try:
|
|
230
|
+
target = load_target(cand, label, config.allow_domains)
|
|
231
|
+
except OSError as exc:
|
|
232
|
+
result.warnings.append(f"cannot read {label}: {exc.strerror or exc}")
|
|
233
|
+
continue
|
|
234
|
+
result.scanned_files += 1
|
|
235
|
+
if target.parse_error:
|
|
236
|
+
result.warnings.append(mask_secrets(f"cannot parse {label}: {target.parse_error}"))
|
|
237
|
+
kept, suppressed, errors = scan_target(target, config, rule_list)
|
|
238
|
+
findings.extend(kept)
|
|
239
|
+
result.suppressed += suppressed
|
|
240
|
+
result.errors.extend(errors)
|
|
241
|
+
|
|
242
|
+
visible = [f for f in _dedupe(findings) if f.severity >= config.min_severity]
|
|
243
|
+
result.findings = sort_findings(visible)
|
|
244
|
+
return result
|
ghostrules/secrets.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Secret detection and masking. Every piece of output goes through this module.
|
|
2
|
+
|
|
3
|
+
Masking rule: a secret of at least 20 characters is shown as its first 4 and last 4 characters
|
|
4
|
+
around ``****…`` (``ghp_****…a1b2``); shorter ones as ``****``. ``mask_secrets`` finds secrets in
|
|
5
|
+
arbitrary text (known token formats, bearer tokens, URL passwords, and ``key=value`` pairs whose
|
|
6
|
+
key name looks secret and whose value looks like one). It is idempotent. ``Finding`` applies it to
|
|
7
|
+
all of its text fields, so no reporter can print an unmasked secret.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
import re
|
|
14
|
+
from collections import Counter
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
from typing import NamedTuple
|
|
17
|
+
|
|
18
|
+
_MASK_MIN_LENGTH = 20
|
|
19
|
+
_MIN_GENERIC_LENGTH = 16
|
|
20
|
+
_MIN_GENERIC_ENTROPY = 3.5
|
|
21
|
+
|
|
22
|
+
# (label, pattern). The named group ``s`` is the secret part; no group means the whole match.
|
|
23
|
+
_KNOWN_SOURCES: tuple[tuple[str, str], ...] = (
|
|
24
|
+
("GitHub token", r"(?<![A-Za-z0-9])gh[pousr]_[A-Za-z0-9]{36,255}"),
|
|
25
|
+
("GitHub fine-grained token", r"(?<![A-Za-z0-9])github_pat_[A-Za-z0-9_]{22,255}"),
|
|
26
|
+
("Anthropic API key", r"(?<![A-Za-z0-9])sk-ant-[A-Za-z0-9_-]{20,}"),
|
|
27
|
+
("OpenAI API key", r"(?<![A-Za-z0-9])sk-(?:proj-)?[A-Za-z0-9_-]{20,}"),
|
|
28
|
+
("AWS access key", r"(?<![A-Za-z0-9])(?:AKIA|ASIA)[0-9A-Z]{16}"),
|
|
29
|
+
("Slack token", r"(?<![A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]{10,}"),
|
|
30
|
+
("Google API key", r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}"),
|
|
31
|
+
("GitLab token", r"(?<![A-Za-z0-9])glpat-[0-9A-Za-z_-]{20,}"),
|
|
32
|
+
("Hugging Face token", r"(?<![A-Za-z0-9])hf_[A-Za-z0-9]{30,}"),
|
|
33
|
+
("npm token", r"(?<![A-Za-z0-9])npm_[A-Za-z0-9]{36,}"),
|
|
34
|
+
(
|
|
35
|
+
"Slack webhook",
|
|
36
|
+
r"https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/(?P<s>[A-Za-z0-9*…]+)",
|
|
37
|
+
),
|
|
38
|
+
(
|
|
39
|
+
"Discord webhook",
|
|
40
|
+
r"https://(?:ptb\.|canary\.)?discord(?:app)?\.com/api/(?:v\d+/)?webhooks/\d+/"
|
|
41
|
+
r"(?P<s>[A-Za-z0-9_*…-]+)",
|
|
42
|
+
),
|
|
43
|
+
("Bearer token", r"Bearer\s+(?P<s>[A-Za-z0-9._~+/-]{20,}={0,2})"),
|
|
44
|
+
)
|
|
45
|
+
_BOUNDARY = r"(?<![A-Za-z0-9])"
|
|
46
|
+
KNOWN_SECRET_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
|
|
47
|
+
(label, re.compile(pattern)) for label, pattern in _KNOWN_SOURCES
|
|
48
|
+
)
|
|
49
|
+
# For masking, secrets glued to other letters ("xghp_...") must still be hidden, so the
|
|
50
|
+
# word-boundary guard used for *detection* is dropped. The ambiguous OpenAI prefix ("sk-" also ends
|
|
51
|
+
# words like "task-") then needs a digit and some entropy to count.
|
|
52
|
+
_MASK_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
|
|
53
|
+
(label, re.compile(pattern.replace(_BOUNDARY, ""))) for label, pattern in _KNOWN_SOURCES
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
_NAME_WORDS = r"(?:token|secret|key|password|passwd|auth)"
|
|
57
|
+
_SECRET_NAME_RE = re.compile(_NAME_WORDS, re.IGNORECASE)
|
|
58
|
+
_VALUE_CHARS = r"[^\s\"'`,;&)}\]]"
|
|
59
|
+
_KEY_VALUE_RES = (
|
|
60
|
+
# key = value / "key": "value" / key: value
|
|
61
|
+
re.compile(
|
|
62
|
+
rf"(?<![\w.-])(?P<k>[\"']?[\w.-]*{_NAME_WORDS}[\w.-]*[\"']?\s*[:=]\s*[\"']?)"
|
|
63
|
+
rf"(?P<s>{_VALUE_CHARS}{{16,}})",
|
|
64
|
+
re.IGNORECASE,
|
|
65
|
+
),
|
|
66
|
+
# --api-token VALUE
|
|
67
|
+
re.compile(
|
|
68
|
+
rf"(?<![\w.-])(?P<k>--[\w-]*{_NAME_WORDS}[\w-]*\s+[\"']?)(?P<s>{_VALUE_CHARS}{{16,}})",
|
|
69
|
+
re.IGNORECASE,
|
|
70
|
+
),
|
|
71
|
+
)
|
|
72
|
+
_URL_PASSWORD_RE = re.compile(r"://[^/\s:@]+:(?P<s>[^@\s/]{3,})@")
|
|
73
|
+
_PLACEHOLDER_WORDS_RE = re.compile(
|
|
74
|
+
r"(?:your|my)[-_ ].*(?:here|key|token|secret|password)"
|
|
75
|
+
r"|x{4,}|\*{4,}|\.{3,}|changeme|placeholder|redacted|example-?token|todo",
|
|
76
|
+
re.IGNORECASE,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class Replacement(NamedTuple):
|
|
81
|
+
"""Replace ``text[start:end]`` by ``text``."""
|
|
82
|
+
|
|
83
|
+
start: int
|
|
84
|
+
end: int
|
|
85
|
+
text: str
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def shannon_entropy(text: str) -> float:
|
|
89
|
+
"""Shannon entropy in bits per character."""
|
|
90
|
+
if not text:
|
|
91
|
+
return 0.0
|
|
92
|
+
total = len(text)
|
|
93
|
+
return -sum(n / total * math.log2(n / total) for n in Counter(text).values())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def mask_value(value: str) -> str:
|
|
97
|
+
"""Mask one secret: ``ghp_****…a1b2`` for long values, ``****`` for short ones."""
|
|
98
|
+
if len(value) < _MASK_MIN_LENGTH:
|
|
99
|
+
return "****"
|
|
100
|
+
return f"{value[:4]}****…{value[-4:]}"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def is_placeholder(value: str) -> bool:
|
|
104
|
+
"""True for variable references (``${VAR}``, ``<token>``) and dummy values."""
|
|
105
|
+
s = value.strip()
|
|
106
|
+
return (
|
|
107
|
+
not s
|
|
108
|
+
or "${" in s
|
|
109
|
+
or "{{" in s
|
|
110
|
+
or s.startswith(("$", "<"))
|
|
111
|
+
or (s.startswith("%") and s.endswith("%"))
|
|
112
|
+
or len(set(s)) == 1
|
|
113
|
+
or "****" in s
|
|
114
|
+
or _PLACEHOLDER_WORDS_RE.fullmatch(s) is not None
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def is_secret_key_name(name: str) -> bool:
|
|
119
|
+
"""Whether a key/flag/variable name suggests it holds a secret."""
|
|
120
|
+
return _SECRET_NAME_RE.search(name) is not None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def looks_like_secret_value(value: str) -> bool:
|
|
124
|
+
"""Heuristic for a generic secret: long, high-entropy, no spaces, not a placeholder/URL/path."""
|
|
125
|
+
if (
|
|
126
|
+
len(value) < _MIN_GENERIC_LENGTH
|
|
127
|
+
or any(c.isspace() for c in value)
|
|
128
|
+
or is_placeholder(value)
|
|
129
|
+
or value.startswith(("/", "~", "./", "../"))
|
|
130
|
+
or re.match(r"[A-Za-z][A-Za-z0-9+.-]*://", value)
|
|
131
|
+
or re.match(r"[A-Za-z]:[\\/]", value)
|
|
132
|
+
):
|
|
133
|
+
return False
|
|
134
|
+
return shannon_entropy(value) >= _MIN_GENERIC_ENTROPY
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def known_secret(text: str) -> tuple[str, str] | None:
|
|
138
|
+
"""The first known-format secret in ``text`` as ``(label, secret)``, or ``None``."""
|
|
139
|
+
for label, pattern in KNOWN_SECRET_PATTERNS:
|
|
140
|
+
group = "s" if "s" in pattern.groupindex else 0
|
|
141
|
+
for m in pattern.finditer(text):
|
|
142
|
+
if not is_placeholder(m.group(group)):
|
|
143
|
+
return label, m.group(group)
|
|
144
|
+
return None
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _plausible_glued_key(value: str) -> bool:
|
|
148
|
+
return any(c.isdigit() for c in value) and shannon_entropy(value) >= _MIN_GENERIC_ENTROPY
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _candidates(text: str, extra: Iterable[str]) -> list[Replacement]:
|
|
152
|
+
found: list[Replacement] = []
|
|
153
|
+
for label, pattern in _MASK_PATTERNS:
|
|
154
|
+
for m in pattern.finditer(text):
|
|
155
|
+
group = "s" if "s" in pattern.groupindex else 0
|
|
156
|
+
value = m.group(group)
|
|
157
|
+
if is_placeholder(value):
|
|
158
|
+
continue
|
|
159
|
+
glued = m.start(group) > 0 and text[m.start(group) - 1].isalnum()
|
|
160
|
+
if label == "OpenAI API key" and glued and not _plausible_glued_key(value):
|
|
161
|
+
continue
|
|
162
|
+
found.append(Replacement(m.start(group), m.end(group), mask_value(value)))
|
|
163
|
+
for m in _URL_PASSWORD_RE.finditer(text):
|
|
164
|
+
found.append(Replacement(m.start("s"), m.end("s"), "****"))
|
|
165
|
+
for pattern in _KEY_VALUE_RES:
|
|
166
|
+
for m in pattern.finditer(text):
|
|
167
|
+
if looks_like_secret_value(m.group("s")):
|
|
168
|
+
found.append(Replacement(m.start("s"), m.end("s"), mask_value(m.group("s"))))
|
|
169
|
+
for secret in extra:
|
|
170
|
+
if secret:
|
|
171
|
+
for m in re.finditer(re.escape(secret), text):
|
|
172
|
+
found.append(Replacement(m.start(), m.end(), mask_value(secret)))
|
|
173
|
+
return found
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def mask_spans(text: str, extra: Iterable[str] = ()) -> list[Replacement]:
|
|
177
|
+
"""Non-overlapping replacements masking every secret in ``text``.
|
|
178
|
+
|
|
179
|
+
``extra`` lists exact secret values the caller already knows (for example a generic secret
|
|
180
|
+
found by a rule) that must be masked wherever they occur.
|
|
181
|
+
"""
|
|
182
|
+
kept: list[Replacement] = []
|
|
183
|
+
for rep in sorted(_candidates(text, tuple(extra)), key=lambda r: (r.start, -r.end)):
|
|
184
|
+
if not kept or rep.start >= kept[-1].end:
|
|
185
|
+
kept.append(rep)
|
|
186
|
+
return kept
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def apply_spans(text: str, spans: Iterable[Replacement]) -> str:
|
|
190
|
+
"""Apply replacements produced by ``mask_spans``."""
|
|
191
|
+
out: list[str] = []
|
|
192
|
+
pos = 0
|
|
193
|
+
for rep in spans:
|
|
194
|
+
out.append(text[pos : rep.start])
|
|
195
|
+
out.append(rep.text)
|
|
196
|
+
pos = rep.end
|
|
197
|
+
out.append(text[pos:])
|
|
198
|
+
return "".join(out)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def map_offset(spans: Iterable[Replacement], offset: int, *, end: bool = False) -> int:
|
|
202
|
+
"""Translate an offset in the original text to the masked text.
|
|
203
|
+
|
|
204
|
+
An offset inside a replaced secret snaps to the start of its replacement (or to its end when
|
|
205
|
+
``end`` is true), so a span covering a secret still covers its mask.
|
|
206
|
+
"""
|
|
207
|
+
shift = 0
|
|
208
|
+
for rep in spans:
|
|
209
|
+
if offset <= rep.start:
|
|
210
|
+
break
|
|
211
|
+
if offset < rep.end:
|
|
212
|
+
return rep.start + shift + (len(rep.text) if end else 0)
|
|
213
|
+
shift += len(rep.text) - (rep.end - rep.start)
|
|
214
|
+
return offset + shift
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def mask_secrets(text: str, extra: Iterable[str] = ()) -> str:
|
|
218
|
+
"""Return ``text`` with every secret masked."""
|
|
219
|
+
return apply_spans(text, mask_spans(text, extra))
|
ghostrules/targets.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Which files ghostrules scans, and how it finds them. Extend the glob lists below."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
from collections.abc import Iterable, Mapping
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from functools import cache
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from ghostrules.models import Kind
|
|
14
|
+
|
|
15
|
+
INSTRUCTION_GLOBS: tuple[str, ...] = (
|
|
16
|
+
"AGENTS.md",
|
|
17
|
+
"**/AGENTS.md",
|
|
18
|
+
"AGENT.md",
|
|
19
|
+
"CLAUDE.md",
|
|
20
|
+
"CLAUDE.local.md",
|
|
21
|
+
"**/CLAUDE.md",
|
|
22
|
+
".claude/commands/**/*.md",
|
|
23
|
+
".claude/agents/**/*.md",
|
|
24
|
+
".claude/skills/**/*.md",
|
|
25
|
+
"**/SKILL.md",
|
|
26
|
+
".cursorrules",
|
|
27
|
+
".cursor/rules/**/*.mdc",
|
|
28
|
+
".cursor/rules/**/*.md",
|
|
29
|
+
".github/copilot-instructions.md",
|
|
30
|
+
".github/instructions/**/*.instructions.md",
|
|
31
|
+
".github/prompts/**/*.prompt.md",
|
|
32
|
+
".github/chatmodes/**/*.md",
|
|
33
|
+
".windsurfrules",
|
|
34
|
+
".windsurf/rules/**",
|
|
35
|
+
".clinerules",
|
|
36
|
+
".clinerules/**",
|
|
37
|
+
".roo/rules*/**",
|
|
38
|
+
".roorules",
|
|
39
|
+
"GEMINI.md",
|
|
40
|
+
".gemini/**/*.md",
|
|
41
|
+
".junie/guidelines.md",
|
|
42
|
+
".kiro/steering/**/*.md",
|
|
43
|
+
".continue/rules/**",
|
|
44
|
+
"CONVENTIONS.md",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
CONFIG_GLOBS: tuple[str, ...] = (
|
|
48
|
+
".mcp.json",
|
|
49
|
+
"mcp.json",
|
|
50
|
+
".cursor/mcp.json",
|
|
51
|
+
".vscode/mcp.json",
|
|
52
|
+
".gemini/settings.json",
|
|
53
|
+
"claude_desktop_config.json",
|
|
54
|
+
".roo/mcp.json",
|
|
55
|
+
".claude/settings.json",
|
|
56
|
+
".claude/settings.local.json",
|
|
57
|
+
".aider.conf.yml",
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
SKIP_DIRS: frozenset[str] = frozenset({".git", "node_modules", ".venv"})
|
|
61
|
+
MAX_OTHER_FILE_BYTES = 2 * 1024 * 1024
|
|
62
|
+
_BINARY_SNIFF_BYTES = 8192
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Candidate:
|
|
67
|
+
"""A file selected for scanning. ``rel`` is the display path (POSIX separators)."""
|
|
68
|
+
|
|
69
|
+
abs_path: Path
|
|
70
|
+
rel: str
|
|
71
|
+
kind: Kind
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def glob_to_regex(pattern: str) -> re.Pattern[str]:
|
|
75
|
+
"""Compile a glob into a case-insensitive regex over POSIX relative paths.
|
|
76
|
+
|
|
77
|
+
``*`` and ``?`` stay within one path segment; ``**/`` matches zero or more directories and a
|
|
78
|
+
trailing ``**`` matches everything below.
|
|
79
|
+
"""
|
|
80
|
+
pattern = pattern.replace("\\", "/") # patterns written with Windows separators
|
|
81
|
+
out: list[str] = []
|
|
82
|
+
i = 0
|
|
83
|
+
while i < len(pattern):
|
|
84
|
+
if pattern.startswith("**/", i):
|
|
85
|
+
out.append("(?:.*/)?")
|
|
86
|
+
i += 3
|
|
87
|
+
elif pattern.startswith("**", i):
|
|
88
|
+
out.append(".*")
|
|
89
|
+
i += 2
|
|
90
|
+
elif pattern[i] == "*":
|
|
91
|
+
out.append("[^/]*")
|
|
92
|
+
i += 1
|
|
93
|
+
elif pattern[i] == "?":
|
|
94
|
+
out.append("[^/]")
|
|
95
|
+
i += 1
|
|
96
|
+
else:
|
|
97
|
+
out.append(re.escape(pattern[i]))
|
|
98
|
+
i += 1
|
|
99
|
+
return re.compile("".join(out), re.IGNORECASE)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@cache
|
|
103
|
+
def _compile_all(globs: tuple[str, ...]) -> tuple[re.Pattern[str], ...]:
|
|
104
|
+
return tuple(glob_to_regex(g) for g in globs)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def matches_any(rel: str, globs: Iterable[str]) -> bool:
|
|
108
|
+
"""True if the relative path (either separator style) matches any glob."""
|
|
109
|
+
rel = rel.replace("\\", "/")
|
|
110
|
+
return any(p.fullmatch(rel) for p in _compile_all(tuple(globs)))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def classify_path(rel: str, extra_targets: Iterable[str] = ()) -> Kind | None:
|
|
114
|
+
"""Kind of a known target file, or ``None`` if the path is not a default/extra target."""
|
|
115
|
+
if matches_any(rel, INSTRUCTION_GLOBS) or matches_any(rel, extra_targets):
|
|
116
|
+
return "instructions"
|
|
117
|
+
if matches_any(rel, CONFIG_GLOBS):
|
|
118
|
+
return "config"
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def looks_binary(path: Path) -> bool:
|
|
123
|
+
"""Heuristic: a NUL byte in the first 8 KiB means binary."""
|
|
124
|
+
try:
|
|
125
|
+
with path.open("rb") as handle:
|
|
126
|
+
return b"\x00" in handle.read(_BINARY_SNIFF_BYTES)
|
|
127
|
+
except OSError:
|
|
128
|
+
return True
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _acceptable_other(path: Path) -> bool:
|
|
132
|
+
"""Whether an unrecognised file is worth scanning under ``--all-files``."""
|
|
133
|
+
try:
|
|
134
|
+
if path.stat().st_size > MAX_OTHER_FILE_BYTES:
|
|
135
|
+
return False
|
|
136
|
+
except OSError:
|
|
137
|
+
return False
|
|
138
|
+
return not looks_binary(path)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def discover(
|
|
142
|
+
root: Path,
|
|
143
|
+
*,
|
|
144
|
+
all_files: bool = False,
|
|
145
|
+
exclude: Iterable[str] = (),
|
|
146
|
+
extra_targets: Iterable[str] = (),
|
|
147
|
+
) -> list[Candidate]:
|
|
148
|
+
"""Find scannable files under ``root`` (or the single file ``root``).
|
|
149
|
+
|
|
150
|
+
Symlinks are never followed. Known targets are always selected whatever their size or
|
|
151
|
+
content, so padding a file cannot hide it; the size and binary checks only gate the
|
|
152
|
+
unrecognised files that ``all_files`` adds. An explicitly named file is always scanned.
|
|
153
|
+
"""
|
|
154
|
+
excludes = tuple(exclude)
|
|
155
|
+
extras = tuple(extra_targets)
|
|
156
|
+
if root.is_file():
|
|
157
|
+
kind = classify_path(root.name, extras) or "other"
|
|
158
|
+
return [Candidate(root, root.name, kind)]
|
|
159
|
+
|
|
160
|
+
found: list[Candidate] = []
|
|
161
|
+
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
|
162
|
+
dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
|
|
163
|
+
base = Path(dirpath)
|
|
164
|
+
for name in sorted(filenames):
|
|
165
|
+
path = base / name
|
|
166
|
+
if path.is_symlink():
|
|
167
|
+
continue
|
|
168
|
+
rel = path.relative_to(root).as_posix()
|
|
169
|
+
if excludes and matches_any(rel, excludes):
|
|
170
|
+
continue
|
|
171
|
+
known = classify_path(rel, extras)
|
|
172
|
+
if known is not None:
|
|
173
|
+
found.append(Candidate(path, rel, known))
|
|
174
|
+
elif all_files and _acceptable_other(path):
|
|
175
|
+
found.append(Candidate(path, rel, "other"))
|
|
176
|
+
return found
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _desktop_config(home: Path, platform: str, env: Mapping[str, str]) -> Path:
|
|
180
|
+
if platform == "darwin":
|
|
181
|
+
return home / "Library/Application Support/Claude/claude_desktop_config.json"
|
|
182
|
+
if platform == "win32":
|
|
183
|
+
appdata = env.get("APPDATA")
|
|
184
|
+
base = Path(appdata) if appdata else home / "AppData/Roaming"
|
|
185
|
+
return base / "Claude/claude_desktop_config.json"
|
|
186
|
+
return home / ".config/Claude/claude_desktop_config.json"
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def home_targets(
|
|
190
|
+
home: Path | None = None,
|
|
191
|
+
platform: str | None = None,
|
|
192
|
+
env: Mapping[str, str] | None = None,
|
|
193
|
+
) -> list[Candidate]:
|
|
194
|
+
"""User-level agent configs that exist on this machine (``--home``). Missing ones are skipped.
|
|
195
|
+
|
|
196
|
+
Unlike repo scanning, symlinks are accepted here: dotfile managers commonly symlink these.
|
|
197
|
+
"""
|
|
198
|
+
home = home or Path.home()
|
|
199
|
+
platform = platform or sys.platform
|
|
200
|
+
env = os.environ if env is None else env
|
|
201
|
+
entries: list[tuple[Path, Kind]] = [
|
|
202
|
+
(home / ".claude.json", "config"),
|
|
203
|
+
(home / ".claude/settings.json", "config"),
|
|
204
|
+
(home / ".claude/CLAUDE.md", "instructions"),
|
|
205
|
+
(home / ".cursor/mcp.json", "config"),
|
|
206
|
+
(home / ".codeium/windsurf/mcp_config.json", "config"),
|
|
207
|
+
(home / ".gemini/settings.json", "config"),
|
|
208
|
+
(home / ".codex/config.toml", "config"),
|
|
209
|
+
(_desktop_config(home, platform, env), "config"),
|
|
210
|
+
]
|
|
211
|
+
out: list[Candidate] = []
|
|
212
|
+
for path, kind in entries:
|
|
213
|
+
if not path.is_file():
|
|
214
|
+
continue
|
|
215
|
+
try:
|
|
216
|
+
label = "~/" + path.relative_to(home).as_posix()
|
|
217
|
+
except ValueError:
|
|
218
|
+
label = path.as_posix()
|
|
219
|
+
out.append(Candidate(path, label, kind))
|
|
220
|
+
return out
|