vulnhound 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vulnhound/__init__.py +3 -0
- vulnhound/__main__.py +3 -0
- vulnhound/cache.py +70 -0
- vulnhound/cli.py +250 -0
- vulnhound/evaluate.py +77 -0
- vulnhound/group.py +75 -0
- vulnhound/index/__init__.py +5 -0
- vulnhound/index/core.py +151 -0
- vulnhound/index/extract.py +182 -0
- vulnhound/index/resolve.py +152 -0
- vulnhound/models.py +82 -0
- vulnhound/poc.py +23 -0
- vulnhound/project.py +70 -0
- vulnhound/prove.py +129 -0
- vulnhound/report.py +181 -0
- vulnhound/sandbox.py +592 -0
- vulnhound/slicer.py +420 -0
- vulnhound/store.py +64 -0
- vulnhound/taint/__init__.py +1 -0
- vulnhound/taint/engine.py +1883 -0
- vulnhound/taint/specs.py +987 -0
- vulnhound-0.1.0.dist-info/METADATA +97 -0
- vulnhound-0.1.0.dist-info/RECORD +26 -0
- vulnhound-0.1.0.dist-info/WHEEL +4 -0
- vulnhound-0.1.0.dist-info/entry_points.txt +2 -0
- vulnhound-0.1.0.dist-info/licenses/LICENSE +21 -0
vulnhound/__init__.py
ADDED
vulnhound/__main__.py
ADDED
vulnhound/cache.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""On-disk JSON cache of per-module extraction results (never pickle)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
|
|
12
|
+
SCHEMA_VERSION = 1
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Cache:
|
|
16
|
+
"""Cache at ``<root>/.vulnhound/cache/``; entries keyed by content + version + schema."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, root: Path) -> None:
|
|
19
|
+
self.dir = Path(root) / ".vulnhound" / "cache"
|
|
20
|
+
self.used: set[str] = set()
|
|
21
|
+
|
|
22
|
+
def _key(self, sha: str, module: str, path: str) -> str:
|
|
23
|
+
raw = f"{__version__}|{SCHEMA_VERSION}|{sha}|{module}|{path}"
|
|
24
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:32]
|
|
25
|
+
|
|
26
|
+
def load(self, sha: str, module: str, path: str) -> dict | None:
|
|
27
|
+
key = self._key(sha, module, path)
|
|
28
|
+
try:
|
|
29
|
+
data = json.loads((self.dir / f"{key}.json").read_text(encoding="utf-8"))
|
|
30
|
+
except (OSError, ValueError):
|
|
31
|
+
return None
|
|
32
|
+
if (
|
|
33
|
+
not isinstance(data, dict)
|
|
34
|
+
or data.get("schema") != SCHEMA_VERSION
|
|
35
|
+
or data.get("version") != __version__
|
|
36
|
+
or data.get("key") != key
|
|
37
|
+
or not isinstance(data.get("module"), dict)
|
|
38
|
+
or not isinstance(data.get("functions"), list)
|
|
39
|
+
or not isinstance(data.get("classes"), dict)
|
|
40
|
+
):
|
|
41
|
+
return None
|
|
42
|
+
self.used.add(key)
|
|
43
|
+
return data
|
|
44
|
+
|
|
45
|
+
def store(self, sha: str, module: str, path: str, payload: dict) -> None:
|
|
46
|
+
key = self._key(sha, module, path)
|
|
47
|
+
entry = {"schema": SCHEMA_VERSION, "version": __version__, "key": key, **payload}
|
|
48
|
+
tmp = self.dir / f"{key}.{os.getpid()}.tmp"
|
|
49
|
+
try:
|
|
50
|
+
self.dir.mkdir(parents=True, exist_ok=True)
|
|
51
|
+
gi = self.dir.parent / ".gitignore"
|
|
52
|
+
if not gi.exists():
|
|
53
|
+
gi.write_text("*\n", encoding="utf-8")
|
|
54
|
+
tmp.write_text(json.dumps(entry, separators=(",", ":")), encoding="utf-8")
|
|
55
|
+
os.replace(tmp, self.dir / f"{key}.json")
|
|
56
|
+
self.used.add(key)
|
|
57
|
+
except OSError:
|
|
58
|
+
try:
|
|
59
|
+
tmp.unlink()
|
|
60
|
+
except OSError:
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
def prune(self) -> None:
|
|
64
|
+
"""Drop entries not used in this run (stale files / old versions)."""
|
|
65
|
+
try:
|
|
66
|
+
for p in self.dir.glob("*.json"):
|
|
67
|
+
if p.stem not in self.used:
|
|
68
|
+
p.unlink()
|
|
69
|
+
except OSError:
|
|
70
|
+
pass
|
vulnhound/cli.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""vulnhound command line interface. Output is deliberately terse."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from vulnhound import __version__
|
|
12
|
+
|
|
13
|
+
SKIP_DIRS = {
|
|
14
|
+
"node_modules",
|
|
15
|
+
"__pycache__",
|
|
16
|
+
"build",
|
|
17
|
+
"dist",
|
|
18
|
+
"venv",
|
|
19
|
+
"env",
|
|
20
|
+
"site-packages",
|
|
21
|
+
}
|
|
22
|
+
EXIT_OK, EXIT_FINDINGS, EXIT_USAGE = 0, 1, 2
|
|
23
|
+
SEVERITIES = ["low", "medium", "high", "critical"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _not_implemented(what: str) -> int:
|
|
27
|
+
print(
|
|
28
|
+
f"vulnhound: '{what}' is not implemented yet in this version ({__version__}).",
|
|
29
|
+
file=sys.stderr,
|
|
30
|
+
)
|
|
31
|
+
return EXIT_USAGE
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _skip_dir(name: str) -> bool:
|
|
35
|
+
return name.startswith(".") or name in SKIP_DIRS or name.endswith(".egg-info")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def estimate_tree(root: Path) -> dict:
|
|
39
|
+
files = lines = size = 0
|
|
40
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
41
|
+
dirnames[:] = [d for d in dirnames if not _skip_dir(d)]
|
|
42
|
+
for fn in filenames:
|
|
43
|
+
if not fn.endswith(".py"):
|
|
44
|
+
continue
|
|
45
|
+
try:
|
|
46
|
+
data = (Path(dirpath) / fn).read_bytes()
|
|
47
|
+
except OSError:
|
|
48
|
+
continue
|
|
49
|
+
files += 1
|
|
50
|
+
size += len(data)
|
|
51
|
+
lines += data.count(b"\n") + (1 if data and not data.endswith(b"\n") else 0)
|
|
52
|
+
return {
|
|
53
|
+
"root": str(root),
|
|
54
|
+
"files": files,
|
|
55
|
+
"lines": lines,
|
|
56
|
+
"bytes": size,
|
|
57
|
+
"naive_tokens": size // 4,
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def cmd_estimate(args: argparse.Namespace) -> int:
|
|
62
|
+
root = Path(args.path)
|
|
63
|
+
if not root.is_dir():
|
|
64
|
+
print(f"vulnhound: not a directory: {root}", file=sys.stderr)
|
|
65
|
+
return EXIT_USAGE
|
|
66
|
+
est = estimate_tree(root)
|
|
67
|
+
if args.json:
|
|
68
|
+
print(json.dumps(est))
|
|
69
|
+
return EXIT_OK
|
|
70
|
+
print(f"{est['files']} .py files, {est['lines']} lines, {est['bytes']} bytes")
|
|
71
|
+
print(f"naive (read whole repo): ~{est['naive_tokens']} tokens (bytes/4)")
|
|
72
|
+
print("vulnhound: Claude reads only `analyze` output + small slices, not the repo.")
|
|
73
|
+
return EXIT_OK
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _run_analysis(root: Path, *, use_cache: bool, include_tests: bool, local: bool):
|
|
77
|
+
from vulnhound.index import build_index
|
|
78
|
+
from vulnhound.taint.engine import analyze
|
|
79
|
+
|
|
80
|
+
index = build_index(root, use_cache=use_cache, include_tests=include_tests)
|
|
81
|
+
if local:
|
|
82
|
+
try:
|
|
83
|
+
return analyze(index, include_local_sources=True)
|
|
84
|
+
except TypeError: # engine without the kwarg yet
|
|
85
|
+
print("vulnhound: --include-local-sources unsupported by engine", file=sys.stderr)
|
|
86
|
+
return analyze(index)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def cmd_analyze(args: argparse.Namespace) -> int:
|
|
90
|
+
from vulnhound import report, store
|
|
91
|
+
from vulnhound.group import group_findings
|
|
92
|
+
|
|
93
|
+
root = Path(args.path).resolve()
|
|
94
|
+
if not root.is_dir():
|
|
95
|
+
print(f"vulnhound: not a directory: {root}", file=sys.stderr)
|
|
96
|
+
return EXIT_USAGE
|
|
97
|
+
findings = _run_analysis(
|
|
98
|
+
root,
|
|
99
|
+
use_cache=not args.no_cache,
|
|
100
|
+
include_tests=args.include_tests,
|
|
101
|
+
local=args.include_local_sources,
|
|
102
|
+
)
|
|
103
|
+
order = {s: i for i, s in enumerate(SEVERITIES)}
|
|
104
|
+
findings = [f for f in findings if order[f.severity.value] >= order[args.min_severity]]
|
|
105
|
+
findings.sort(key=lambda f: (-order[f.severity.value], f.sink.path, f.sink.line))
|
|
106
|
+
meta = {"version": __version__, "root": str(root)}
|
|
107
|
+
try:
|
|
108
|
+
store.save_findings(root, findings, meta)
|
|
109
|
+
except OSError as e:
|
|
110
|
+
print(f"vulnhound: could not save findings: {e}", file=sys.stderr)
|
|
111
|
+
if args.json:
|
|
112
|
+
print(report.to_json(findings, meta, grouped=not args.no_group))
|
|
113
|
+
elif args.no_group:
|
|
114
|
+
for f in findings:
|
|
115
|
+
print(f"{f.severity.value.upper()} {f.id} {f.rule} {f.sink} {f.message}")
|
|
116
|
+
else:
|
|
117
|
+
for g in group_findings(findings):
|
|
118
|
+
f = g.rep
|
|
119
|
+
extra = f" (+{g.count - 1}: {', '.join(g.also)})" if g.count > 1 and g.also else ""
|
|
120
|
+
if g.count > 1 and not g.also:
|
|
121
|
+
extra = f" (+{g.count - 1})"
|
|
122
|
+
print(f"{f.severity.value.upper()} {f.id} {f.rule} {f.sink} {f.message}{extra}")
|
|
123
|
+
if not findings:
|
|
124
|
+
print("no findings")
|
|
125
|
+
return EXIT_FINDINGS if findings else EXIT_OK
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def cmd_slice(args: argparse.Namespace) -> int:
|
|
129
|
+
try:
|
|
130
|
+
from vulnhound import slicer # type: ignore[attr-defined]
|
|
131
|
+
except ImportError:
|
|
132
|
+
return _not_implemented("slice")
|
|
133
|
+
return slicer.run(Path(args.root).resolve(), args.id)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def cmd_prove(args: argparse.Namespace) -> int:
|
|
137
|
+
from vulnhound import prove
|
|
138
|
+
|
|
139
|
+
return prove.run(
|
|
140
|
+
Path(args.root).resolve(),
|
|
141
|
+
args.id,
|
|
142
|
+
Path(args.poc),
|
|
143
|
+
backend=args.backend,
|
|
144
|
+
timeout=args.timeout,
|
|
145
|
+
as_json=args.json,
|
|
146
|
+
verbose=args.verbose,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def cmd_report(args: argparse.Namespace) -> int:
|
|
151
|
+
from vulnhound import report, store
|
|
152
|
+
|
|
153
|
+
root = Path(args.root).resolve()
|
|
154
|
+
try:
|
|
155
|
+
findings, meta = store.load_findings(root)
|
|
156
|
+
except (OSError, ValueError, KeyError):
|
|
157
|
+
findings = _run_analysis(
|
|
158
|
+
root,
|
|
159
|
+
use_cache=True,
|
|
160
|
+
include_tests=args.include_tests,
|
|
161
|
+
local=args.include_local_sources,
|
|
162
|
+
)
|
|
163
|
+
meta = {}
|
|
164
|
+
meta = {**meta, "version": __version__, "root": meta.get("root", str(root))}
|
|
165
|
+
render = {"json": report.to_json, "sarif": report.to_sarif, "markdown": report.to_markdown}
|
|
166
|
+
out = render[args.format](findings, meta)
|
|
167
|
+
if args.output:
|
|
168
|
+
Path(args.output).write_text(out, encoding="utf-8")
|
|
169
|
+
else:
|
|
170
|
+
print(out)
|
|
171
|
+
return EXIT_OK
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def cmd_version(_: argparse.Namespace) -> int:
|
|
175
|
+
print(__version__)
|
|
176
|
+
return EXIT_OK
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _analysis_flags(p: argparse.ArgumentParser) -> None:
|
|
180
|
+
p.add_argument(
|
|
181
|
+
"--include-tests",
|
|
182
|
+
action="store_true",
|
|
183
|
+
help="also analyze tests/examples/docs dirs and test_*.py files (skipped by default)",
|
|
184
|
+
)
|
|
185
|
+
p.add_argument(
|
|
186
|
+
"--include-local-sources",
|
|
187
|
+
action="store_true",
|
|
188
|
+
help="also treat local/CLI-style inputs as taint sources",
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
193
|
+
p = argparse.ArgumentParser(
|
|
194
|
+
prog="vulnhound",
|
|
195
|
+
description="Token-thrifty vulnerability finder for Python projects.",
|
|
196
|
+
)
|
|
197
|
+
sub = p.add_subparsers(dest="cmd", metavar="<command>")
|
|
198
|
+
|
|
199
|
+
a = sub.add_parser("analyze", help="static taint analysis, short JSON output")
|
|
200
|
+
a.add_argument("path", nargs="?", default=".")
|
|
201
|
+
a.add_argument("--json", action="store_true", help="machine-readable output")
|
|
202
|
+
a.add_argument("--no-cache", action="store_true", help="ignore the index cache")
|
|
203
|
+
a.add_argument("--min-severity", choices=SEVERITIES, default="low")
|
|
204
|
+
a.add_argument("--no-group", action="store_true", help="one line per finding, no grouping")
|
|
205
|
+
_analysis_flags(a)
|
|
206
|
+
a.set_defaults(func=cmd_analyze)
|
|
207
|
+
|
|
208
|
+
s = sub.add_parser("slice", help="print the minimal code slice for a finding")
|
|
209
|
+
s.add_argument("id")
|
|
210
|
+
s.add_argument("--root", default=".", help="project root of the last analyze run")
|
|
211
|
+
s.set_defaults(func=cmd_slice)
|
|
212
|
+
|
|
213
|
+
pr = sub.add_parser("prove", help="run a PoC for a finding in a sandbox")
|
|
214
|
+
pr.add_argument("id")
|
|
215
|
+
pr.add_argument("--poc", required=True, metavar="FILE")
|
|
216
|
+
pr.add_argument("--root", default=".", help="project root of the last analyze run")
|
|
217
|
+
pr.add_argument("--backend", choices=["auto", "docker", "unshare"], default="auto")
|
|
218
|
+
pr.add_argument("--timeout", type=float, default=10.0, help="PoC time limit in seconds")
|
|
219
|
+
pr.add_argument("--json", action="store_true")
|
|
220
|
+
pr.add_argument("--verbose", action="store_true", help="also print PoC stdout/stderr")
|
|
221
|
+
pr.set_defaults(func=cmd_prove)
|
|
222
|
+
|
|
223
|
+
e = sub.add_parser("estimate", help="estimate tokens to naively read the repo")
|
|
224
|
+
e.add_argument("path", nargs="?", default=".")
|
|
225
|
+
e.add_argument("--json", action="store_true")
|
|
226
|
+
e.set_defaults(func=cmd_estimate)
|
|
227
|
+
|
|
228
|
+
r = sub.add_parser("report", help="render findings as json, sarif or markdown")
|
|
229
|
+
r.add_argument("--format", choices=["json", "sarif", "markdown"], default="markdown")
|
|
230
|
+
r.add_argument("-o", "--output", metavar="FILE")
|
|
231
|
+
r.add_argument("--root", default=".", help="project root (default: current directory)")
|
|
232
|
+
_analysis_flags(r)
|
|
233
|
+
r.set_defaults(func=cmd_report)
|
|
234
|
+
|
|
235
|
+
v = sub.add_parser("version", help="print version")
|
|
236
|
+
v.set_defaults(func=cmd_version)
|
|
237
|
+
return p
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def main(argv: list[str] | None = None) -> int:
|
|
241
|
+
parser = build_parser()
|
|
242
|
+
args = parser.parse_args(argv)
|
|
243
|
+
if not getattr(args, "func", None):
|
|
244
|
+
parser.print_help(sys.stderr)
|
|
245
|
+
return EXIT_USAGE
|
|
246
|
+
return args.func(args)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
if __name__ == "__main__":
|
|
250
|
+
raise SystemExit(main())
|
vulnhound/evaluate.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Fixture-based quality evaluation: compares findings to ``# VH-EXPECT:`` comments."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .index import build_index
|
|
10
|
+
from .models import Finding
|
|
11
|
+
|
|
12
|
+
_EXPECT = re.compile(r"#\s*VH-EXPECT:\s*([A-Za-z0-9_,\-\s]+)")
|
|
13
|
+
|
|
14
|
+
Key = tuple[str, int, str]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def load_expectations(root: Path) -> set[Key]:
|
|
18
|
+
"""Scan ``*.py`` under root for ``# VH-EXPECT: rule[, rule]`` comments."""
|
|
19
|
+
root = Path(root)
|
|
20
|
+
out: set[Key] = set()
|
|
21
|
+
for path in sorted(root.rglob("*.py")):
|
|
22
|
+
rel = path.relative_to(root).as_posix()
|
|
23
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
24
|
+
for lineno, line in enumerate(text.splitlines(), 1):
|
|
25
|
+
m = _EXPECT.search(line)
|
|
26
|
+
if not m:
|
|
27
|
+
continue
|
|
28
|
+
for rule in re.split(r"[,\s]+", m.group(1).strip()):
|
|
29
|
+
if rule:
|
|
30
|
+
out.add((rel, lineno, rule))
|
|
31
|
+
return out
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class EvalResult:
|
|
36
|
+
tp: list[Key] = field(default_factory=list)
|
|
37
|
+
fp: list[Key] = field(default_factory=list)
|
|
38
|
+
fn: list[Key] = field(default_factory=list)
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def precision(self) -> float:
|
|
42
|
+
n = len(self.tp) + len(self.fp)
|
|
43
|
+
return len(self.tp) / n if n else 1.0
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def recall(self) -> float:
|
|
47
|
+
n = len(self.tp) + len(self.fn)
|
|
48
|
+
return len(self.tp) / n if n else 1.0
|
|
49
|
+
|
|
50
|
+
def format(self) -> str:
|
|
51
|
+
lines = [
|
|
52
|
+
f"tp={len(self.tp)} fp={len(self.fp)} fn={len(self.fn)} "
|
|
53
|
+
f"precision={self.precision:.2f} recall={self.recall:.2f}"
|
|
54
|
+
]
|
|
55
|
+
for tag, items in (("FN", self.fn), ("FP", self.fp)):
|
|
56
|
+
for path, line, rule in sorted(items):
|
|
57
|
+
lines.append(f" {tag} {path}:{line} {rule}")
|
|
58
|
+
return "\n".join(lines)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def score(expected: set[Key], findings: list[Finding]) -> EvalResult:
|
|
62
|
+
found = {(f.sink.path, f.sink.line, f.rule) for f in findings}
|
|
63
|
+
return EvalResult(
|
|
64
|
+
tp=sorted(expected & found), fp=sorted(found - expected), fn=sorted(expected - found)
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def analyze(index) -> list[Finding]:
|
|
69
|
+
from .taint import engine
|
|
70
|
+
|
|
71
|
+
return engine.analyze(index, include_local_sources=True) # fixtures cover argv/env flows
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def evaluate(root: Path) -> EvalResult:
|
|
75
|
+
root = Path(root)
|
|
76
|
+
index = build_index(root, use_cache=False)
|
|
77
|
+
return score(load_expectations(root), analyze(index))
|
vulnhound/group.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Group findings that share a sink (or a helper source) so Claude judges each once."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from vulnhound.models import Finding
|
|
8
|
+
|
|
9
|
+
_SEV = {"low": 0, "medium": 1, "high": 2, "critical": 3}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Group:
|
|
14
|
+
rep: Finding # highest severity, then confidence
|
|
15
|
+
members: list[Finding] = field(default_factory=list) # all findings incl. rep
|
|
16
|
+
|
|
17
|
+
@property
|
|
18
|
+
def count(self) -> int:
|
|
19
|
+
return len(self.members)
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def also(self) -> list[str]:
|
|
23
|
+
"""Compact ``file:line`` of what differs from the representative.
|
|
24
|
+
|
|
25
|
+
Other sources when the sink is shared, else the other sinks.
|
|
26
|
+
"""
|
|
27
|
+
seen = {str(self.rep.sink), str(self.rep.source)}
|
|
28
|
+
out: list[str] = []
|
|
29
|
+
for f in self.members:
|
|
30
|
+
if f is self.rep:
|
|
31
|
+
continue
|
|
32
|
+
loc = str(f.source) if f.sink == self.rep.sink else str(f.sink)
|
|
33
|
+
if loc not in seen:
|
|
34
|
+
seen.add(loc)
|
|
35
|
+
out.append(loc)
|
|
36
|
+
return out
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _rank(f: Finding) -> tuple[int, float]:
|
|
40
|
+
return (_SEV[f.severity.value], f.confidence)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def group_findings(findings: list[Finding]) -> list[Group]:
|
|
44
|
+
"""Union findings that share (rule, sink path+line), (rule, sink path+function),
|
|
45
|
+
or (rule, source) when the source lives in another file (a helper)."""
|
|
46
|
+
parent = list(range(len(findings)))
|
|
47
|
+
|
|
48
|
+
def find(i: int) -> int:
|
|
49
|
+
while parent[i] != i:
|
|
50
|
+
parent[i] = parent[parent[i]]
|
|
51
|
+
i = parent[i]
|
|
52
|
+
return i
|
|
53
|
+
|
|
54
|
+
seen: dict[tuple, int] = {}
|
|
55
|
+
for i, f in enumerate(findings):
|
|
56
|
+
keys = [
|
|
57
|
+
(f.rule, "sink", f.sink.path, f.sink.line),
|
|
58
|
+
(f.rule, "fn", f.sink.path, f.function),
|
|
59
|
+
]
|
|
60
|
+
if f.source.path != f.sink.path:
|
|
61
|
+
keys.append((f.rule, "src", f.source.path, f.source.line))
|
|
62
|
+
for k in keys:
|
|
63
|
+
if k in seen:
|
|
64
|
+
parent[find(i)] = find(seen[k])
|
|
65
|
+
else:
|
|
66
|
+
seen[k] = i
|
|
67
|
+
buckets: dict[int, list[Finding]] = {}
|
|
68
|
+
for i, f in enumerate(findings):
|
|
69
|
+
buckets.setdefault(find(i), []).append(f)
|
|
70
|
+
groups = []
|
|
71
|
+
for members in buckets.values():
|
|
72
|
+
rep = max(members, key=_rank) # first among equals keeps input order
|
|
73
|
+
groups.append(Group(rep, members))
|
|
74
|
+
groups.sort(key=lambda g: (-_SEV[g.rep.severity.value], g.rep.sink.path, g.rep.sink.line))
|
|
75
|
+
return groups
|
vulnhound/index/core.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""ProjectIndex and build_index."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import hashlib
|
|
7
|
+
import tokenize
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from ..cache import Cache
|
|
11
|
+
from ..models import FunctionInfo, Location, ModuleInfo
|
|
12
|
+
from ..project import discover, is_package_file, module_name_for, relative_posix
|
|
13
|
+
from . import extract
|
|
14
|
+
from .resolve import Resolver
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def decode_source(data: bytes) -> str:
|
|
18
|
+
"""Decode bytes: utf-8 first, then the declared encoding, then lossy."""
|
|
19
|
+
try:
|
|
20
|
+
return data.decode("utf-8-sig")
|
|
21
|
+
except UnicodeDecodeError:
|
|
22
|
+
pass
|
|
23
|
+
try:
|
|
24
|
+
enc, _ = tokenize.detect_encoding(iter(data.splitlines(True)).__next__)
|
|
25
|
+
return data.decode(enc, errors="replace")
|
|
26
|
+
except (SyntaxError, LookupError, StopIteration):
|
|
27
|
+
return data.decode("utf-8", errors="replace")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ProjectIndex:
|
|
31
|
+
def __init__(self, root: Path) -> None:
|
|
32
|
+
self.root = root
|
|
33
|
+
self.modules: dict[str, ModuleInfo] = {}
|
|
34
|
+
self.functions: dict[str, FunctionInfo] = {}
|
|
35
|
+
self._classes: dict[str, tuple[str, list[str]]] = {}
|
|
36
|
+
self._func_cls: dict[str, str | None] = {}
|
|
37
|
+
self._resolver: Resolver | None = None
|
|
38
|
+
self._callers: dict[str, list[str]] | None = None
|
|
39
|
+
self._ast: dict[str, ast.Module] = {}
|
|
40
|
+
self._lines: dict[str, list[str]] = {}
|
|
41
|
+
|
|
42
|
+
# -- lazy per-module data -------------------------------------------------
|
|
43
|
+
def _text(self, module_name: str) -> str:
|
|
44
|
+
return decode_source((self.root / self.modules[module_name].path).read_bytes())
|
|
45
|
+
|
|
46
|
+
def ast_for(self, module_name: str) -> ast.Module:
|
|
47
|
+
tree = self._ast.get(module_name)
|
|
48
|
+
if tree is None:
|
|
49
|
+
mi = self.modules[module_name]
|
|
50
|
+
tree = ast.parse(self._text(module_name), filename=mi.path)
|
|
51
|
+
self._ast[module_name] = tree
|
|
52
|
+
return tree
|
|
53
|
+
|
|
54
|
+
def source_lines(self, module_name: str) -> list[str]:
|
|
55
|
+
lines = self._lines.get(module_name)
|
|
56
|
+
if lines is None:
|
|
57
|
+
lines = self._lines[module_name] = self._text(module_name).splitlines()
|
|
58
|
+
return lines
|
|
59
|
+
|
|
60
|
+
# -- call graph -----------------------------------------------------------
|
|
61
|
+
def callees(self, qualname: str) -> list[str]:
|
|
62
|
+
fn = self.functions.get(qualname)
|
|
63
|
+
if fn is None:
|
|
64
|
+
return []
|
|
65
|
+
return [c for c in fn.calls if c in self.functions]
|
|
66
|
+
|
|
67
|
+
def callers(self, qualname: str) -> list[str]:
|
|
68
|
+
if self._callers is None:
|
|
69
|
+
rev: dict[str, list[str]] = {}
|
|
70
|
+
for q, fn in self.functions.items():
|
|
71
|
+
for c in fn.calls:
|
|
72
|
+
if c in self.functions:
|
|
73
|
+
rev.setdefault(c, []).append(q)
|
|
74
|
+
self._callers = rev
|
|
75
|
+
return list(self._callers.get(qualname, []))
|
|
76
|
+
|
|
77
|
+
# -- resolution -----------------------------------------------------------
|
|
78
|
+
def resolve(self, module_name: str, dotted_name: str, function: str | None = None) -> str:
|
|
79
|
+
"""Resolve *dotted_name* as seen from *module_name* (optionally inside *function*).
|
|
80
|
+
|
|
81
|
+
``self.x``/``cls.x`` need class context: pass *function* (a qualname), or
|
|
82
|
+
a unique same-named method in the module's classes is used.
|
|
83
|
+
"""
|
|
84
|
+
assert self._resolver is not None
|
|
85
|
+
cls = self._func_cls.get(function) if function else None
|
|
86
|
+
head, _, rest = dotted_name.partition(".")
|
|
87
|
+
if head in ("self", "cls") and cls is None and rest and "." not in rest:
|
|
88
|
+
hits = [
|
|
89
|
+
f"{c}.{rest}"
|
|
90
|
+
for c, (m, _b) in self._classes.items()
|
|
91
|
+
if m == module_name and f"{c}.{rest}" in self.functions
|
|
92
|
+
]
|
|
93
|
+
if len(hits) == 1:
|
|
94
|
+
return hits[0]
|
|
95
|
+
return self._resolver.resolve(module_name, dotted_name, cls, function)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def build_index(root: Path, use_cache: bool = True, include_tests: bool = True) -> ProjectIndex:
|
|
99
|
+
root = Path(root).resolve()
|
|
100
|
+
idx = ProjectIndex(root)
|
|
101
|
+
cache = Cache(root) if use_cache else None
|
|
102
|
+
raw: dict[str, list[str]] = {}
|
|
103
|
+
for path in discover(root, include_tests):
|
|
104
|
+
try:
|
|
105
|
+
data = path.read_bytes()
|
|
106
|
+
except OSError:
|
|
107
|
+
continue
|
|
108
|
+
name = module_name_for(root, path)
|
|
109
|
+
if name in idx.modules:
|
|
110
|
+
continue
|
|
111
|
+
rel = relative_posix(root, path)
|
|
112
|
+
sha = hashlib.sha256(data).hexdigest()
|
|
113
|
+
payload = cache.load(sha, name, rel) if cache else None
|
|
114
|
+
mi = fns = None
|
|
115
|
+
if payload is not None:
|
|
116
|
+
try:
|
|
117
|
+
mi = ModuleInfo(**payload["module"])
|
|
118
|
+
fns = payload["functions"]
|
|
119
|
+
except (TypeError, KeyError):
|
|
120
|
+
payload = mi = fns = None
|
|
121
|
+
if payload is None:
|
|
122
|
+
text = decode_source(data)
|
|
123
|
+
payload = extract.extract_module(name, rel, text, sha, is_package_file(path))
|
|
124
|
+
if cache:
|
|
125
|
+
cache.store(sha, name, rel, payload)
|
|
126
|
+
mi = ModuleInfo(**payload["module"])
|
|
127
|
+
fns = payload["functions"]
|
|
128
|
+
idx.modules[name] = mi
|
|
129
|
+
for q, bases in payload["classes"].items():
|
|
130
|
+
idx._classes[q] = (name, bases)
|
|
131
|
+
for f in fns:
|
|
132
|
+
q = f["qualname"]
|
|
133
|
+
idx.functions[q] = FunctionInfo(
|
|
134
|
+
qualname=q, module=f["module"], name=f["name"],
|
|
135
|
+
location=Location(rel, f["line"], f["col"]), end_line=f["end_line"],
|
|
136
|
+
params=f["params"], decorators=f["decorators"], calls=[],
|
|
137
|
+
is_method=f["is_method"],
|
|
138
|
+
) # fmt: skip
|
|
139
|
+
idx._func_cls[q] = f["cls"]
|
|
140
|
+
raw[q] = f["calls"]
|
|
141
|
+
if cache:
|
|
142
|
+
cache.prune()
|
|
143
|
+
res = idx._resolver = Resolver(idx.modules, idx.functions, idx._classes)
|
|
144
|
+
for q, calls in raw.items():
|
|
145
|
+
fn = idx.functions[q]
|
|
146
|
+
cls = idx._func_cls[q]
|
|
147
|
+
seen: dict[str, None] = {}
|
|
148
|
+
for c in calls:
|
|
149
|
+
seen[res.resolve(fn.module, c, cls, q)] = None
|
|
150
|
+
fn.calls = list(seen)
|
|
151
|
+
return idx
|