vulnlens 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vulnlens/__init__.py +3 -0
- vulnlens/__main__.py +5 -0
- vulnlens/cli.py +278 -0
- vulnlens/context.py +39 -0
- vulnlens/doctor.py +102 -0
- vulnlens/engines/__init__.py +20 -0
- vulnlens/engines/base.py +67 -0
- vulnlens/engines/deps.py +59 -0
- vulnlens/engines/secrets.py +220 -0
- vulnlens/engines/semgrep.py +150 -0
- vulnlens/fixplan.py +61 -0
- vulnlens/htmlreport.py +29 -0
- vulnlens/ignore.py +140 -0
- vulnlens/intel/__init__.py +0 -0
- vulnlens/intel/cvss.py +49 -0
- vulnlens/intel/exploit.py +86 -0
- vulnlens/intel/osv.py +243 -0
- vulnlens/lockfiles.py +384 -0
- vulnlens/models.py +161 -0
- vulnlens/net.py +93 -0
- vulnlens/priority.py +241 -0
- vulnlens/report.py +155 -0
- vulnlens/sarif.py +97 -0
- vulnlens/scan.py +148 -0
- vulnlens/server.py +321 -0
- vulnlens/source.py +191 -0
- vulnlens/triage.py +381 -0
- vulnlens/web/index.html +65 -0
- vulnlens-0.3.0.dist-info/METADATA +368 -0
- vulnlens-0.3.0.dist-info/RECORD +34 -0
- vulnlens-0.3.0.dist-info/WHEEL +5 -0
- vulnlens-0.3.0.dist-info/entry_points.txt +2 -0
- vulnlens-0.3.0.dist-info/licenses/LICENSE +21 -0
- vulnlens-0.3.0.dist-info/top_level.txt +1 -0
vulnlens/__init__.py
ADDED
vulnlens/__main__.py
ADDED
vulnlens/cli.py
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
"""Command-line interface: `vulnlens scan <path-or-repo-url>`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import traceback
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from . import __version__, net
|
|
13
|
+
from .engines import ALIASES, ENGINES, resolve_engine
|
|
14
|
+
from .htmlreport import render_html
|
|
15
|
+
from .ignore import IgnoreRules
|
|
16
|
+
from .models import PRIORITIES
|
|
17
|
+
from .report import render, use_color
|
|
18
|
+
from .sarif import to_sarif
|
|
19
|
+
from .scan import run_scan
|
|
20
|
+
from .source import TargetError, is_remote, may_verify_secrets, resolve_target
|
|
21
|
+
from .triage import DEFAULT_MODEL, Triager
|
|
22
|
+
|
|
23
|
+
EXIT_OK, EXIT_FINDINGS, EXIT_ERROR = 0, 1, 2
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
27
|
+
p = argparse.ArgumentParser(
|
|
28
|
+
prog="vulnlens",
|
|
29
|
+
description="Scan code, dependencies and secrets, then rank findings by real-world exploitability.",
|
|
30
|
+
)
|
|
31
|
+
p.add_argument("--version", action="version", version=f"vulnlens {__version__}")
|
|
32
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
33
|
+
|
|
34
|
+
s = sub.add_parser("scan", help="scan a local directory or a public git repository URL")
|
|
35
|
+
s.add_argument("target", help="directory path or https://github.com/owner/repo")
|
|
36
|
+
s.add_argument("-f", "--format", choices=["table", "json", "sarif"], default="table",
|
|
37
|
+
help="output format for stdout / --output (default: table)")
|
|
38
|
+
s.add_argument("-o", "--output", type=Path, help="write the --format output to this file instead of stdout")
|
|
39
|
+
s.add_argument("--sarif", type=Path, metavar="FILE", help="also write a SARIF 2.1.0 file")
|
|
40
|
+
s.add_argument("--json", type=Path, metavar="FILE", dest="json_out", help="also write a JSON report")
|
|
41
|
+
s.add_argument("--html", type=Path, metavar="FILE", dest="html_out",
|
|
42
|
+
help="also write a standalone HTML report (the dashboard, no server needed)")
|
|
43
|
+
s.add_argument("--engines", default=",".join(ENGINES),
|
|
44
|
+
help=f"comma-separated engines to run (default: {','.join(ENGINES)})")
|
|
45
|
+
s.add_argument("--offline", action="store_true",
|
|
46
|
+
help="no network: skip OSV lookups, KEV/EPSS enrichment and secret verification")
|
|
47
|
+
s.add_argument("--ai-triage", action="store_true",
|
|
48
|
+
help="have Claude review each code finding in context "
|
|
49
|
+
"(needs ANTHROPIC_API_KEY; sends code snippets to the API)")
|
|
50
|
+
s.add_argument("--ai-model", default=DEFAULT_MODEL, help=f"model for --ai-triage (default: {DEFAULT_MODEL})")
|
|
51
|
+
s.add_argument("--ai-max-cost", type=float, default=2.0, metavar="USD",
|
|
52
|
+
help="skip triage if its estimated cost is above this (default: 2.00)")
|
|
53
|
+
s.add_argument("--ai-dry-run", action="store_true",
|
|
54
|
+
help="print how many findings triage would review and the estimated cost, then send nothing")
|
|
55
|
+
s.add_argument("--ai-effort", default="medium", choices=["low", "medium", "high", "xhigh", "max"],
|
|
56
|
+
help="reasoning effort for --ai-triage (default: medium)")
|
|
57
|
+
s.add_argument("--ai-max", type=int, default=50, metavar="N",
|
|
58
|
+
help="review at most N code findings, most urgent first (default: 50)")
|
|
59
|
+
s.add_argument("--no-verify-secrets", action="store_true",
|
|
60
|
+
help="do not test found credentials against their provider (remote repository URLs are "
|
|
61
|
+
"never verified; use this for local copies of code you don't own)")
|
|
62
|
+
s.add_argument("--fail-on", choices=[*PRIORITIES, "act", "review"], metavar="{act,review,P0-P4}",
|
|
63
|
+
help="exit 1 if any finding is in the Act-now tier (act), in Act or Review (review), or at this "
|
|
64
|
+
"priority or more urgent (P0-P4); exit 2 if an engine did not run (for CI)")
|
|
65
|
+
s.add_argument("--exclude", action="append", default=[], metavar="GLOB",
|
|
66
|
+
help="skip findings in matching paths, e.g. 'tests/**' (repeatable)")
|
|
67
|
+
s.add_argument("--baseline", type=Path, metavar="REPORT.json",
|
|
68
|
+
help="only report findings that are not in this earlier JSON report (for CI on pull requests)")
|
|
69
|
+
s.add_argument("--ignore-file", type=Path, metavar="FILE",
|
|
70
|
+
help="extra ignore rules in .vulnlensignore format (the scan root's .vulnlensignore is "
|
|
71
|
+
"read automatically for local scans)")
|
|
72
|
+
s.add_argument("--top", type=int, default=15, help="findings to show in the table (default: 15)")
|
|
73
|
+
s.add_argument("--semgrep-config", default="p/default", help="Semgrep ruleset (default: p/default)")
|
|
74
|
+
s.add_argument("--bin-dir", type=Path, help="directory containing semgrep/trufflehog binaries")
|
|
75
|
+
s.add_argument("--no-cache", action="store_true", help="do not read or write the intel cache")
|
|
76
|
+
s.add_argument("--no-color", action="store_true")
|
|
77
|
+
s.add_argument("-q", "--quiet", action="store_true", help="no progress messages on stderr")
|
|
78
|
+
|
|
79
|
+
t = sub.add_parser("html", help="turn saved JSON reports (from scan --json) into one standalone HTML page")
|
|
80
|
+
t.add_argument("reports", nargs="+", type=Path, metavar="REPORT.json")
|
|
81
|
+
t.add_argument("-o", "--output", type=Path, required=True, help="HTML file to write")
|
|
82
|
+
t.add_argument("--title", help="page title")
|
|
83
|
+
|
|
84
|
+
d = sub.add_parser("doctor", help="check that the scan engines are installed and current, and how to fix them")
|
|
85
|
+
d.add_argument("--bin-dir", type=Path, help="directory containing semgrep/trufflehog binaries")
|
|
86
|
+
d.add_argument("--offline", action="store_true", help="skip the network reachability checks")
|
|
87
|
+
|
|
88
|
+
v = sub.add_parser("serve", help="start the local web dashboard")
|
|
89
|
+
v.add_argument("--host", default="127.0.0.1", help="interface to bind (default: 127.0.0.1, local only)")
|
|
90
|
+
v.add_argument("--port", type=int, default=8765, help="port (default: 8765; 0 picks a free one)")
|
|
91
|
+
v.add_argument("--load", type=Path, action="append", default=[], metavar="REPORT.json",
|
|
92
|
+
help="preload a JSON report written by `scan --json` (repeatable)")
|
|
93
|
+
v.add_argument("--semgrep-config", default="p/default", help="Semgrep ruleset (default: p/default)")
|
|
94
|
+
v.add_argument("--bin-dir", type=Path, help="directory containing semgrep/trufflehog binaries")
|
|
95
|
+
v.add_argument("--open", action="store_true", help="open the dashboard in your browser")
|
|
96
|
+
v.add_argument("--no-history", action="store_true",
|
|
97
|
+
help="do not save finished scans or restore earlier ones (kept in the VulnLens cache folder)")
|
|
98
|
+
return p
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def cmd_doctor(args: argparse.Namespace) -> int:
|
|
102
|
+
from .doctor import render, run_checks
|
|
103
|
+
|
|
104
|
+
checks = run_checks(bin_dir=args.bin_dir, offline=args.offline)
|
|
105
|
+
print(render(checks))
|
|
106
|
+
return EXIT_OK if all(c.ok for c in checks if c.required) else EXIT_FINDINGS
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def cmd_html(args: argparse.Namespace) -> int:
|
|
110
|
+
reports = []
|
|
111
|
+
for path in args.reports:
|
|
112
|
+
try:
|
|
113
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
114
|
+
except (OSError, ValueError) as e:
|
|
115
|
+
print(f"vulnlens: could not read {path}: {e}", file=sys.stderr)
|
|
116
|
+
return EXIT_ERROR
|
|
117
|
+
if "findings" not in data or "summary" not in data:
|
|
118
|
+
print(f"vulnlens: {path} is not a VulnLens JSON report", file=sys.stderr)
|
|
119
|
+
return EXIT_ERROR
|
|
120
|
+
reports.append(data)
|
|
121
|
+
_write(args.output, render_html(reports, title=args.title))
|
|
122
|
+
print(f"wrote {args.output} ({len(reports)} report(s))", file=sys.stderr)
|
|
123
|
+
return EXIT_OK
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def cmd_serve(args: argparse.Namespace) -> int:
|
|
127
|
+
import webbrowser
|
|
128
|
+
|
|
129
|
+
from .server import ScanStore, serve
|
|
130
|
+
|
|
131
|
+
if args.host not in ("127.0.0.1", "localhost", "::1"):
|
|
132
|
+
print(f"vulnlens: warning: binding to {args.host} exposes the dashboard (and the ability to start "
|
|
133
|
+
"scans) to your network", file=sys.stderr)
|
|
134
|
+
store = ScanStore(bin_dir=args.bin_dir, semgrep_config=args.semgrep_config,
|
|
135
|
+
history_dir=None if args.no_history else net.default_cache_dir() / "history")
|
|
136
|
+
for path in args.load:
|
|
137
|
+
try:
|
|
138
|
+
store.load_report(path)
|
|
139
|
+
except (OSError, ValueError) as e:
|
|
140
|
+
print(f"vulnlens: could not load {path}: {e}", file=sys.stderr)
|
|
141
|
+
return EXIT_ERROR
|
|
142
|
+
|
|
143
|
+
def ready(url: str) -> None:
|
|
144
|
+
print(f"VulnLens dashboard running at {url} (Ctrl+C to stop)", flush=True)
|
|
145
|
+
if args.open:
|
|
146
|
+
webbrowser.open(url)
|
|
147
|
+
|
|
148
|
+
try:
|
|
149
|
+
serve(args.host, args.port, store, on_ready=ready)
|
|
150
|
+
except OSError as e:
|
|
151
|
+
print(f"vulnlens: could not start server on {args.host}:{args.port}: {e}", file=sys.stderr)
|
|
152
|
+
return EXIT_ERROR
|
|
153
|
+
return EXIT_OK
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _write(path: Path, text: str) -> None:
|
|
157
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
158
|
+
path.write_text(text, encoding="utf-8")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _next_steps(args: argparse.Namespace, report) -> list[str]:
|
|
162
|
+
"""What a first-time user should do after reading the table."""
|
|
163
|
+
s, hints = report.summary, []
|
|
164
|
+
target = args.target if " " not in args.target else f'"{args.target}"'
|
|
165
|
+
if not s["complete"]:
|
|
166
|
+
hints.append("vulnlens doctor see which engine is missing and how to install it")
|
|
167
|
+
if not args.html_out:
|
|
168
|
+
hints.append(f"vulnlens scan {target} --html report.html full interactive report (one offline file)")
|
|
169
|
+
if s.get("by_tier", {}).get("review") and not args.ai_triage:
|
|
170
|
+
hints.append("add --ai-dry-run, then --ai-triage let Claude judge the Review pile (shows the cost first)")
|
|
171
|
+
if not args.fail_on:
|
|
172
|
+
hints.append("add it to CI: see 'GitHub Action' in the README (fails the build on Act-now findings)")
|
|
173
|
+
return hints
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def cmd_scan(args: argparse.Namespace) -> int:
|
|
177
|
+
requested = [e.strip() for e in args.engines.split(",") if e.strip()]
|
|
178
|
+
unknown = [e for e in requested if not resolve_engine(e)]
|
|
179
|
+
engines = list(dict.fromkeys(resolve_engine(e) for e in requested if resolve_engine(e)))
|
|
180
|
+
if unknown:
|
|
181
|
+
names = ", ".join(f"{k} ({v})" for v, k in ALIASES.items())
|
|
182
|
+
print(f"vulnlens: unknown engine(s): {', '.join(unknown)} (choose from {names})", file=sys.stderr)
|
|
183
|
+
return EXIT_ERROR
|
|
184
|
+
|
|
185
|
+
def progress(msg: str) -> None:
|
|
186
|
+
if not args.quiet:
|
|
187
|
+
print(f"[vulnlens] {msg}", file=sys.stderr, flush=True)
|
|
188
|
+
|
|
189
|
+
verify = may_verify_secrets(args.target, requested=not args.no_verify_secrets)
|
|
190
|
+
if is_remote(args.target) and not args.no_verify_secrets:
|
|
191
|
+
progress("secret verification is off for remote repositories (their credentials are not yours to test)")
|
|
192
|
+
try:
|
|
193
|
+
with resolve_target(args.target) as root:
|
|
194
|
+
progress(f"cloned {args.target}" if is_remote(args.target) else f"scanning {root}")
|
|
195
|
+
report = run_scan(
|
|
196
|
+
root,
|
|
197
|
+
target_label=args.target,
|
|
198
|
+
engines=engines,
|
|
199
|
+
offline=args.offline,
|
|
200
|
+
verify_secrets=verify,
|
|
201
|
+
triager=(Triager(model=args.ai_model, effort=args.ai_effort, max_findings=args.ai_max,
|
|
202
|
+
cache=net.Cache(enabled=not args.no_cache), max_cost=args.ai_max_cost,
|
|
203
|
+
dry_run=args.ai_dry_run)
|
|
204
|
+
if args.ai_triage or args.ai_dry_run else None),
|
|
205
|
+
semgrep_config=args.semgrep_config,
|
|
206
|
+
bin_dir=args.bin_dir,
|
|
207
|
+
cache=net.Cache(enabled=not args.no_cache),
|
|
208
|
+
progress=progress,
|
|
209
|
+
ignore=IgnoreRules.load(root, excludes=args.exclude, baseline=args.baseline,
|
|
210
|
+
trust_repo=not is_remote(args.target), ignore_file=args.ignore_file),
|
|
211
|
+
)
|
|
212
|
+
except TargetError as e:
|
|
213
|
+
print(f"vulnlens: {e}", file=sys.stderr)
|
|
214
|
+
return EXIT_ERROR
|
|
215
|
+
|
|
216
|
+
outputs = {
|
|
217
|
+
"table": lambda: render(report, top=args.top, color=use_color() and not args.no_color and not args.output,
|
|
218
|
+
hints=_next_steps(args, report)),
|
|
219
|
+
"json": lambda: json.dumps(report.to_dict(), indent=2),
|
|
220
|
+
"sarif": lambda: json.dumps(to_sarif(report.findings), indent=2),
|
|
221
|
+
}
|
|
222
|
+
main_out = outputs[args.format]()
|
|
223
|
+
if args.output:
|
|
224
|
+
_write(args.output, main_out)
|
|
225
|
+
progress(f"wrote {args.format} report to {args.output}")
|
|
226
|
+
else:
|
|
227
|
+
sys.stdout.write(main_out if main_out.endswith("\n") else main_out + "\n")
|
|
228
|
+
if args.sarif:
|
|
229
|
+
_write(args.sarif, outputs["sarif"]())
|
|
230
|
+
progress(f"wrote SARIF to {args.sarif}")
|
|
231
|
+
if args.json_out:
|
|
232
|
+
_write(args.json_out, outputs["json"]())
|
|
233
|
+
progress(f"wrote JSON to {args.json_out}")
|
|
234
|
+
if args.html_out:
|
|
235
|
+
_write(args.html_out, render_html([report.to_dict()], title=args.target))
|
|
236
|
+
progress(f"wrote HTML report to {args.html_out}")
|
|
237
|
+
|
|
238
|
+
summary = report.summary
|
|
239
|
+
if not summary["complete"]:
|
|
240
|
+
progress(f"warning: scan incomplete ({', '.join(summary['incomplete_engines'])} did not run); "
|
|
241
|
+
"`vulnlens doctor` shows what is missing")
|
|
242
|
+
if args.fail_on:
|
|
243
|
+
if not summary["complete"]:
|
|
244
|
+
return EXIT_ERROR # a CI gate must not pass on a scan that did not fully run
|
|
245
|
+
if args.fail_on in ("act", "review"):
|
|
246
|
+
tiers = {"act"} if args.fail_on == "act" else {"act", "review"}
|
|
247
|
+
failing = any(f.tier in tiers for f in report.findings)
|
|
248
|
+
else:
|
|
249
|
+
threshold = PRIORITIES.index(args.fail_on)
|
|
250
|
+
failing = any(PRIORITIES.index(f.priority) <= threshold for f in report.findings)
|
|
251
|
+
if failing:
|
|
252
|
+
return EXIT_FINDINGS
|
|
253
|
+
return EXIT_OK
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def main(argv: list[str] | None = None) -> int:
|
|
257
|
+
for stream in (sys.stdout, sys.stderr):
|
|
258
|
+
if hasattr(stream, "reconfigure"):
|
|
259
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
260
|
+
args = build_parser().parse_args(argv)
|
|
261
|
+
commands = {"scan": cmd_scan, "serve": cmd_serve, "html": cmd_html, "doctor": cmd_doctor}
|
|
262
|
+
try:
|
|
263
|
+
return commands[args.command](args)
|
|
264
|
+
except KeyboardInterrupt:
|
|
265
|
+
print("vulnlens: interrupted", file=sys.stderr)
|
|
266
|
+
return 130
|
|
267
|
+
except Exception as e:
|
|
268
|
+
# Exit 1 means "findings at or above --fail-on". A crash must never look like that to CI.
|
|
269
|
+
if os.environ.get("VULNLENS_DEBUG"):
|
|
270
|
+
traceback.print_exc()
|
|
271
|
+
print(f"vulnlens: internal error: {type(e).__name__}: {e}"
|
|
272
|
+
+ ("" if os.environ.get("VULNLENS_DEBUG") else " (set VULNLENS_DEBUG=1 for the traceback)"),
|
|
273
|
+
file=sys.stderr)
|
|
274
|
+
return EXIT_ERROR
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
if __name__ == "__main__":
|
|
278
|
+
sys.exit(main())
|
vulnlens/context.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Where a finding lives: production code, or tests / examples / docs / vendored third-party code.
|
|
2
|
+
|
|
3
|
+
The same bug matters far less in a test fixture or an example notebook than in the shipped app,
|
|
4
|
+
and in the 50-repo study 62% of secret hits and over a fifth of code and dependency findings were
|
|
5
|
+
in such paths. Prioritization caps non-production findings at P3 "Low" (see priority.py).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
CONTEXTS = ("prod", "test", "example", "docs", "vendored")
|
|
13
|
+
LABELS = {"test": "test code", "example": "example code", "docs": "documentation", "vendored": "vendored third-party code"}
|
|
14
|
+
|
|
15
|
+
_RULES = [
|
|
16
|
+
("vendored", re.compile(r"(^|/)(vendor|vendored|third[_-]?party|external|extern|bower_components|site-packages)(/|$)"
|
|
17
|
+
r"|\.min\.(js|css)$|\.bundle\.js$", re.I)),
|
|
18
|
+
("test", re.compile(r"(^|/)(tests?|testing|__tests__|__mocks__|specs?|fixtures?|mocks?|testdata|e2e)(/|$)"
|
|
19
|
+
r"|(^|/)(tests?[_-][^/]*|[^/]*[_-]tests?\.[a-z0-9]+|[^/]*\.(test|spec)\.[a-z0-9]+|conftest\.py)$",
|
|
20
|
+
re.I)),
|
|
21
|
+
("example", re.compile(r"(^|/)(examples?|samples?|demos?|cookbooks?|tutorials?|notebooks?|playground|"
|
|
22
|
+
r"research_projects|benchmarks?)(/|$)|\.ipynb$", re.I)),
|
|
23
|
+
("docs", re.compile(r"(^|/)(docs?|documentation)(/|$)|\.(md|mdx|rst|adoc)$", re.I)),
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def classify(path: str | None) -> str:
|
|
28
|
+
if not path:
|
|
29
|
+
return "prod"
|
|
30
|
+
for context, pattern in _RULES:
|
|
31
|
+
if pattern.search(path):
|
|
32
|
+
return context
|
|
33
|
+
return "prod"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def best(paths: list[str | None]) -> str:
|
|
37
|
+
"""Context of a finding seen in several files: production wins, since that's where it matters."""
|
|
38
|
+
found = [classify(p) for p in paths] or ["prod"]
|
|
39
|
+
return "prod" if "prod" in found else found[0]
|
vulnlens/doctor.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""`vulnlens doctor`: check that every engine is installed and current, and say exactly how to fix it.
|
|
2
|
+
|
|
3
|
+
`pip install vulnlens` brings no scanners, so without this a first scan just says "INCOMPLETE".
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import platform
|
|
9
|
+
import shutil
|
|
10
|
+
import sys
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from . import net
|
|
15
|
+
from .engines.base import find_tool
|
|
16
|
+
from .engines.secrets import MIN_VERSION, RECOMMENDED_VERSION, tool_version
|
|
17
|
+
from .engines.semgrep import _semgrep_version
|
|
18
|
+
from .intel.exploit import EPSS_API, KEV_FEED
|
|
19
|
+
from .intel.osv import OSV_API
|
|
20
|
+
|
|
21
|
+
RELEASES = "https://github.com/trufflesecurity/trufflehog/releases"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Check:
|
|
26
|
+
name: str
|
|
27
|
+
ok: bool
|
|
28
|
+
detail: str
|
|
29
|
+
fix: str = ""
|
|
30
|
+
required: bool = True
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _trufflehog_fix() -> str:
|
|
34
|
+
v = RECOMMENDED_VERSION
|
|
35
|
+
system = platform.system()
|
|
36
|
+
if system == "Darwin":
|
|
37
|
+
return "brew install trufflehog"
|
|
38
|
+
arch = "arm64" if platform.machine().lower() in ("arm64", "aarch64") else "amd64"
|
|
39
|
+
os_name = "windows" if system == "Windows" else "linux"
|
|
40
|
+
return (f"download trufflehog_{v}_{os_name}_{arch}.tar.gz and trufflehog_{v}_checksums.txt from "
|
|
41
|
+
f"{RELEASES}/tag/v{v}, check the SHA-256, extract it, and put trufflehog on your PATH "
|
|
42
|
+
"(or pass --bin-dir to vulnlens)")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def run_checks(bin_dir: Path | None = None, offline: bool = False) -> list[Check]:
|
|
46
|
+
checks = [Check("python", sys.version_info >= (3, 10), platform.python_version(), "VulnLens needs Python 3.10+")]
|
|
47
|
+
checks.append(Check("git", bool(shutil.which("git")), shutil.which("git") or "not found",
|
|
48
|
+
"install git (needed to scan repository URLs)", required=False))
|
|
49
|
+
|
|
50
|
+
semgrep = find_tool("semgrep", bin_dir) or find_tool("pysemgrep", bin_dir)
|
|
51
|
+
if semgrep:
|
|
52
|
+
checks.append(Check("semgrep", True, f"{_semgrep_version(semgrep) or 'unknown version'} at {semgrep}"))
|
|
53
|
+
else:
|
|
54
|
+
checks.append(Check("semgrep", False, "not found", 'pip install semgrep (or: pip install "vulnlens[semgrep]")'))
|
|
55
|
+
|
|
56
|
+
trufflehog = find_tool("trufflehog", bin_dir)
|
|
57
|
+
if trufflehog:
|
|
58
|
+
version = tool_version(trufflehog)
|
|
59
|
+
shown = ".".join(map(str, version)) if version else "unknown version"
|
|
60
|
+
current = version is not None and version >= MIN_VERSION
|
|
61
|
+
checks.append(Check("trufflehog", current, f"{shown} at {trufflehog}",
|
|
62
|
+
"" if current else f"outdated (noisy detectors); upgrade to {RECOMMENDED_VERSION}: "
|
|
63
|
+
+ _trufflehog_fix()))
|
|
64
|
+
else:
|
|
65
|
+
checks.append(Check("trufflehog", False, "not found", _trufflehog_fix()))
|
|
66
|
+
|
|
67
|
+
if not offline:
|
|
68
|
+
for name, url in (("osv.dev", f"{OSV_API}/vulns/GHSA-jfh8-c2jp-5v3q"), ("CISA KEV", KEV_FEED),
|
|
69
|
+
("FIRST EPSS", f"{EPSS_API}?cve=CVE-2021-44228")):
|
|
70
|
+
try:
|
|
71
|
+
net.get_json(url, timeout=20)
|
|
72
|
+
checks.append(Check(name, True, "reachable"))
|
|
73
|
+
except net.NetworkError as e:
|
|
74
|
+
checks.append(Check(name, False, str(e)[:120], "check your network or proxy; scans will be incomplete"))
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
import anthropic # noqa: F401
|
|
78
|
+
has_sdk = True
|
|
79
|
+
except ImportError:
|
|
80
|
+
has_sdk = False
|
|
81
|
+
import os
|
|
82
|
+
|
|
83
|
+
has_key = bool(os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_AUTH_TOKEN"))
|
|
84
|
+
checks.append(Check("ai triage", has_sdk and has_key,
|
|
85
|
+
"ready" if has_sdk and has_key else ("no API key set" if has_sdk else "SDK not installed"),
|
|
86
|
+
'optional: pip install "vulnlens[ai]" and set ANTHROPIC_API_KEY', required=False))
|
|
87
|
+
return checks
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def render(checks: list[Check]) -> str:
|
|
91
|
+
lines = ["", "VulnLens doctor", ""]
|
|
92
|
+
for c in checks:
|
|
93
|
+
mark = "ok " if c.ok else ("MISSING" if c.required else "optional")
|
|
94
|
+
lines.append(f" {c.name:<11} {mark:<9} {c.detail}")
|
|
95
|
+
fixes = [c for c in checks if not c.ok and c.fix]
|
|
96
|
+
if fixes:
|
|
97
|
+
lines += ["", "To fix:"]
|
|
98
|
+
lines += [f" {c.name}: {c.fix}" for c in fixes]
|
|
99
|
+
ready = all(c.ok for c in checks if c.required)
|
|
100
|
+
lines += ["", "Ready to scan." if ready else
|
|
101
|
+
"Not ready: a missing engine makes scans incomplete, an outdated one makes them noisy.", ""]
|
|
102
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from .deps import DependencyEngine
|
|
2
|
+
from .secrets import TruffleHogEngine
|
|
3
|
+
from .semgrep import SemgrepEngine
|
|
4
|
+
|
|
5
|
+
ENGINES = {
|
|
6
|
+
"code": SemgrepEngine,
|
|
7
|
+
"deps": DependencyEngine,
|
|
8
|
+
"secrets": TruffleHogEngine,
|
|
9
|
+
}
|
|
10
|
+
# Reports name engines by tool; the CLI by area. Accept either.
|
|
11
|
+
ALIASES = {"semgrep": "code", "osv": "deps", "trufflehog": "secrets"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def resolve_engine(name: str) -> str | None:
|
|
15
|
+
name = name.strip().lower()
|
|
16
|
+
name = ALIASES.get(name, name)
|
|
17
|
+
return name if name in ENGINES else None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
__all__ = ["ALIASES", "ENGINES", "DependencyEngine", "SemgrepEngine", "TruffleHogEngine", "resolve_engine"]
|
vulnlens/engines/base.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Shared helpers for engines that wrap an external CLI tool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import subprocess
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from ..models import EngineState
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class ToolRun:
|
|
16
|
+
state: EngineState
|
|
17
|
+
stdout: str = ""
|
|
18
|
+
stderr: str = ""
|
|
19
|
+
exit_code: int | None = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def find_tool(name: str, bin_dir: Path | None = None) -> str | None:
|
|
23
|
+
if bin_dir:
|
|
24
|
+
for candidate in (bin_dir / name, bin_dir / f"{name}.exe"):
|
|
25
|
+
if candidate.is_file() and candidate.stat().st_size > 1024:
|
|
26
|
+
return str(candidate)
|
|
27
|
+
return shutil.which(name)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def run_tool(cmd: list[str], cwd: Path, timeout: int, ok_codes: tuple[int, ...] = (0,)) -> ToolRun:
|
|
31
|
+
env = {**os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8"}
|
|
32
|
+
try:
|
|
33
|
+
proc = subprocess.run(
|
|
34
|
+
cmd,
|
|
35
|
+
cwd=cwd,
|
|
36
|
+
capture_output=True,
|
|
37
|
+
timeout=timeout,
|
|
38
|
+
env=env,
|
|
39
|
+
encoding="utf-8",
|
|
40
|
+
errors="replace",
|
|
41
|
+
)
|
|
42
|
+
except subprocess.TimeoutExpired:
|
|
43
|
+
return ToolRun(EngineState.TIMEOUT, stderr=f"timed out after {timeout}s")
|
|
44
|
+
except OSError as e:
|
|
45
|
+
return ToolRun(EngineState.ERROR, stderr=str(e))
|
|
46
|
+
state = EngineState.OK if proc.returncode in ok_codes else EngineState.ERROR
|
|
47
|
+
return ToolRun(state, proc.stdout, proc.stderr, proc.returncode)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def failure_message(run: ToolRun) -> str:
|
|
51
|
+
"""Most useful part of a failed tool's stderr: error lines first, skipping info-level log noise."""
|
|
52
|
+
lines = [ln for ln in run.stderr.splitlines() if ln.strip() and '"level":"info' not in ln]
|
|
53
|
+
errors = [ln for ln in lines if "error" in ln.lower() or "fatal" in ln.lower()] or lines
|
|
54
|
+
prefix = f"exit code {run.exit_code}: " if run.exit_code is not None else ""
|
|
55
|
+
return (prefix + " | ".join(errors[-3:]))[-400:] or prefix.rstrip(": ")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def relpath(path: str | None, root: Path) -> str | None:
|
|
59
|
+
if not path:
|
|
60
|
+
return None
|
|
61
|
+
p = Path(path)
|
|
62
|
+
if p.is_absolute():
|
|
63
|
+
try:
|
|
64
|
+
p = p.resolve().relative_to(root.resolve())
|
|
65
|
+
except ValueError:
|
|
66
|
+
pass
|
|
67
|
+
return p.as_posix().removeprefix("./")
|
vulnlens/engines/deps.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Software composition analysis: lockfiles -> OSV.dev, no external binary needed."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from .. import net
|
|
9
|
+
from ..intel import osv
|
|
10
|
+
from ..lockfiles import collect_packages, uncovered_manifests
|
|
11
|
+
from ..models import EngineResult, EngineState
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DependencyEngine:
|
|
15
|
+
name = "osv"
|
|
16
|
+
|
|
17
|
+
def __init__(self, offline: bool = False, cache: net.Cache | None = None, **_):
|
|
18
|
+
self.offline = offline
|
|
19
|
+
self.cache = cache or net.Cache()
|
|
20
|
+
|
|
21
|
+
def run(self, root: Path) -> EngineResult:
|
|
22
|
+
start = time.monotonic()
|
|
23
|
+
unpinned: dict[str, int] = {}
|
|
24
|
+
packages, lockfiles, errors = collect_packages(root, unpinned)
|
|
25
|
+
unread = uncovered_manifests(root)
|
|
26
|
+
gaps = [f"{_names(unread)} not read (format not supported yet)"] if unread else []
|
|
27
|
+
if errors:
|
|
28
|
+
gaps.append(f"{_names(e.split(':', 1)[0] for e in errors)} failed to parse")
|
|
29
|
+
if unpinned:
|
|
30
|
+
gaps.append(f"{sum(unpinned.values())} dependencies without an exact version in {_names(sorted(unpinned))} "
|
|
31
|
+
"(a range like >=2.0 can't be matched to advisories; pin them or add a lockfile)")
|
|
32
|
+
if not lockfiles:
|
|
33
|
+
msg = "no supported lockfile found" + (f"; found {_names(unread)}, which VulnLens can't read yet" if unread else "")
|
|
34
|
+
return EngineResult(self.name, EngineState.SKIPPED, message=msg)
|
|
35
|
+
if self.offline:
|
|
36
|
+
return EngineResult(self.name, EngineState.SKIPPED,
|
|
37
|
+
message=f"offline mode: {len(packages)} packages not checked against OSV")
|
|
38
|
+
stats: dict[str, int] = {}
|
|
39
|
+
try:
|
|
40
|
+
findings = osv.scan_packages(packages, self.cache, stats=stats)
|
|
41
|
+
except net.NetworkError as e:
|
|
42
|
+
return EngineResult(self.name, EngineState.ERROR, duration=time.monotonic() - start, message=str(e))
|
|
43
|
+
msg = f"{len(packages)} packages from {len(lockfiles)} file(s)"
|
|
44
|
+
if errors:
|
|
45
|
+
msg += f"; {len(errors)} file(s) failed to parse"
|
|
46
|
+
if stats.get("withdrawn"):
|
|
47
|
+
msg += f"; {stats['withdrawn']} withdrawn advisory(ies) ignored"
|
|
48
|
+
state = EngineState.OK
|
|
49
|
+
if stats.get("details_failed"):
|
|
50
|
+
# Without the advisory record a finding has no severity, summary or fix: the ranking is incomplete.
|
|
51
|
+
state = EngineState.PARTIAL
|
|
52
|
+
msg += f"; {stats['details_failed']} advisory record(s) could not be fetched, so their findings are unranked"
|
|
53
|
+
return EngineResult(self.name, state, findings, time.monotonic() - start, msg, coverage_gaps=gaps)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _names(paths, limit: int = 3) -> str:
|
|
57
|
+
paths = list(paths)
|
|
58
|
+
shown = ", ".join(paths[:limit])
|
|
59
|
+
return shown + (f" and {len(paths) - limit} more" if len(paths) > limit else "")
|