dkatchr 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkatchr/__init__.py +3 -0
- dkatchr/__main__.py +6 -0
- dkatchr/cli/__init__.py +7 -0
- dkatchr/cli/args.py +84 -0
- dkatchr/cli/dry_run.py +68 -0
- dkatchr/cli/main.py +14 -0
- dkatchr/cli/run.py +814 -0
- dkatchr/clients/__init__.py +3 -0
- dkatchr/clients/epss.py +153 -0
- dkatchr/clients/exploitdb.py +155 -0
- dkatchr/clients/github.py +482 -0
- dkatchr/clients/kev.py +143 -0
- dkatchr/clients/metasploit.py +147 -0
- dkatchr/clients/nvd.py +121 -0
- dkatchr/clients/osv.py +191 -0
- dkatchr/clients/ratelimit.py +33 -0
- dkatchr/clients/registry_meta.py +280 -0
- dkatchr/clients/resolvers/__init__.py +335 -0
- dkatchr/clients/resolvers/_base.py +153 -0
- dkatchr/clients/resolvers/crates.py +51 -0
- dkatchr/clients/resolvers/maven.py +144 -0
- dkatchr/clients/resolvers/nuget.py +216 -0
- dkatchr/clients/resolvers/packagist.py +164 -0
- dkatchr/clients/resolvers/pypi.py +190 -0
- dkatchr/clients/resolvers/rubygems.py +132 -0
- dkatchr/clients/top_packages.py +302 -0
- dkatchr/config.py +428 -0
- dkatchr/core/__init__.py +3 -0
- dkatchr/core/attribution.py +464 -0
- dkatchr/core/confusion.py +334 -0
- dkatchr/core/inventory.py +104 -0
- dkatchr/core/osv_enrichment.py +358 -0
- dkatchr/core/reachability.py +793 -0
- dkatchr/core/rules.py +132 -0
- dkatchr/core/scanner.py +170 -0
- dkatchr/core/typosquat.py +294 -0
- dkatchr/logger.py +121 -0
- dkatchr/output/__init__.py +2 -0
- dkatchr/output/csv_writer.py +37 -0
- dkatchr/output/schema.py +83 -0
- dkatchr/parsers/__init__.py +74 -0
- dkatchr/parsers/_common.py +22 -0
- dkatchr/parsers/cargo.py +49 -0
- dkatchr/parsers/composer.py +86 -0
- dkatchr/parsers/go.py +42 -0
- dkatchr/parsers/maven.py +71 -0
- dkatchr/parsers/npm.py +278 -0
- dkatchr/parsers/nuget.py +71 -0
- dkatchr/parsers/pypi.py +214 -0
- dkatchr/parsers/ruby.py +119 -0
- dkatchr/storage/__init__.py +2 -0
- dkatchr/storage/attribution_cache.py +101 -0
- dkatchr/storage/inventory_cache.py +84 -0
- dkatchr/storage/reachability_cache.py +108 -0
- dkatchr-0.3.0.dist-info/METADATA +228 -0
- dkatchr-0.3.0.dist-info/RECORD +60 -0
- dkatchr-0.3.0.dist-info/WHEEL +5 -0
- dkatchr-0.3.0.dist-info/entry_points.txt +2 -0
- dkatchr-0.3.0.dist-info/licenses/LICENSE +21 -0
- dkatchr-0.3.0.dist-info/top_level.txt +1 -0
dkatchr/cli/run.py
ADDED
|
@@ -0,0 +1,814 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CLI scan orchestration — the body of `main()`.
|
|
3
|
+
|
|
4
|
+
EXTRACTED FROM: dkatchr/cli.py (former main() + _scan_batch).
|
|
5
|
+
WHY: The old main() was 100+ lines mixing argparse parsing, batch building,
|
|
6
|
+
threading, CSV writing, and OSV pass orchestration. This module owns the
|
|
7
|
+
orchestration only — argparse lives in args.py, CSV in output/csv_writer.py,
|
|
8
|
+
dry-run in dry_run.py.
|
|
9
|
+
|
|
10
|
+
This is the CLI's equivalent of web/services/scan_runner.py: same Scanner,
|
|
11
|
+
same enrichment, different I/O at the edges (CSV vs SQLite, stdout/tqdm
|
|
12
|
+
vs WebSocket).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
21
|
+
|
|
22
|
+
from dkatchr.clients.epss import EPSSClient
|
|
23
|
+
from dkatchr.clients.exploitdb import ExploitDBClient
|
|
24
|
+
from dkatchr.clients.github import GitHubClient
|
|
25
|
+
from dkatchr.clients.kev import KEVClient
|
|
26
|
+
from dkatchr.clients.metasploit import MetasploitClient
|
|
27
|
+
from dkatchr.clients.osv import OSVClient
|
|
28
|
+
from dkatchr.clients.registry_meta import RegistryMetaClient
|
|
29
|
+
from dkatchr.clients.top_packages import TopPackagesClient
|
|
30
|
+
from dkatchr.config import CACHE_SCHEMA
|
|
31
|
+
from dkatchr.core.attribution import enrich_attribution
|
|
32
|
+
from dkatchr.core.confusion import detect_confusion, load_internal_patterns
|
|
33
|
+
from dkatchr.core.typosquat import detect_typosquats
|
|
34
|
+
from dkatchr.core.osv_enrichment import osv_enrich, osv_rows_for_repo
|
|
35
|
+
from dkatchr.core.reachability import enrich_reachability
|
|
36
|
+
from dkatchr.core.rules import compile_rules, load_package_config
|
|
37
|
+
from dkatchr.core.scanner import Scanner
|
|
38
|
+
from dkatchr.logger import HAS_TQDM, log, reachability_badge, set_log_file, set_quiet_repos, tqdm
|
|
39
|
+
from dkatchr.output.csv_writer import open_csv_writer
|
|
40
|
+
from dkatchr.output.schema import empty_row
|
|
41
|
+
from dkatchr.storage.attribution_cache import AttributionCache
|
|
42
|
+
from dkatchr.storage.inventory_cache import InventoryCache
|
|
43
|
+
from dkatchr.storage.reachability_cache import ReachabilityCache
|
|
44
|
+
|
|
45
|
+
from dkatchr.cli.dry_run import print_dry_run_report
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _repo_uses_cache(cache: InventoryCache, full_name: str, full_rescan: bool) -> bool:
|
|
49
|
+
"""Cheap check — only consults the index, doesn't hit GitHub."""
|
|
50
|
+
if full_rescan:
|
|
51
|
+
return False
|
|
52
|
+
return bool(cache.get_index_sha(full_name))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def run_cli(args) -> None:
|
|
56
|
+
"""Top-level orchestration for the CLI. `args` is the parsed argparse Namespace."""
|
|
57
|
+
# ---- optional file logging -----------------------------------------
|
|
58
|
+
if getattr(args, "log_file", None):
|
|
59
|
+
set_log_file(args.log_file)
|
|
60
|
+
log(f"[+] Logging to file: {args.log_file}")
|
|
61
|
+
|
|
62
|
+
# ---- shared infra ---------------------------------------------------
|
|
63
|
+
github = GitHubClient(rps=args.rps, burst=args.burst)
|
|
64
|
+
cache = InventoryCache(cache_dir=args.cache_dir)
|
|
65
|
+
rules = compile_rules(load_package_config(args.config))
|
|
66
|
+
scanner = Scanner(github=github, cache=cache, package_config=rules)
|
|
67
|
+
|
|
68
|
+
# ---- resolve targets ------------------------------------------------
|
|
69
|
+
repo_specs, orgs = _resolve_targets(github, args)
|
|
70
|
+
if not repo_specs and not orgs:
|
|
71
|
+
log("[!] Nothing to scan. Pass --orgs or --repos.")
|
|
72
|
+
sys.exit(1)
|
|
73
|
+
|
|
74
|
+
_print_run_header(repo_specs, orgs, rules, args)
|
|
75
|
+
|
|
76
|
+
# ---- build the (label, repos) batches -------------------------------
|
|
77
|
+
batches = _build_batches(github, repo_specs, orgs)
|
|
78
|
+
|
|
79
|
+
# ---- dry-run --------------------------------------------------------
|
|
80
|
+
if args.dry_run:
|
|
81
|
+
print_dry_run_report(batches, cache, args)
|
|
82
|
+
sys.exit(0)
|
|
83
|
+
|
|
84
|
+
# ---- progress bar setup ---------------------------------------------
|
|
85
|
+
use_bar = HAS_TQDM and not args.no_progress
|
|
86
|
+
set_quiet_repos(use_bar)
|
|
87
|
+
if not HAS_TQDM and not args.no_progress:
|
|
88
|
+
log("[!] tqdm not installed — progress bar disabled. `pip install tqdm` for a real bar.")
|
|
89
|
+
|
|
90
|
+
# ---- run ------------------------------------------------------------
|
|
91
|
+
totals = {"repos": 0, "rule_hits": 0, "rule_vuln": 0, "osv_hits": 0, "kev_hits": 0,
|
|
92
|
+
"fix_available_hits": 0, "exploitdb_hits": 0, "metasploit_hits": 0,
|
|
93
|
+
"exploit_hits": 0, "confusion_hits": 0, "typosquat_hits": 0}
|
|
94
|
+
inventory_lock = threading.Lock()
|
|
95
|
+
per_repo_inventory: dict[str, list[dict]] = {}
|
|
96
|
+
|
|
97
|
+
# When --reachability OR --attribution is active, we buffer all vuln rows
|
|
98
|
+
# per repo and write them to CSV only after the enrichment pass(es) mutate
|
|
99
|
+
# the row dicts in place (reachability adds reachability columns;
|
|
100
|
+
# attribution adds introduced_by_* columns). A single flush writes them.
|
|
101
|
+
buffer_rows = bool(args.reachability or args.attribution)
|
|
102
|
+
per_repo_rule_rows: dict[str, list[dict]] = {}
|
|
103
|
+
per_repo_osv_rows: dict[str, list[dict]] = {}
|
|
104
|
+
per_repo_sha: dict[str, str] = {}
|
|
105
|
+
|
|
106
|
+
with open_csv_writer(args.output) as writer:
|
|
107
|
+
for label, repos in batches:
|
|
108
|
+
totals["repos"] += len(repos)
|
|
109
|
+
_scan_batch(
|
|
110
|
+
label=label, repos=repos, scanner=scanner, cache=cache, args=args,
|
|
111
|
+
writer=writer,
|
|
112
|
+
inventory_lock=inventory_lock, per_repo_inventory=per_repo_inventory,
|
|
113
|
+
per_repo_rule_rows=per_repo_rule_rows if buffer_rows else None,
|
|
114
|
+
per_repo_sha=per_repo_sha if buffer_rows else None,
|
|
115
|
+
totals=totals, use_bar=use_bar,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
# ---- OSV pass ---------------------------------------------------
|
|
119
|
+
tuple_to_vulns: dict = {}
|
|
120
|
+
kev_cve_set: set[str] | None = None
|
|
121
|
+
|
|
122
|
+
if args.osv and per_repo_inventory:
|
|
123
|
+
log(f"\n===== OSV ENRICHMENT =====")
|
|
124
|
+
osv = OSVClient(
|
|
125
|
+
cache_dir=f"{args.cache_dir}/osv",
|
|
126
|
+
ttl_seconds=args.osv_ttl,
|
|
127
|
+
rps=args.osv_rps,
|
|
128
|
+
burst=args.osv_burst,
|
|
129
|
+
)
|
|
130
|
+
try:
|
|
131
|
+
tuple_to_vulns = osv_enrich(per_repo_inventory, osv, detail_workers=args.workers)
|
|
132
|
+
except Exception as e:
|
|
133
|
+
log(f"[!] OSV enrichment failed: {e}")
|
|
134
|
+
tuple_to_vulns = {}
|
|
135
|
+
|
|
136
|
+
if not args.no_kev and tuple_to_vulns:
|
|
137
|
+
kev = KEVClient(cache_dir=f"{args.cache_dir}/kev", ttl_seconds=args.kev_ttl)
|
|
138
|
+
kev_cve_set = kev.load()
|
|
139
|
+
|
|
140
|
+
# EPSS exploit-probability enrichment. Fetched once (EPSSClient owns
|
|
141
|
+
# its date-keyed file cache + fail-safe — returns {} on total
|
|
142
|
+
# failure, leaving epss_score blank). Only OSV/CVE rows can match.
|
|
143
|
+
epss_scores = EPSSClient(cache_dir=args.cache_dir).get_scores()
|
|
144
|
+
|
|
145
|
+
# Public-exploit catalogs ("Has Exploit"). Same place + posture as
|
|
146
|
+
# KEV: built after enrichment, each owns a TTL'd file cache + fail-
|
|
147
|
+
# safe (returns an empty set on total failure), and the lookup is a
|
|
148
|
+
# Python CVE-token match, not a SQL/DB step. Disabled by
|
|
149
|
+
# --no-exploit-intel.
|
|
150
|
+
exploitdb_set: set[str] = set()
|
|
151
|
+
metasploit_set: set[str] = set()
|
|
152
|
+
if not args.no_exploit_intel and tuple_to_vulns:
|
|
153
|
+
try:
|
|
154
|
+
exploitdb_set = ExploitDBClient(
|
|
155
|
+
cache_dir=f"{args.cache_dir}/exploitdb",
|
|
156
|
+
ttl_seconds=args.exploit_ttl).load()
|
|
157
|
+
except Exception as e:
|
|
158
|
+
log(f"[!] Exploit-DB load failed: {e}")
|
|
159
|
+
try:
|
|
160
|
+
metasploit_set = MetasploitClient(
|
|
161
|
+
cache_dir=f"{args.cache_dir}/metasploit",
|
|
162
|
+
ttl_seconds=args.exploit_ttl).load()
|
|
163
|
+
except Exception as e:
|
|
164
|
+
log(f"[!] Metasploit load failed: {e}")
|
|
165
|
+
|
|
166
|
+
for full_name, inv in per_repo_inventory.items():
|
|
167
|
+
osv_rows = osv_rows_for_repo(full_name, inv, tuple_to_vulns,
|
|
168
|
+
kev_cve_set=kev_cve_set)
|
|
169
|
+
for row in osv_rows:
|
|
170
|
+
row["epss_score"] = _epss_pct(epss_scores, row.get("vuln_id"), row.get("aliases"))
|
|
171
|
+
has_exploit = _finalize_finding_row(row, exploitdb_set, metasploit_set)
|
|
172
|
+
if row["has_fix"] == "true": totals["fix_available_hits"] += 1
|
|
173
|
+
if row["has_exploitdb"] == "true": totals["exploitdb_hits"] += 1
|
|
174
|
+
if row["has_metasploit"] == "true": totals["metasploit_hits"] += 1
|
|
175
|
+
if has_exploit: totals["exploit_hits"] += 1
|
|
176
|
+
totals["osv_hits"] += len(osv_rows)
|
|
177
|
+
totals["kev_hits"] += sum(1 for r in osv_rows if r.get("is_kev") == "true")
|
|
178
|
+
if buffer_rows:
|
|
179
|
+
per_repo_osv_rows[full_name] = osv_rows
|
|
180
|
+
else:
|
|
181
|
+
writer.write_rows(osv_rows)
|
|
182
|
+
|
|
183
|
+
if not buffer_rows:
|
|
184
|
+
log(f"[✓] OSV: wrote {totals['osv_hits']} CVE row(s)")
|
|
185
|
+
if kev_cve_set is not None:
|
|
186
|
+
log(f"[✓] KEV: {totals['kev_hits']} finding(s) are known exploited vulnerabilities")
|
|
187
|
+
_print_exploit_intel_lines(args, totals)
|
|
188
|
+
|
|
189
|
+
# ---- dependency-confusion pass ----------------------------------
|
|
190
|
+
# Runs after the OSV pass (mirrors the web runner's ordering) but is
|
|
191
|
+
# NOT gated on --osv: candidates come from inventory origin + patterns,
|
|
192
|
+
# not OSV data. Confusion rows are written directly — the attribution
|
|
193
|
+
# and reachability passes deliberately ignore them, so they never need
|
|
194
|
+
# buffering. A pass failure never aborts the run.
|
|
195
|
+
if not args.no_confusion and per_repo_inventory:
|
|
196
|
+
_confusion_pass(args, per_repo_inventory, writer, totals)
|
|
197
|
+
|
|
198
|
+
# ---- typosquatting pass -----------------------------------------
|
|
199
|
+
# Opt-in (--typosquat, default OFF). Like confusion it writes rows
|
|
200
|
+
# directly (attribution/reachability ignore them) and never aborts the
|
|
201
|
+
# run. Runs after the confusion pass.
|
|
202
|
+
if args.typosquat and per_repo_inventory:
|
|
203
|
+
_typosquat_pass(args, per_repo_inventory, writer, totals)
|
|
204
|
+
|
|
205
|
+
# ---- attribution pass -------------------------------------------
|
|
206
|
+
# Runs AFTER the OSV pass so OSV findings (the primary vuln source) get
|
|
207
|
+
# attributed too — they don't exist as rows until the OSV pass builds
|
|
208
|
+
# them. Mirrors osv_enrich: dedups by manifest, one git-blame call per
|
|
209
|
+
# unique manifest file (cached by branch SHA). Merges introduced_by_*
|
|
210
|
+
# into the buffered finding rows in place; the reachability pass or the
|
|
211
|
+
# final flush below writes them.
|
|
212
|
+
if args.attribution:
|
|
213
|
+
_attribution_pass(github, args, per_repo_rule_rows, per_repo_osv_rows,
|
|
214
|
+
per_repo_sha, totals)
|
|
215
|
+
|
|
216
|
+
# ---- reachability pass ------------------------------------------
|
|
217
|
+
if args.reachability:
|
|
218
|
+
log(f"\n===== REACHABILITY ANALYSIS =====")
|
|
219
|
+
all_repos = set(per_repo_rule_rows) | set(per_repo_osv_rows)
|
|
220
|
+
reach_cache = ReachabilityCache(cache_dir=args.cache_dir)
|
|
221
|
+
reach_totals: dict[str, int] = {
|
|
222
|
+
"REACHABLE": 0, "UNUSED": 0, "UNKNOWN": 0, "packages": 0,
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
for full_name in all_repos:
|
|
226
|
+
rule_rows = per_repo_rule_rows.get(full_name, [])
|
|
227
|
+
osv_rows = per_repo_osv_rows.get(full_name, [])
|
|
228
|
+
all_rows = rule_rows + osv_rows
|
|
229
|
+
if not all_rows:
|
|
230
|
+
continue
|
|
231
|
+
|
|
232
|
+
sha = per_repo_sha.get(full_name)
|
|
233
|
+
if not sha:
|
|
234
|
+
log(f"[{full_name}] reachability skipped — no SHA available")
|
|
235
|
+
writer.write_rows(rule_rows)
|
|
236
|
+
writer.write_rows(osv_rows)
|
|
237
|
+
continue
|
|
238
|
+
|
|
239
|
+
cli_dl_state = {"last_emit_at": 0.0}
|
|
240
|
+
|
|
241
|
+
def _cli_reach_progress(payload: dict, _repo=full_name,
|
|
242
|
+
_state=cli_dl_state) -> None:
|
|
243
|
+
phase = payload.get("phase", "")
|
|
244
|
+
if phase == "cache_hit":
|
|
245
|
+
log(f"[{_repo}] reachability: cache hit @ {payload.get('sha','')}")
|
|
246
|
+
elif phase == "tarball_downloading":
|
|
247
|
+
# Print at most one line per second to keep stdout sane.
|
|
248
|
+
now = time.monotonic()
|
|
249
|
+
if now - _state["last_emit_at"] < 1.0:
|
|
250
|
+
return
|
|
251
|
+
_state["last_emit_at"] = now
|
|
252
|
+
done = payload.get("downloaded_bytes", 0)
|
|
253
|
+
total = payload.get("total_bytes")
|
|
254
|
+
if total:
|
|
255
|
+
pct = done * 100 // max(1, total)
|
|
256
|
+
log(f"[{_repo}] downloading tarball: "
|
|
257
|
+
f"{done / 1024 / 1024:.1f}/{total / 1024 / 1024:.1f}MB ({pct}%)")
|
|
258
|
+
else:
|
|
259
|
+
log(f"[{_repo}] downloading tarball: {done / 1024 / 1024:.1f}MB")
|
|
260
|
+
elif phase == "tarball_downloaded":
|
|
261
|
+
log(f"[{_repo}] tarball downloaded: "
|
|
262
|
+
f"{payload.get('size_bytes', 0) / 1024 / 1024:.1f}MB")
|
|
263
|
+
elif phase == "tarball_failed":
|
|
264
|
+
log(f"[{_repo}] tarball download FAILED")
|
|
265
|
+
elif phase == "building_automaton":
|
|
266
|
+
log(f"[{_repo}] building automaton: "
|
|
267
|
+
f"{payload.get('patterns', 0)} pattern(s) across "
|
|
268
|
+
f"{payload.get('unique_packages', 0)} package(s)")
|
|
269
|
+
elif phase == "scanning_progress":
|
|
270
|
+
log(f"[{_repo}] scanning: "
|
|
271
|
+
f"{payload.get('files_scanned', 0)} source files scanned, "
|
|
272
|
+
f"{payload.get('patterns_matched', 0)}/"
|
|
273
|
+
f"{payload.get('patterns_total', 0)} patterns matched")
|
|
274
|
+
elif phase == "scanning_complete":
|
|
275
|
+
log(f"[{_repo}] scan done: "
|
|
276
|
+
f"{payload.get('files_scanned', 0)} files scanned, "
|
|
277
|
+
f"{payload.get('files_skipped', 0)} skipped")
|
|
278
|
+
elif phase == "resolving_start":
|
|
279
|
+
log(f"[{_repo}] resolving import names: "
|
|
280
|
+
f"{payload.get('total', 0)} unique package(s)")
|
|
281
|
+
elif phase == "resolved":
|
|
282
|
+
log(f"[{_repo}] resolved {payload.get('ecosystem','')}"
|
|
283
|
+
f"/{payload.get('package','')} → "
|
|
284
|
+
f"{payload.get('names')} (via {payload.get('source','')})")
|
|
285
|
+
elif phase == "resolution_failed":
|
|
286
|
+
log(f"[{_repo}] resolution FAILED "
|
|
287
|
+
f"{payload.get('ecosystem','')}/{payload.get('package','')}: "
|
|
288
|
+
f"{payload.get('reason','')}")
|
|
289
|
+
elif phase == "resolving_done":
|
|
290
|
+
log(f"[{_repo}] import name resolution: "
|
|
291
|
+
f"{payload.get('resolved',0)} resolved, "
|
|
292
|
+
f"{payload.get('cached',0)} cached, "
|
|
293
|
+
f"{payload.get('failed',0)} failed")
|
|
294
|
+
|
|
295
|
+
enrich_reachability(
|
|
296
|
+
all_rows, full_name, sha, github, reach_cache,
|
|
297
|
+
on_progress=_cli_reach_progress,
|
|
298
|
+
cache_dir=args.cache_dir,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
seen_pkgs: set[tuple[str, str]] = set()
|
|
302
|
+
for row in all_rows:
|
|
303
|
+
pkg_key = (row.get("package", ""), row.get("ecosystem", ""))
|
|
304
|
+
if pkg_key not in seen_pkgs:
|
|
305
|
+
seen_pkgs.add(pkg_key)
|
|
306
|
+
r = row.get("reachability", "UNKNOWN") or "UNKNOWN"
|
|
307
|
+
reach_totals[r] = reach_totals.get(r, 0) + 1
|
|
308
|
+
reach_totals["packages"] += 1
|
|
309
|
+
badge = reachability_badge(row.get("reachability", ""))
|
|
310
|
+
vuln_ref = row.get("vuln_id") or row.get("reason") or ""
|
|
311
|
+
log(f"[{full_name}] {row.get('package','')}=={row.get('version','')} {vuln_ref} {badge}")
|
|
312
|
+
|
|
313
|
+
writer.write_rows(rule_rows)
|
|
314
|
+
writer.write_rows(osv_rows)
|
|
315
|
+
|
|
316
|
+
if args.osv:
|
|
317
|
+
log(f"[✓] OSV: wrote {totals['osv_hits']} CVE row(s)")
|
|
318
|
+
if kev_cve_set is not None:
|
|
319
|
+
log(f"[✓] KEV: {totals['kev_hits']} finding(s) are known exploited vulnerabilities")
|
|
320
|
+
_print_exploit_intel_lines(args, totals)
|
|
321
|
+
|
|
322
|
+
_print_reachability_summary(reach_totals)
|
|
323
|
+
|
|
324
|
+
# ---- flush buffered rows ----------------------------------------
|
|
325
|
+
# When buffering was on for attribution only (no reachability pass that
|
|
326
|
+
# writes the rows itself), emit the enriched rows now — still inside the
|
|
327
|
+
# CSV writer context so the file is closed cleanly afterwards.
|
|
328
|
+
if buffer_rows and not args.reachability:
|
|
329
|
+
for full_name in set(per_repo_rule_rows) | set(per_repo_osv_rows):
|
|
330
|
+
writer.write_rows(per_repo_rule_rows.get(full_name, []))
|
|
331
|
+
writer.write_rows(per_repo_osv_rows.get(full_name, []))
|
|
332
|
+
|
|
333
|
+
cache.save_index()
|
|
334
|
+
_print_run_footer(args, totals)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# ---------------------------------------------------------------------------
|
|
338
|
+
# Helpers — small, single-purpose, easy to read.
|
|
339
|
+
# ---------------------------------------------------------------------------
|
|
340
|
+
|
|
341
|
+
_CVE_RE = re.compile(r"CVE-\d{4}-\d+", re.IGNORECASE)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _cve_tokens(vuln_id: str | None, aliases: str | None) -> set[str]:
|
|
345
|
+
"""Uppercased CVE ids from a finding's vuln_id + aliases.
|
|
346
|
+
|
|
347
|
+
Shared by the CVE-keyed CLI enrichments (EPSS, exploit-intel). OSV's primary
|
|
348
|
+
`vuln_id` is almost always GHSA-/PYSEC-/GO- with the CVE in the comma-
|
|
349
|
+
separated `aliases`, so both fields are scanned. Mirrors the web repository
|
|
350
|
+
helper of the same name.
|
|
351
|
+
"""
|
|
352
|
+
return {c.upper() for c in _CVE_RE.findall(vuln_id or "")} \
|
|
353
|
+
| {c.upper() for c in _CVE_RE.findall(aliases or "")}
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _epss_pct(scores: dict[str, float], vuln_id: str | None, aliases: str | None = "") -> str:
|
|
357
|
+
"""EPSS probability for a finding, as a percentage string (e.g. "12.34%").
|
|
358
|
+
|
|
359
|
+
EPSS is keyed on CVE, so we extract every CVE token from vuln_id + aliases
|
|
360
|
+
and take the highest matching score. Empty string when no CVE token matches
|
|
361
|
+
the dataset (Custom Rule rows, no coverage, etc).
|
|
362
|
+
"""
|
|
363
|
+
vals = [scores[c] for c in _cve_tokens(vuln_id, aliases) if c in scores]
|
|
364
|
+
if not vals:
|
|
365
|
+
return ""
|
|
366
|
+
return f"{max(vals) * 100:.2f}%"
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _finalize_finding_row(row: dict, exploitdb_set: set[str],
|
|
370
|
+
metasploit_set: set[str]) -> bool:
|
|
371
|
+
"""Set has_fix / has_exploitdb / has_metasploit on a finding row, in place.
|
|
372
|
+
|
|
373
|
+
Returns True if the row counts as 'has exploit' (is_kev OR ExploitDB OR
|
|
374
|
+
Metasploit) — the composite verdict, reported in totals but never written as
|
|
375
|
+
a CSV column. `has_fix` is derived from fix_versions. Custom Rule rows pass
|
|
376
|
+
empty exploit sets, so they get has_exploitdb/has_metasploit = "false".
|
|
377
|
+
"""
|
|
378
|
+
row["has_fix"] = "true" if (row.get("fix_versions") or "").strip() else "false"
|
|
379
|
+
cves = _cve_tokens(row.get("vuln_id"), row.get("aliases"))
|
|
380
|
+
edb = bool(cves) and bool(exploitdb_set) and not cves.isdisjoint(exploitdb_set)
|
|
381
|
+
msf = bool(cves) and bool(metasploit_set) and not cves.isdisjoint(metasploit_set)
|
|
382
|
+
row["has_exploitdb"] = "true" if edb else "false"
|
|
383
|
+
row["has_metasploit"] = "true" if msf else "false"
|
|
384
|
+
return (row.get("is_kev") == "true") or edb or msf
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _confusion_csv_row(f: dict) -> dict:
|
|
388
|
+
"""Turn a core confusion finding dict into a unified CSV row."""
|
|
389
|
+
row = empty_row(f["repo"], {
|
|
390
|
+
"file": f.get("file", ""),
|
|
391
|
+
"ecosystem": f.get("ecosystem", ""),
|
|
392
|
+
"package": f.get("package", ""),
|
|
393
|
+
"version": f.get("version", ""),
|
|
394
|
+
"version_source": f.get("version_source", ""),
|
|
395
|
+
"origin": f.get("origin", "registry"),
|
|
396
|
+
})
|
|
397
|
+
row["source"] = "CONFUSION"
|
|
398
|
+
row["status"] = "vulnerable"
|
|
399
|
+
row["vuln_id"] = f.get("vuln_id", "")
|
|
400
|
+
row["severity"] = f.get("severity", "")
|
|
401
|
+
row["summary"] = f.get("summary", "")
|
|
402
|
+
row["suspicion_signals"] = json.dumps(f.get("suspicion_signals") or {}, sort_keys=True)
|
|
403
|
+
# No CVE, no fix list — but keep the boolean columns uniform ("false").
|
|
404
|
+
_finalize_finding_row(row, set(), set())
|
|
405
|
+
return row
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _confusion_pass(args, per_repo_inventory: dict, writer, totals: dict) -> None:
|
|
409
|
+
"""CLI dependency-confusion pass — patterns + registry lookups + CSV rows.
|
|
410
|
+
|
|
411
|
+
Wrapped end-to-end: a pass failure logs and the run continues (same
|
|
412
|
+
isolation contract as the OSV/KEV/exploit-intel passes).
|
|
413
|
+
"""
|
|
414
|
+
log("\n===== DEPENDENCY CONFUSION DETECTION =====")
|
|
415
|
+
try:
|
|
416
|
+
patterns = load_internal_patterns(args.internal_patterns)
|
|
417
|
+
client = RegistryMetaClient(cache_dir=f"{args.cache_dir}/registry_meta")
|
|
418
|
+
|
|
419
|
+
state = {"last_emit_at": 0.0}
|
|
420
|
+
|
|
421
|
+
def _progress(payload: dict, _state=state) -> None:
|
|
422
|
+
# Throttle the per-candidate firehose to ~1 line/sec.
|
|
423
|
+
now = time.monotonic()
|
|
424
|
+
if now - _state["last_emit_at"] < 1.0:
|
|
425
|
+
return
|
|
426
|
+
_state["last_emit_at"] = now
|
|
427
|
+
log(f"[confusion] checking candidate {payload.get('current', 0)}"
|
|
428
|
+
f"/{payload.get('total', 0)} — "
|
|
429
|
+
f"{payload.get('ecosystem', '')}/{payload.get('package', '')}")
|
|
430
|
+
|
|
431
|
+
result = detect_confusion(
|
|
432
|
+
per_repo_inventory, patterns, client.get_package_meta,
|
|
433
|
+
on_progress=_progress,
|
|
434
|
+
supported_ecosystems=RegistryMetaClient.SUPPORTED_ECOSYSTEMS,
|
|
435
|
+
)
|
|
436
|
+
rows = [_confusion_csv_row(f) for f in result["findings"]]
|
|
437
|
+
writer.write_rows(rows)
|
|
438
|
+
totals["confusion_hits"] = len(rows)
|
|
439
|
+
s = result["stats"]
|
|
440
|
+
log(f"[✓] Confusion: {len(rows)} finding(s) — {s['checked']} candidate(s) "
|
|
441
|
+
f"checked, {s['public']} exist publicly"
|
|
442
|
+
+ (f", {s['skipped_unsupported']} skipped (unsupported registry)"
|
|
443
|
+
if s["skipped_unsupported"] else "")
|
|
444
|
+
+ (f", {s['skipped_unresolved']} lookup failure(s)"
|
|
445
|
+
if s["skipped_unresolved"] else ""))
|
|
446
|
+
except Exception as e:
|
|
447
|
+
log(f"[!] Dependency-confusion pass failed: {e} — continuing")
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _typosquat_csv_row(f: dict) -> dict:
|
|
451
|
+
"""Turn a core typosquat finding dict into a unified CSV row (mirror of
|
|
452
|
+
_confusion_csv_row — reuses the same origin + suspicion_signals columns)."""
|
|
453
|
+
row = empty_row(f["repo"], {
|
|
454
|
+
"file": f.get("file", ""),
|
|
455
|
+
"ecosystem": f.get("ecosystem", ""),
|
|
456
|
+
"package": f.get("package", ""),
|
|
457
|
+
"version": f.get("version", ""),
|
|
458
|
+
"version_source": f.get("version_source", ""),
|
|
459
|
+
"origin": f.get("origin", "registry"),
|
|
460
|
+
})
|
|
461
|
+
row["source"] = "TYPOSQUAT"
|
|
462
|
+
row["status"] = "vulnerable"
|
|
463
|
+
row["vuln_id"] = f.get("vuln_id", "")
|
|
464
|
+
row["severity"] = f.get("severity", "")
|
|
465
|
+
row["summary"] = f.get("summary", "")
|
|
466
|
+
row["suspicion_signals"] = json.dumps(f.get("suspicion_signals") or {}, sort_keys=True)
|
|
467
|
+
_finalize_finding_row(row, set(), set())
|
|
468
|
+
return row
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _typosquat_pass(args, per_repo_inventory: dict, writer, totals: dict) -> None:
|
|
472
|
+
"""CLI typosquatting pass — top-package feeds + edit distance + metadata gate.
|
|
473
|
+
|
|
474
|
+
Wrapped end-to-end: a pass failure logs and the run continues (same isolation
|
|
475
|
+
contract as the OSV/confusion passes). The metadata gate reuses
|
|
476
|
+
RegistryMetaClient (no second registry client).
|
|
477
|
+
"""
|
|
478
|
+
log("\n===== TYPOSQUATTING DETECTION =====")
|
|
479
|
+
try:
|
|
480
|
+
top_client = TopPackagesClient(cache_dir=f"{args.cache_dir}/top_packages")
|
|
481
|
+
meta_client = RegistryMetaClient(cache_dir=f"{args.cache_dir}/registry_meta")
|
|
482
|
+
|
|
483
|
+
present_ecos = {
|
|
484
|
+
dep.get("ecosystem")
|
|
485
|
+
for inv in per_repo_inventory.values() for dep in (inv or [])
|
|
486
|
+
if dep.get("ecosystem")
|
|
487
|
+
}
|
|
488
|
+
feed_state = {"last_emit_at": 0.0}
|
|
489
|
+
|
|
490
|
+
def _feed_progress(payload: dict, _state=feed_state) -> None:
|
|
491
|
+
now = time.monotonic()
|
|
492
|
+
if now - _state["last_emit_at"] < 1.0:
|
|
493
|
+
return
|
|
494
|
+
_state["last_emit_at"] = now
|
|
495
|
+
log(f"[typosquat] fetching {payload.get('ecosystem', '')} top packages "
|
|
496
|
+
f"({payload.get('fetched', 0)} so far)")
|
|
497
|
+
|
|
498
|
+
top_sets = {
|
|
499
|
+
eco: top_client.get_top(eco, on_progress=_feed_progress)
|
|
500
|
+
for eco in present_ecos
|
|
501
|
+
if eco in top_client.SUPPORTED_ECOSYSTEMS
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
lk_state = {"last_emit_at": 0.0}
|
|
505
|
+
|
|
506
|
+
def _progress(payload: dict, _state=lk_state) -> None:
|
|
507
|
+
if payload.get("phase") != "typosquat_lookup":
|
|
508
|
+
return
|
|
509
|
+
now = time.monotonic()
|
|
510
|
+
if now - _state["last_emit_at"] < 1.0:
|
|
511
|
+
return
|
|
512
|
+
_state["last_emit_at"] = now
|
|
513
|
+
log(f"[typosquat] checking candidate {payload.get('current', 0)}"
|
|
514
|
+
f"/{payload.get('total', 0)} — "
|
|
515
|
+
f"{payload.get('ecosystem', '')}/{payload.get('package', '')}")
|
|
516
|
+
|
|
517
|
+
result = detect_typosquats(
|
|
518
|
+
per_repo_inventory, top_sets, meta_client.get_package_meta,
|
|
519
|
+
on_progress=_progress,
|
|
520
|
+
supported_ecosystems=TopPackagesClient.SUPPORTED_ECOSYSTEMS,
|
|
521
|
+
)
|
|
522
|
+
rows = [_typosquat_csv_row(f) for f in result["findings"]]
|
|
523
|
+
writer.write_rows(rows)
|
|
524
|
+
totals["typosquat_hits"] = len(rows)
|
|
525
|
+
s = result["stats"]
|
|
526
|
+
log(f"[✓] Typosquat: {len(rows)} finding(s) — {s['candidates']} candidate(s) "
|
|
527
|
+
f"evaluated, {s['distance_hits']} close-name match(es), {s['public']} "
|
|
528
|
+
f"confirmed on the public registry"
|
|
529
|
+
+ (f", {s['skipped_unsupported']} skipped (unsupported ecosystem)"
|
|
530
|
+
if s["skipped_unsupported"] else "")
|
|
531
|
+
+ (f", {s['skipped_unresolved']} lookup failure(s)"
|
|
532
|
+
if s["skipped_unresolved"] else ""))
|
|
533
|
+
except Exception as e:
|
|
534
|
+
log(f"[!] Typosquatting pass failed: {e} — continuing")
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _print_exploit_intel_lines(args, totals: dict) -> None:
|
|
538
|
+
"""Mid-run '[✓]' summary lines for the public-exploit cross-reference,
|
|
539
|
+
mirroring the KEV summary line's format. No-op when --no-exploit-intel."""
|
|
540
|
+
if args.no_exploit_intel:
|
|
541
|
+
return
|
|
542
|
+
log(f"[✓] Exploit-DB: {totals['exploitdb_hits']} finding(s) have known public exploit code")
|
|
543
|
+
log(f"[✓] Metasploit: {totals['metasploit_hits']} finding(s) have a Metasploit module")
|
|
544
|
+
log(f"[✓] Has Exploit: {totals['exploit_hits']} finding(s) (KEV/ExploitDB/Metasploit)")
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _merge_attribution(rows: list[dict], attr_map: dict) -> int:
|
|
548
|
+
"""Write introduced_by_* CSV columns onto finding rows from attr_map.
|
|
549
|
+
|
|
550
|
+
Keyed on (repo, file, package, version) — the same manifest line a finding
|
|
551
|
+
points at. Returns the count of rows that received a real commit. Only
|
|
552
|
+
RESULT_FIELDS keys are written, so the rows stay DictWriter-safe.
|
|
553
|
+
"""
|
|
554
|
+
n = 0
|
|
555
|
+
for r in rows:
|
|
556
|
+
a = attr_map.get((r.get("repo"), r.get("file"), r.get("package"), r.get("version")))
|
|
557
|
+
if not a:
|
|
558
|
+
continue
|
|
559
|
+
sha = a.get("commit_sha")
|
|
560
|
+
r["introduced_by_handle"] = a.get("author_handle") or ""
|
|
561
|
+
r["introduced_by_name"] = a.get("author_name") or ""
|
|
562
|
+
r["introduced_by_commit"] = sha or ""
|
|
563
|
+
r["introduced_by_date"] = a.get("commit_date") or ""
|
|
564
|
+
r["introduced_by_message"] = a.get("commit_message") or ""
|
|
565
|
+
r["introduced_by_is_bot"] = ("true" if a.get("is_bot") else "false") if sha else ""
|
|
566
|
+
if sha:
|
|
567
|
+
n += 1
|
|
568
|
+
return n
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _attribution_pass(github: GitHubClient, args, per_repo_rule_rows: dict,
|
|
572
|
+
per_repo_osv_rows: dict, per_repo_sha: dict, totals: dict) -> None:
|
|
573
|
+
"""CLI attribution pass — the direct call into core enrich_attribution().
|
|
574
|
+
|
|
575
|
+
Builds deduped attribution inputs from the buffered finding rows (+ each
|
|
576
|
+
repo's branch SHA), runs enrich_attribution, and merges introduced_by_*
|
|
577
|
+
columns back into the buffered rows by (repo, file, package, version).
|
|
578
|
+
"""
|
|
579
|
+
log("\n===== ATTRIBUTION (Introduced By) =====")
|
|
580
|
+
attr_cache = AttributionCache(cache_dir=args.cache_dir)
|
|
581
|
+
|
|
582
|
+
all_finding_rows: list[dict] = []
|
|
583
|
+
for rows in per_repo_rule_rows.values():
|
|
584
|
+
all_finding_rows.extend(rows)
|
|
585
|
+
for rows in per_repo_osv_rows.values():
|
|
586
|
+
all_finding_rows.extend(rows)
|
|
587
|
+
|
|
588
|
+
if not all_finding_rows:
|
|
589
|
+
log("[attribution] no findings to attribute — skipping")
|
|
590
|
+
return
|
|
591
|
+
|
|
592
|
+
# One attribution input per unique (repo, file, package, version) finding.
|
|
593
|
+
seen_keys: set = set()
|
|
594
|
+
attr_input: list[dict] = []
|
|
595
|
+
for r in all_finding_rows:
|
|
596
|
+
key = (r.get("repo"), r.get("file"), r.get("package"), r.get("version"))
|
|
597
|
+
if key in seen_keys:
|
|
598
|
+
continue
|
|
599
|
+
seen_keys.add(key)
|
|
600
|
+
attr_input.append({
|
|
601
|
+
"repo_full_name": r.get("repo"),
|
|
602
|
+
"file": r.get("file"),
|
|
603
|
+
"sha": per_repo_sha.get(r.get("repo")),
|
|
604
|
+
"ecosystem": r.get("ecosystem"),
|
|
605
|
+
"package": r.get("package"),
|
|
606
|
+
"version": r.get("version"),
|
|
607
|
+
})
|
|
608
|
+
|
|
609
|
+
manifests = len({(i["repo_full_name"], i["file"], i["sha"]) for i in attr_input})
|
|
610
|
+
state = {"last_emit_at": 0.0}
|
|
611
|
+
|
|
612
|
+
def _attr_progress(payload: dict, _state=state) -> None:
|
|
613
|
+
# Throttle the per-manifest firehose to ~1 line/sec for readable stdout.
|
|
614
|
+
if payload.get("phase") != "attribution_progress":
|
|
615
|
+
return
|
|
616
|
+
now = time.monotonic()
|
|
617
|
+
if now - _state["last_emit_at"] < 1.0:
|
|
618
|
+
return
|
|
619
|
+
_state["last_emit_at"] = now
|
|
620
|
+
log(f"[attribution] {payload.get('current', 0)}/{payload.get('total', 0)} "
|
|
621
|
+
f"manifests — {payload.get('repo', '')}:{payload.get('path', '')}")
|
|
622
|
+
|
|
623
|
+
enriched = enrich_attribution(attr_input, github, attr_cache, on_progress=_attr_progress)
|
|
624
|
+
|
|
625
|
+
attr_map = {
|
|
626
|
+
(e.get("repo_full_name"), e.get("file"), e.get("package"), e.get("version")): e.get("attribution")
|
|
627
|
+
for e in enriched
|
|
628
|
+
}
|
|
629
|
+
attributed = _merge_attribution(all_finding_rows, attr_map)
|
|
630
|
+
totals["attributed"] = attributed
|
|
631
|
+
log(f"[✓] Attribution complete: {manifests} manifests queried, {attributed} findings attributed")
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def _resolve_targets(github: GitHubClient, args) -> tuple[list[str], list[str]]:
|
|
635
|
+
"""Targeting precedence: --repos > --orgs > /user/orgs auto-discover."""
|
|
636
|
+
if args.repos:
|
|
637
|
+
return args.repos, []
|
|
638
|
+
if args.orgs:
|
|
639
|
+
return [], args.orgs
|
|
640
|
+
try:
|
|
641
|
+
return [], github.list_user_orgs()
|
|
642
|
+
except Exception as e:
|
|
643
|
+
log(f"[!] Could not auto-discover orgs via /user/orgs: {e}")
|
|
644
|
+
log("[!] Pass --orgs <org> [...] or --repos owner/name [...] to target explicitly.")
|
|
645
|
+
sys.exit(1)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _print_run_header(repo_specs: list[str], orgs: list[str], rules: dict, args) -> None:
|
|
649
|
+
if repo_specs:
|
|
650
|
+
log(f"[+] Repos: {repo_specs}")
|
|
651
|
+
else:
|
|
652
|
+
log(f"[+] Orgs: {orgs}")
|
|
653
|
+
log(f"[+] Output: {args.output}")
|
|
654
|
+
log(f"[+] Cache: {args.cache_dir} (schema v{CACHE_SCHEMA}) Rules: {len(rules)}")
|
|
655
|
+
log(f"[+] Workers: {args.workers} Rate cap: {args.rps} req/s Burst: {args.burst}")
|
|
656
|
+
log(f"[+] OSV mode: {'ON' if args.osv else 'off'}"
|
|
657
|
+
+ (f" TTL: {args.osv_ttl}s Rate cap: {args.osv_rps} req/s" if args.osv else ""))
|
|
658
|
+
if args.osv:
|
|
659
|
+
log(f"[+] KEV mode: {'off (--no-kev)' if args.no_kev else 'ON'} TTL: {args.kev_ttl}s")
|
|
660
|
+
log(f"[+] Exploit: {'off (--no-exploit-intel)' if args.no_exploit_intel else 'ON'} TTL: {args.exploit_ttl}s")
|
|
661
|
+
log(f"[+] Confusion: {'off (--no-confusion)' if args.no_confusion else 'ON'}"
|
|
662
|
+
+ (f" Patterns: {args.internal_patterns}" if args.internal_patterns else ""))
|
|
663
|
+
log(f"[+] Typosquat: {'ON' if args.typosquat else 'off'}")
|
|
664
|
+
if not rules and not args.osv:
|
|
665
|
+
log("[!] No rules and no --osv. Inventory will be cached but no CSV rows will be written.")
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def _print_run_footer(args, totals: dict) -> None:
|
|
669
|
+
log(f"\n[✓] Done. repos={totals['repos']}")
|
|
670
|
+
log(f" Rule rows: {totals['rule_hits']} (vulnerable: {totals['rule_vuln']})")
|
|
671
|
+
if args.osv:
|
|
672
|
+
log(f" OSV rows: {totals['osv_hits']}")
|
|
673
|
+
if not args.no_kev:
|
|
674
|
+
log(f" KEV: {totals['kev_hits']} finding(s) are known exploited vulnerabilities")
|
|
675
|
+
if not args.no_exploit_intel:
|
|
676
|
+
log(f" Exploit-DB: {totals['exploitdb_hits']} finding(s) have known public exploit code")
|
|
677
|
+
log(f" Metasploit: {totals['metasploit_hits']} finding(s) have a Metasploit module")
|
|
678
|
+
log(f" Has Exploit: {totals['exploit_hits']} finding(s) (KEV/ExploitDB/Metasploit)")
|
|
679
|
+
log(f" Fix avail: {totals['fix_available_hits']} finding(s) have a fix version available")
|
|
680
|
+
if not args.no_confusion:
|
|
681
|
+
log(f" Confusion: {totals['confusion_hits']} dependency-confusion finding(s)")
|
|
682
|
+
if args.typosquat:
|
|
683
|
+
log(f" Typosquat: {totals['typosquat_hits']} typosquatting finding(s)")
|
|
684
|
+
log(f" Output: {args.output}")
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _print_reachability_summary(reach_totals: dict) -> None:
|
|
688
|
+
sep = "─" * 36
|
|
689
|
+
log(f"\n{sep}")
|
|
690
|
+
log("Reachability Summary")
|
|
691
|
+
log(sep)
|
|
692
|
+
log(f"REACHABLE {reach_totals.get('REACHABLE', 0):4d} (fix these)")
|
|
693
|
+
log(f"UNUSED {reach_totals.get('UNUSED', 0):4d} (deprioritise)")
|
|
694
|
+
log(f"UNKNOWN {reach_totals.get('UNKNOWN', 0):4d} (verify manually)")
|
|
695
|
+
log(sep)
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _build_batches(github: GitHubClient, repo_specs: list[str],
|
|
699
|
+
orgs: list[str]) -> list[tuple[str, list[dict]]]:
|
|
700
|
+
"""Group target repos by label for per-batch progress reporting."""
|
|
701
|
+
batches: list[tuple[str, list[dict]]] = []
|
|
702
|
+
if repo_specs:
|
|
703
|
+
repos: list[dict] = []
|
|
704
|
+
for spec in repo_specs:
|
|
705
|
+
spec = spec.strip().strip("/") # zsh/fish can prepend a slash on tab-complete
|
|
706
|
+
if "/" not in spec:
|
|
707
|
+
log(f"[!] Bad --repos value '{spec}' — expected owner/name. Skipping.")
|
|
708
|
+
continue
|
|
709
|
+
owner, name = spec.split("/", 1)
|
|
710
|
+
try:
|
|
711
|
+
repos.append(github.get_repo_meta(owner, name))
|
|
712
|
+
except Exception as e:
|
|
713
|
+
log(f"[!] Could not fetch repo {spec}: {e}")
|
|
714
|
+
if repos:
|
|
715
|
+
batches.append(("--repos targets", repos))
|
|
716
|
+
else:
|
|
717
|
+
for org in orgs:
|
|
718
|
+
log(f"\n===== ORG: {org} =====")
|
|
719
|
+
try:
|
|
720
|
+
batches.append((org, github.list_org_repos(org)))
|
|
721
|
+
except Exception as e:
|
|
722
|
+
log(f"[!] Failed to list repos for '{org}': {e}")
|
|
723
|
+
continue
|
|
724
|
+
return batches
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
def _scan_batch(
|
|
728
|
+
label: str, repos: list[dict], scanner: Scanner, cache: InventoryCache, args,
|
|
729
|
+
writer, inventory_lock: threading.Lock, per_repo_inventory: dict,
|
|
730
|
+
totals: dict, use_bar: bool,
|
|
731
|
+
per_repo_rule_rows: dict | None = None,
|
|
732
|
+
per_repo_sha: dict | None = None,
|
|
733
|
+
) -> None:
|
|
734
|
+
"""Submit a batch of repos to the thread pool and drain results."""
|
|
735
|
+
n = len(repos)
|
|
736
|
+
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
737
|
+
futures = {
|
|
738
|
+
pool.submit(
|
|
739
|
+
scanner.scan_repo,
|
|
740
|
+
repo,
|
|
741
|
+
f"[{idx}/{n}]",
|
|
742
|
+
args.full_rescan or not _repo_uses_cache(cache, repo["full_name"], args.full_rescan),
|
|
743
|
+
): repo["full_name"]
|
|
744
|
+
for idx, repo in enumerate(repos, 1)
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
completed_iter = as_completed(futures)
|
|
748
|
+
bar = None
|
|
749
|
+
if use_bar:
|
|
750
|
+
bar = tqdm(
|
|
751
|
+
completed_iter,
|
|
752
|
+
total=len(futures),
|
|
753
|
+
desc=f"Scanning [{label}]",
|
|
754
|
+
unit="repo",
|
|
755
|
+
dynamic_ncols=True,
|
|
756
|
+
)
|
|
757
|
+
completed_iter = bar
|
|
758
|
+
|
|
759
|
+
done_count = 0
|
|
760
|
+
batch_inv = 0
|
|
761
|
+
|
|
762
|
+
for fut in completed_iter:
|
|
763
|
+
done_count += 1
|
|
764
|
+
full_name = futures[fut]
|
|
765
|
+
try:
|
|
766
|
+
result = fut.result()
|
|
767
|
+
except Exception as e:
|
|
768
|
+
log(f"[!] {full_name}: worker crashed: {e}")
|
|
769
|
+
continue
|
|
770
|
+
|
|
771
|
+
if result["skipped"] or result.get("error"):
|
|
772
|
+
if bar:
|
|
773
|
+
bar.set_postfix(rule_hits=totals["rule_hits"], vuln=totals["rule_vuln"], inv=batch_inv)
|
|
774
|
+
continue
|
|
775
|
+
|
|
776
|
+
rows = result["rows"]
|
|
777
|
+
vuln = sum(1 for r in rows if r.get("status") == "vulnerable")
|
|
778
|
+
batch_inv += len(result["inventory"])
|
|
779
|
+
|
|
780
|
+
# Custom Rule rows carry no CVE, so they never match the exploit
|
|
781
|
+
# catalogs (empty sets) — but they still need the has_* columns set
|
|
782
|
+
# uniformly ("false") and has_fix derived from fix_versions.
|
|
783
|
+
for r in rows:
|
|
784
|
+
_finalize_finding_row(r, set(), set())
|
|
785
|
+
totals["fix_available_hits"] += sum(1 for r in rows if r.get("has_fix") == "true")
|
|
786
|
+
|
|
787
|
+
totals["rule_hits"] += len(rows)
|
|
788
|
+
totals["rule_vuln"] += vuln
|
|
789
|
+
|
|
790
|
+
if per_repo_rule_rows is not None:
|
|
791
|
+
with inventory_lock:
|
|
792
|
+
per_repo_rule_rows[full_name] = rows
|
|
793
|
+
else:
|
|
794
|
+
writer.write_rows(rows)
|
|
795
|
+
|
|
796
|
+
# Inventory feeds the OSV pass, the confusion pass AND the typosquat
|
|
797
|
+
# pass — collect it when any is active.
|
|
798
|
+
if args.osv or not args.no_confusion or args.typosquat:
|
|
799
|
+
with inventory_lock:
|
|
800
|
+
per_repo_inventory[full_name] = result["inventory"]
|
|
801
|
+
|
|
802
|
+
if result["sha"]:
|
|
803
|
+
cache.update_index(full_name, result["sha"])
|
|
804
|
+
if per_repo_sha is not None:
|
|
805
|
+
with inventory_lock:
|
|
806
|
+
per_repo_sha[full_name] = result["sha"]
|
|
807
|
+
|
|
808
|
+
if bar:
|
|
809
|
+
bar.set_postfix(rule_hits=totals["rule_hits"], vuln=totals["rule_vuln"], inv=batch_inv)
|
|
810
|
+
elif args.summary_every > 0 and done_count % args.summary_every == 0:
|
|
811
|
+
log(f"[~] [{label}] {done_count}/{n} repos done — inv={batch_inv}, rule_hits={totals['rule_hits']}, vuln={totals['rule_vuln']}")
|
|
812
|
+
|
|
813
|
+
if bar:
|
|
814
|
+
bar.close()
|