dkatchr 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. dkatchr/__init__.py +3 -0
  2. dkatchr/__main__.py +6 -0
  3. dkatchr/cli/__init__.py +7 -0
  4. dkatchr/cli/args.py +84 -0
  5. dkatchr/cli/dry_run.py +68 -0
  6. dkatchr/cli/main.py +14 -0
  7. dkatchr/cli/run.py +814 -0
  8. dkatchr/clients/__init__.py +3 -0
  9. dkatchr/clients/epss.py +153 -0
  10. dkatchr/clients/exploitdb.py +155 -0
  11. dkatchr/clients/github.py +482 -0
  12. dkatchr/clients/kev.py +143 -0
  13. dkatchr/clients/metasploit.py +147 -0
  14. dkatchr/clients/nvd.py +121 -0
  15. dkatchr/clients/osv.py +191 -0
  16. dkatchr/clients/ratelimit.py +33 -0
  17. dkatchr/clients/registry_meta.py +280 -0
  18. dkatchr/clients/resolvers/__init__.py +335 -0
  19. dkatchr/clients/resolvers/_base.py +153 -0
  20. dkatchr/clients/resolvers/crates.py +51 -0
  21. dkatchr/clients/resolvers/maven.py +144 -0
  22. dkatchr/clients/resolvers/nuget.py +216 -0
  23. dkatchr/clients/resolvers/packagist.py +164 -0
  24. dkatchr/clients/resolvers/pypi.py +190 -0
  25. dkatchr/clients/resolvers/rubygems.py +132 -0
  26. dkatchr/clients/top_packages.py +302 -0
  27. dkatchr/config.py +428 -0
  28. dkatchr/core/__init__.py +3 -0
  29. dkatchr/core/attribution.py +464 -0
  30. dkatchr/core/confusion.py +334 -0
  31. dkatchr/core/inventory.py +104 -0
  32. dkatchr/core/osv_enrichment.py +358 -0
  33. dkatchr/core/reachability.py +793 -0
  34. dkatchr/core/rules.py +132 -0
  35. dkatchr/core/scanner.py +170 -0
  36. dkatchr/core/typosquat.py +294 -0
  37. dkatchr/logger.py +121 -0
  38. dkatchr/output/__init__.py +2 -0
  39. dkatchr/output/csv_writer.py +37 -0
  40. dkatchr/output/schema.py +83 -0
  41. dkatchr/parsers/__init__.py +74 -0
  42. dkatchr/parsers/_common.py +22 -0
  43. dkatchr/parsers/cargo.py +49 -0
  44. dkatchr/parsers/composer.py +86 -0
  45. dkatchr/parsers/go.py +42 -0
  46. dkatchr/parsers/maven.py +71 -0
  47. dkatchr/parsers/npm.py +278 -0
  48. dkatchr/parsers/nuget.py +71 -0
  49. dkatchr/parsers/pypi.py +214 -0
  50. dkatchr/parsers/ruby.py +119 -0
  51. dkatchr/storage/__init__.py +2 -0
  52. dkatchr/storage/attribution_cache.py +101 -0
  53. dkatchr/storage/inventory_cache.py +84 -0
  54. dkatchr/storage/reachability_cache.py +108 -0
  55. dkatchr-0.3.0.dist-info/METADATA +228 -0
  56. dkatchr-0.3.0.dist-info/RECORD +60 -0
  57. dkatchr-0.3.0.dist-info/WHEEL +5 -0
  58. dkatchr-0.3.0.dist-info/entry_points.txt +2 -0
  59. dkatchr-0.3.0.dist-info/licenses/LICENSE +21 -0
  60. dkatchr-0.3.0.dist-info/top_level.txt +1 -0
dkatchr/cli/run.py ADDED
@@ -0,0 +1,814 @@
1
+ """
2
+ CLI scan orchestration — the body of `main()`.
3
+
4
+ EXTRACTED FROM: dkatchr/cli.py (former main() + _scan_batch).
5
+ WHY: The old main() was 100+ lines mixing argparse parsing, batch building,
6
+ threading, CSV writing, and OSV pass orchestration. This module owns the
7
+ orchestration only — argparse lives in args.py, CSV in output/csv_writer.py,
8
+ dry-run in dry_run.py.
9
+
10
+ This is the CLI's equivalent of web/services/scan_runner.py: same Scanner,
11
+ same enrichment, different I/O at the edges (CSV vs SQLite, stdout/tqdm
12
+ vs WebSocket).
13
+ """
14
+
15
+ import json
16
+ import re
17
+ import sys
18
+ import threading
19
+ import time
20
+ from concurrent.futures import ThreadPoolExecutor, as_completed
21
+
22
+ from dkatchr.clients.epss import EPSSClient
23
+ from dkatchr.clients.exploitdb import ExploitDBClient
24
+ from dkatchr.clients.github import GitHubClient
25
+ from dkatchr.clients.kev import KEVClient
26
+ from dkatchr.clients.metasploit import MetasploitClient
27
+ from dkatchr.clients.osv import OSVClient
28
+ from dkatchr.clients.registry_meta import RegistryMetaClient
29
+ from dkatchr.clients.top_packages import TopPackagesClient
30
+ from dkatchr.config import CACHE_SCHEMA
31
+ from dkatchr.core.attribution import enrich_attribution
32
+ from dkatchr.core.confusion import detect_confusion, load_internal_patterns
33
+ from dkatchr.core.typosquat import detect_typosquats
34
+ from dkatchr.core.osv_enrichment import osv_enrich, osv_rows_for_repo
35
+ from dkatchr.core.reachability import enrich_reachability
36
+ from dkatchr.core.rules import compile_rules, load_package_config
37
+ from dkatchr.core.scanner import Scanner
38
+ from dkatchr.logger import HAS_TQDM, log, reachability_badge, set_log_file, set_quiet_repos, tqdm
39
+ from dkatchr.output.csv_writer import open_csv_writer
40
+ from dkatchr.output.schema import empty_row
41
+ from dkatchr.storage.attribution_cache import AttributionCache
42
+ from dkatchr.storage.inventory_cache import InventoryCache
43
+ from dkatchr.storage.reachability_cache import ReachabilityCache
44
+
45
+ from dkatchr.cli.dry_run import print_dry_run_report
46
+
47
+
48
+ def _repo_uses_cache(cache: InventoryCache, full_name: str, full_rescan: bool) -> bool:
49
+ """Cheap check — only consults the index, doesn't hit GitHub."""
50
+ if full_rescan:
51
+ return False
52
+ return bool(cache.get_index_sha(full_name))
53
+
54
+
55
+ def run_cli(args) -> None:
56
+ """Top-level orchestration for the CLI. `args` is the parsed argparse Namespace."""
57
+ # ---- optional file logging -----------------------------------------
58
+ if getattr(args, "log_file", None):
59
+ set_log_file(args.log_file)
60
+ log(f"[+] Logging to file: {args.log_file}")
61
+
62
+ # ---- shared infra ---------------------------------------------------
63
+ github = GitHubClient(rps=args.rps, burst=args.burst)
64
+ cache = InventoryCache(cache_dir=args.cache_dir)
65
+ rules = compile_rules(load_package_config(args.config))
66
+ scanner = Scanner(github=github, cache=cache, package_config=rules)
67
+
68
+ # ---- resolve targets ------------------------------------------------
69
+ repo_specs, orgs = _resolve_targets(github, args)
70
+ if not repo_specs and not orgs:
71
+ log("[!] Nothing to scan. Pass --orgs or --repos.")
72
+ sys.exit(1)
73
+
74
+ _print_run_header(repo_specs, orgs, rules, args)
75
+
76
+ # ---- build the (label, repos) batches -------------------------------
77
+ batches = _build_batches(github, repo_specs, orgs)
78
+
79
+ # ---- dry-run --------------------------------------------------------
80
+ if args.dry_run:
81
+ print_dry_run_report(batches, cache, args)
82
+ sys.exit(0)
83
+
84
+ # ---- progress bar setup ---------------------------------------------
85
+ use_bar = HAS_TQDM and not args.no_progress
86
+ set_quiet_repos(use_bar)
87
+ if not HAS_TQDM and not args.no_progress:
88
+ log("[!] tqdm not installed — progress bar disabled. `pip install tqdm` for a real bar.")
89
+
90
+ # ---- run ------------------------------------------------------------
91
+ totals = {"repos": 0, "rule_hits": 0, "rule_vuln": 0, "osv_hits": 0, "kev_hits": 0,
92
+ "fix_available_hits": 0, "exploitdb_hits": 0, "metasploit_hits": 0,
93
+ "exploit_hits": 0, "confusion_hits": 0, "typosquat_hits": 0}
94
+ inventory_lock = threading.Lock()
95
+ per_repo_inventory: dict[str, list[dict]] = {}
96
+
97
+ # When --reachability OR --attribution is active, we buffer all vuln rows
98
+ # per repo and write them to CSV only after the enrichment pass(es) mutate
99
+ # the row dicts in place (reachability adds reachability columns;
100
+ # attribution adds introduced_by_* columns). A single flush writes them.
101
+ buffer_rows = bool(args.reachability or args.attribution)
102
+ per_repo_rule_rows: dict[str, list[dict]] = {}
103
+ per_repo_osv_rows: dict[str, list[dict]] = {}
104
+ per_repo_sha: dict[str, str] = {}
105
+
106
+ with open_csv_writer(args.output) as writer:
107
+ for label, repos in batches:
108
+ totals["repos"] += len(repos)
109
+ _scan_batch(
110
+ label=label, repos=repos, scanner=scanner, cache=cache, args=args,
111
+ writer=writer,
112
+ inventory_lock=inventory_lock, per_repo_inventory=per_repo_inventory,
113
+ per_repo_rule_rows=per_repo_rule_rows if buffer_rows else None,
114
+ per_repo_sha=per_repo_sha if buffer_rows else None,
115
+ totals=totals, use_bar=use_bar,
116
+ )
117
+
118
+ # ---- OSV pass ---------------------------------------------------
119
+ tuple_to_vulns: dict = {}
120
+ kev_cve_set: set[str] | None = None
121
+
122
+ if args.osv and per_repo_inventory:
123
+ log(f"\n===== OSV ENRICHMENT =====")
124
+ osv = OSVClient(
125
+ cache_dir=f"{args.cache_dir}/osv",
126
+ ttl_seconds=args.osv_ttl,
127
+ rps=args.osv_rps,
128
+ burst=args.osv_burst,
129
+ )
130
+ try:
131
+ tuple_to_vulns = osv_enrich(per_repo_inventory, osv, detail_workers=args.workers)
132
+ except Exception as e:
133
+ log(f"[!] OSV enrichment failed: {e}")
134
+ tuple_to_vulns = {}
135
+
136
+ if not args.no_kev and tuple_to_vulns:
137
+ kev = KEVClient(cache_dir=f"{args.cache_dir}/kev", ttl_seconds=args.kev_ttl)
138
+ kev_cve_set = kev.load()
139
+
140
+ # EPSS exploit-probability enrichment. Fetched once (EPSSClient owns
141
+ # its date-keyed file cache + fail-safe — returns {} on total
142
+ # failure, leaving epss_score blank). Only OSV/CVE rows can match.
143
+ epss_scores = EPSSClient(cache_dir=args.cache_dir).get_scores()
144
+
145
+ # Public-exploit catalogs ("Has Exploit"). Same place + posture as
146
+ # KEV: built after enrichment, each owns a TTL'd file cache + fail-
147
+ # safe (returns an empty set on total failure), and the lookup is a
148
+ # Python CVE-token match, not a SQL/DB step. Disabled by
149
+ # --no-exploit-intel.
150
+ exploitdb_set: set[str] = set()
151
+ metasploit_set: set[str] = set()
152
+ if not args.no_exploit_intel and tuple_to_vulns:
153
+ try:
154
+ exploitdb_set = ExploitDBClient(
155
+ cache_dir=f"{args.cache_dir}/exploitdb",
156
+ ttl_seconds=args.exploit_ttl).load()
157
+ except Exception as e:
158
+ log(f"[!] Exploit-DB load failed: {e}")
159
+ try:
160
+ metasploit_set = MetasploitClient(
161
+ cache_dir=f"{args.cache_dir}/metasploit",
162
+ ttl_seconds=args.exploit_ttl).load()
163
+ except Exception as e:
164
+ log(f"[!] Metasploit load failed: {e}")
165
+
166
+ for full_name, inv in per_repo_inventory.items():
167
+ osv_rows = osv_rows_for_repo(full_name, inv, tuple_to_vulns,
168
+ kev_cve_set=kev_cve_set)
169
+ for row in osv_rows:
170
+ row["epss_score"] = _epss_pct(epss_scores, row.get("vuln_id"), row.get("aliases"))
171
+ has_exploit = _finalize_finding_row(row, exploitdb_set, metasploit_set)
172
+ if row["has_fix"] == "true": totals["fix_available_hits"] += 1
173
+ if row["has_exploitdb"] == "true": totals["exploitdb_hits"] += 1
174
+ if row["has_metasploit"] == "true": totals["metasploit_hits"] += 1
175
+ if has_exploit: totals["exploit_hits"] += 1
176
+ totals["osv_hits"] += len(osv_rows)
177
+ totals["kev_hits"] += sum(1 for r in osv_rows if r.get("is_kev") == "true")
178
+ if buffer_rows:
179
+ per_repo_osv_rows[full_name] = osv_rows
180
+ else:
181
+ writer.write_rows(osv_rows)
182
+
183
+ if not buffer_rows:
184
+ log(f"[✓] OSV: wrote {totals['osv_hits']} CVE row(s)")
185
+ if kev_cve_set is not None:
186
+ log(f"[✓] KEV: {totals['kev_hits']} finding(s) are known exploited vulnerabilities")
187
+ _print_exploit_intel_lines(args, totals)
188
+
189
+ # ---- dependency-confusion pass ----------------------------------
190
+ # Runs after the OSV pass (mirrors the web runner's ordering) but is
191
+ # NOT gated on --osv: candidates come from inventory origin + patterns,
192
+ # not OSV data. Confusion rows are written directly — the attribution
193
+ # and reachability passes deliberately ignore them, so they never need
194
+ # buffering. A pass failure never aborts the run.
195
+ if not args.no_confusion and per_repo_inventory:
196
+ _confusion_pass(args, per_repo_inventory, writer, totals)
197
+
198
+ # ---- typosquatting pass -----------------------------------------
199
+ # Opt-in (--typosquat, default OFF). Like confusion it writes rows
200
+ # directly (attribution/reachability ignore them) and never aborts the
201
+ # run. Runs after the confusion pass.
202
+ if args.typosquat and per_repo_inventory:
203
+ _typosquat_pass(args, per_repo_inventory, writer, totals)
204
+
205
+ # ---- attribution pass -------------------------------------------
206
+ # Runs AFTER the OSV pass so OSV findings (the primary vuln source) get
207
+ # attributed too — they don't exist as rows until the OSV pass builds
208
+ # them. Mirrors osv_enrich: dedups by manifest, one git-blame call per
209
+ # unique manifest file (cached by branch SHA). Merges introduced_by_*
210
+ # into the buffered finding rows in place; the reachability pass or the
211
+ # final flush below writes them.
212
+ if args.attribution:
213
+ _attribution_pass(github, args, per_repo_rule_rows, per_repo_osv_rows,
214
+ per_repo_sha, totals)
215
+
216
+ # ---- reachability pass ------------------------------------------
217
+ if args.reachability:
218
+ log(f"\n===== REACHABILITY ANALYSIS =====")
219
+ all_repos = set(per_repo_rule_rows) | set(per_repo_osv_rows)
220
+ reach_cache = ReachabilityCache(cache_dir=args.cache_dir)
221
+ reach_totals: dict[str, int] = {
222
+ "REACHABLE": 0, "UNUSED": 0, "UNKNOWN": 0, "packages": 0,
223
+ }
224
+
225
+ for full_name in all_repos:
226
+ rule_rows = per_repo_rule_rows.get(full_name, [])
227
+ osv_rows = per_repo_osv_rows.get(full_name, [])
228
+ all_rows = rule_rows + osv_rows
229
+ if not all_rows:
230
+ continue
231
+
232
+ sha = per_repo_sha.get(full_name)
233
+ if not sha:
234
+ log(f"[{full_name}] reachability skipped — no SHA available")
235
+ writer.write_rows(rule_rows)
236
+ writer.write_rows(osv_rows)
237
+ continue
238
+
239
+ cli_dl_state = {"last_emit_at": 0.0}
240
+
241
+ def _cli_reach_progress(payload: dict, _repo=full_name,
242
+ _state=cli_dl_state) -> None:
243
+ phase = payload.get("phase", "")
244
+ if phase == "cache_hit":
245
+ log(f"[{_repo}] reachability: cache hit @ {payload.get('sha','')}")
246
+ elif phase == "tarball_downloading":
247
+ # Print at most one line per second to keep stdout sane.
248
+ now = time.monotonic()
249
+ if now - _state["last_emit_at"] < 1.0:
250
+ return
251
+ _state["last_emit_at"] = now
252
+ done = payload.get("downloaded_bytes", 0)
253
+ total = payload.get("total_bytes")
254
+ if total:
255
+ pct = done * 100 // max(1, total)
256
+ log(f"[{_repo}] downloading tarball: "
257
+ f"{done / 1024 / 1024:.1f}/{total / 1024 / 1024:.1f}MB ({pct}%)")
258
+ else:
259
+ log(f"[{_repo}] downloading tarball: {done / 1024 / 1024:.1f}MB")
260
+ elif phase == "tarball_downloaded":
261
+ log(f"[{_repo}] tarball downloaded: "
262
+ f"{payload.get('size_bytes', 0) / 1024 / 1024:.1f}MB")
263
+ elif phase == "tarball_failed":
264
+ log(f"[{_repo}] tarball download FAILED")
265
+ elif phase == "building_automaton":
266
+ log(f"[{_repo}] building automaton: "
267
+ f"{payload.get('patterns', 0)} pattern(s) across "
268
+ f"{payload.get('unique_packages', 0)} package(s)")
269
+ elif phase == "scanning_progress":
270
+ log(f"[{_repo}] scanning: "
271
+ f"{payload.get('files_scanned', 0)} source files scanned, "
272
+ f"{payload.get('patterns_matched', 0)}/"
273
+ f"{payload.get('patterns_total', 0)} patterns matched")
274
+ elif phase == "scanning_complete":
275
+ log(f"[{_repo}] scan done: "
276
+ f"{payload.get('files_scanned', 0)} files scanned, "
277
+ f"{payload.get('files_skipped', 0)} skipped")
278
+ elif phase == "resolving_start":
279
+ log(f"[{_repo}] resolving import names: "
280
+ f"{payload.get('total', 0)} unique package(s)")
281
+ elif phase == "resolved":
282
+ log(f"[{_repo}] resolved {payload.get('ecosystem','')}"
283
+ f"/{payload.get('package','')} → "
284
+ f"{payload.get('names')} (via {payload.get('source','')})")
285
+ elif phase == "resolution_failed":
286
+ log(f"[{_repo}] resolution FAILED "
287
+ f"{payload.get('ecosystem','')}/{payload.get('package','')}: "
288
+ f"{payload.get('reason','')}")
289
+ elif phase == "resolving_done":
290
+ log(f"[{_repo}] import name resolution: "
291
+ f"{payload.get('resolved',0)} resolved, "
292
+ f"{payload.get('cached',0)} cached, "
293
+ f"{payload.get('failed',0)} failed")
294
+
295
+ enrich_reachability(
296
+ all_rows, full_name, sha, github, reach_cache,
297
+ on_progress=_cli_reach_progress,
298
+ cache_dir=args.cache_dir,
299
+ )
300
+
301
+ seen_pkgs: set[tuple[str, str]] = set()
302
+ for row in all_rows:
303
+ pkg_key = (row.get("package", ""), row.get("ecosystem", ""))
304
+ if pkg_key not in seen_pkgs:
305
+ seen_pkgs.add(pkg_key)
306
+ r = row.get("reachability", "UNKNOWN") or "UNKNOWN"
307
+ reach_totals[r] = reach_totals.get(r, 0) + 1
308
+ reach_totals["packages"] += 1
309
+ badge = reachability_badge(row.get("reachability", ""))
310
+ vuln_ref = row.get("vuln_id") or row.get("reason") or ""
311
+ log(f"[{full_name}] {row.get('package','')}=={row.get('version','')} {vuln_ref} {badge}")
312
+
313
+ writer.write_rows(rule_rows)
314
+ writer.write_rows(osv_rows)
315
+
316
+ if args.osv:
317
+ log(f"[✓] OSV: wrote {totals['osv_hits']} CVE row(s)")
318
+ if kev_cve_set is not None:
319
+ log(f"[✓] KEV: {totals['kev_hits']} finding(s) are known exploited vulnerabilities")
320
+ _print_exploit_intel_lines(args, totals)
321
+
322
+ _print_reachability_summary(reach_totals)
323
+
324
+ # ---- flush buffered rows ----------------------------------------
325
+ # When buffering was on for attribution only (no reachability pass that
326
+ # writes the rows itself), emit the enriched rows now — still inside the
327
+ # CSV writer context so the file is closed cleanly afterwards.
328
+ if buffer_rows and not args.reachability:
329
+ for full_name in set(per_repo_rule_rows) | set(per_repo_osv_rows):
330
+ writer.write_rows(per_repo_rule_rows.get(full_name, []))
331
+ writer.write_rows(per_repo_osv_rows.get(full_name, []))
332
+
333
+ cache.save_index()
334
+ _print_run_footer(args, totals)
335
+
336
+
337
+ # ---------------------------------------------------------------------------
338
+ # Helpers — small, single-purpose, easy to read.
339
+ # ---------------------------------------------------------------------------
340
+
341
+ _CVE_RE = re.compile(r"CVE-\d{4}-\d+", re.IGNORECASE)
342
+
343
+
344
+ def _cve_tokens(vuln_id: str | None, aliases: str | None) -> set[str]:
345
+ """Uppercased CVE ids from a finding's vuln_id + aliases.
346
+
347
+ Shared by the CVE-keyed CLI enrichments (EPSS, exploit-intel). OSV's primary
348
+ `vuln_id` is almost always GHSA-/PYSEC-/GO- with the CVE in the comma-
349
+ separated `aliases`, so both fields are scanned. Mirrors the web repository
350
+ helper of the same name.
351
+ """
352
+ return {c.upper() for c in _CVE_RE.findall(vuln_id or "")} \
353
+ | {c.upper() for c in _CVE_RE.findall(aliases or "")}
354
+
355
+
356
+ def _epss_pct(scores: dict[str, float], vuln_id: str | None, aliases: str | None = "") -> str:
357
+ """EPSS probability for a finding, as a percentage string (e.g. "12.34%").
358
+
359
+ EPSS is keyed on CVE, so we extract every CVE token from vuln_id + aliases
360
+ and take the highest matching score. Empty string when no CVE token matches
361
+ the dataset (Custom Rule rows, no coverage, etc).
362
+ """
363
+ vals = [scores[c] for c in _cve_tokens(vuln_id, aliases) if c in scores]
364
+ if not vals:
365
+ return ""
366
+ return f"{max(vals) * 100:.2f}%"
367
+
368
+
369
+ def _finalize_finding_row(row: dict, exploitdb_set: set[str],
370
+ metasploit_set: set[str]) -> bool:
371
+ """Set has_fix / has_exploitdb / has_metasploit on a finding row, in place.
372
+
373
+ Returns True if the row counts as 'has exploit' (is_kev OR ExploitDB OR
374
+ Metasploit) — the composite verdict, reported in totals but never written as
375
+ a CSV column. `has_fix` is derived from fix_versions. Custom Rule rows pass
376
+ empty exploit sets, so they get has_exploitdb/has_metasploit = "false".
377
+ """
378
+ row["has_fix"] = "true" if (row.get("fix_versions") or "").strip() else "false"
379
+ cves = _cve_tokens(row.get("vuln_id"), row.get("aliases"))
380
+ edb = bool(cves) and bool(exploitdb_set) and not cves.isdisjoint(exploitdb_set)
381
+ msf = bool(cves) and bool(metasploit_set) and not cves.isdisjoint(metasploit_set)
382
+ row["has_exploitdb"] = "true" if edb else "false"
383
+ row["has_metasploit"] = "true" if msf else "false"
384
+ return (row.get("is_kev") == "true") or edb or msf
385
+
386
+
387
+ def _confusion_csv_row(f: dict) -> dict:
388
+ """Turn a core confusion finding dict into a unified CSV row."""
389
+ row = empty_row(f["repo"], {
390
+ "file": f.get("file", ""),
391
+ "ecosystem": f.get("ecosystem", ""),
392
+ "package": f.get("package", ""),
393
+ "version": f.get("version", ""),
394
+ "version_source": f.get("version_source", ""),
395
+ "origin": f.get("origin", "registry"),
396
+ })
397
+ row["source"] = "CONFUSION"
398
+ row["status"] = "vulnerable"
399
+ row["vuln_id"] = f.get("vuln_id", "")
400
+ row["severity"] = f.get("severity", "")
401
+ row["summary"] = f.get("summary", "")
402
+ row["suspicion_signals"] = json.dumps(f.get("suspicion_signals") or {}, sort_keys=True)
403
+ # No CVE, no fix list — but keep the boolean columns uniform ("false").
404
+ _finalize_finding_row(row, set(), set())
405
+ return row
406
+
407
+
408
+ def _confusion_pass(args, per_repo_inventory: dict, writer, totals: dict) -> None:
409
+ """CLI dependency-confusion pass — patterns + registry lookups + CSV rows.
410
+
411
+ Wrapped end-to-end: a pass failure logs and the run continues (same
412
+ isolation contract as the OSV/KEV/exploit-intel passes).
413
+ """
414
+ log("\n===== DEPENDENCY CONFUSION DETECTION =====")
415
+ try:
416
+ patterns = load_internal_patterns(args.internal_patterns)
417
+ client = RegistryMetaClient(cache_dir=f"{args.cache_dir}/registry_meta")
418
+
419
+ state = {"last_emit_at": 0.0}
420
+
421
+ def _progress(payload: dict, _state=state) -> None:
422
+ # Throttle the per-candidate firehose to ~1 line/sec.
423
+ now = time.monotonic()
424
+ if now - _state["last_emit_at"] < 1.0:
425
+ return
426
+ _state["last_emit_at"] = now
427
+ log(f"[confusion] checking candidate {payload.get('current', 0)}"
428
+ f"/{payload.get('total', 0)} — "
429
+ f"{payload.get('ecosystem', '')}/{payload.get('package', '')}")
430
+
431
+ result = detect_confusion(
432
+ per_repo_inventory, patterns, client.get_package_meta,
433
+ on_progress=_progress,
434
+ supported_ecosystems=RegistryMetaClient.SUPPORTED_ECOSYSTEMS,
435
+ )
436
+ rows = [_confusion_csv_row(f) for f in result["findings"]]
437
+ writer.write_rows(rows)
438
+ totals["confusion_hits"] = len(rows)
439
+ s = result["stats"]
440
+ log(f"[✓] Confusion: {len(rows)} finding(s) — {s['checked']} candidate(s) "
441
+ f"checked, {s['public']} exist publicly"
442
+ + (f", {s['skipped_unsupported']} skipped (unsupported registry)"
443
+ if s["skipped_unsupported"] else "")
444
+ + (f", {s['skipped_unresolved']} lookup failure(s)"
445
+ if s["skipped_unresolved"] else ""))
446
+ except Exception as e:
447
+ log(f"[!] Dependency-confusion pass failed: {e} — continuing")
448
+
449
+
450
+ def _typosquat_csv_row(f: dict) -> dict:
451
+ """Turn a core typosquat finding dict into a unified CSV row (mirror of
452
+ _confusion_csv_row — reuses the same origin + suspicion_signals columns)."""
453
+ row = empty_row(f["repo"], {
454
+ "file": f.get("file", ""),
455
+ "ecosystem": f.get("ecosystem", ""),
456
+ "package": f.get("package", ""),
457
+ "version": f.get("version", ""),
458
+ "version_source": f.get("version_source", ""),
459
+ "origin": f.get("origin", "registry"),
460
+ })
461
+ row["source"] = "TYPOSQUAT"
462
+ row["status"] = "vulnerable"
463
+ row["vuln_id"] = f.get("vuln_id", "")
464
+ row["severity"] = f.get("severity", "")
465
+ row["summary"] = f.get("summary", "")
466
+ row["suspicion_signals"] = json.dumps(f.get("suspicion_signals") or {}, sort_keys=True)
467
+ _finalize_finding_row(row, set(), set())
468
+ return row
469
+
470
+
471
+ def _typosquat_pass(args, per_repo_inventory: dict, writer, totals: dict) -> None:
472
+ """CLI typosquatting pass — top-package feeds + edit distance + metadata gate.
473
+
474
+ Wrapped end-to-end: a pass failure logs and the run continues (same isolation
475
+ contract as the OSV/confusion passes). The metadata gate reuses
476
+ RegistryMetaClient (no second registry client).
477
+ """
478
+ log("\n===== TYPOSQUATTING DETECTION =====")
479
+ try:
480
+ top_client = TopPackagesClient(cache_dir=f"{args.cache_dir}/top_packages")
481
+ meta_client = RegistryMetaClient(cache_dir=f"{args.cache_dir}/registry_meta")
482
+
483
+ present_ecos = {
484
+ dep.get("ecosystem")
485
+ for inv in per_repo_inventory.values() for dep in (inv or [])
486
+ if dep.get("ecosystem")
487
+ }
488
+ feed_state = {"last_emit_at": 0.0}
489
+
490
+ def _feed_progress(payload: dict, _state=feed_state) -> None:
491
+ now = time.monotonic()
492
+ if now - _state["last_emit_at"] < 1.0:
493
+ return
494
+ _state["last_emit_at"] = now
495
+ log(f"[typosquat] fetching {payload.get('ecosystem', '')} top packages "
496
+ f"({payload.get('fetched', 0)} so far)")
497
+
498
+ top_sets = {
499
+ eco: top_client.get_top(eco, on_progress=_feed_progress)
500
+ for eco in present_ecos
501
+ if eco in top_client.SUPPORTED_ECOSYSTEMS
502
+ }
503
+
504
+ lk_state = {"last_emit_at": 0.0}
505
+
506
+ def _progress(payload: dict, _state=lk_state) -> None:
507
+ if payload.get("phase") != "typosquat_lookup":
508
+ return
509
+ now = time.monotonic()
510
+ if now - _state["last_emit_at"] < 1.0:
511
+ return
512
+ _state["last_emit_at"] = now
513
+ log(f"[typosquat] checking candidate {payload.get('current', 0)}"
514
+ f"/{payload.get('total', 0)} — "
515
+ f"{payload.get('ecosystem', '')}/{payload.get('package', '')}")
516
+
517
+ result = detect_typosquats(
518
+ per_repo_inventory, top_sets, meta_client.get_package_meta,
519
+ on_progress=_progress,
520
+ supported_ecosystems=TopPackagesClient.SUPPORTED_ECOSYSTEMS,
521
+ )
522
+ rows = [_typosquat_csv_row(f) for f in result["findings"]]
523
+ writer.write_rows(rows)
524
+ totals["typosquat_hits"] = len(rows)
525
+ s = result["stats"]
526
+ log(f"[✓] Typosquat: {len(rows)} finding(s) — {s['candidates']} candidate(s) "
527
+ f"evaluated, {s['distance_hits']} close-name match(es), {s['public']} "
528
+ f"confirmed on the public registry"
529
+ + (f", {s['skipped_unsupported']} skipped (unsupported ecosystem)"
530
+ if s["skipped_unsupported"] else "")
531
+ + (f", {s['skipped_unresolved']} lookup failure(s)"
532
+ if s["skipped_unresolved"] else ""))
533
+ except Exception as e:
534
+ log(f"[!] Typosquatting pass failed: {e} — continuing")
535
+
536
+
537
+ def _print_exploit_intel_lines(args, totals: dict) -> None:
538
+ """Mid-run '[✓]' summary lines for the public-exploit cross-reference,
539
+ mirroring the KEV summary line's format. No-op when --no-exploit-intel."""
540
+ if args.no_exploit_intel:
541
+ return
542
+ log(f"[✓] Exploit-DB: {totals['exploitdb_hits']} finding(s) have known public exploit code")
543
+ log(f"[✓] Metasploit: {totals['metasploit_hits']} finding(s) have a Metasploit module")
544
+ log(f"[✓] Has Exploit: {totals['exploit_hits']} finding(s) (KEV/ExploitDB/Metasploit)")
545
+
546
+
547
+ def _merge_attribution(rows: list[dict], attr_map: dict) -> int:
548
+ """Write introduced_by_* CSV columns onto finding rows from attr_map.
549
+
550
+ Keyed on (repo, file, package, version) — the same manifest line a finding
551
+ points at. Returns the count of rows that received a real commit. Only
552
+ RESULT_FIELDS keys are written, so the rows stay DictWriter-safe.
553
+ """
554
+ n = 0
555
+ for r in rows:
556
+ a = attr_map.get((r.get("repo"), r.get("file"), r.get("package"), r.get("version")))
557
+ if not a:
558
+ continue
559
+ sha = a.get("commit_sha")
560
+ r["introduced_by_handle"] = a.get("author_handle") or ""
561
+ r["introduced_by_name"] = a.get("author_name") or ""
562
+ r["introduced_by_commit"] = sha or ""
563
+ r["introduced_by_date"] = a.get("commit_date") or ""
564
+ r["introduced_by_message"] = a.get("commit_message") or ""
565
+ r["introduced_by_is_bot"] = ("true" if a.get("is_bot") else "false") if sha else ""
566
+ if sha:
567
+ n += 1
568
+ return n
569
+
570
+
571
+ def _attribution_pass(github: GitHubClient, args, per_repo_rule_rows: dict,
572
+ per_repo_osv_rows: dict, per_repo_sha: dict, totals: dict) -> None:
573
+ """CLI attribution pass — the direct call into core enrich_attribution().
574
+
575
+ Builds deduped attribution inputs from the buffered finding rows (+ each
576
+ repo's branch SHA), runs enrich_attribution, and merges introduced_by_*
577
+ columns back into the buffered rows by (repo, file, package, version).
578
+ """
579
+ log("\n===== ATTRIBUTION (Introduced By) =====")
580
+ attr_cache = AttributionCache(cache_dir=args.cache_dir)
581
+
582
+ all_finding_rows: list[dict] = []
583
+ for rows in per_repo_rule_rows.values():
584
+ all_finding_rows.extend(rows)
585
+ for rows in per_repo_osv_rows.values():
586
+ all_finding_rows.extend(rows)
587
+
588
+ if not all_finding_rows:
589
+ log("[attribution] no findings to attribute — skipping")
590
+ return
591
+
592
+ # One attribution input per unique (repo, file, package, version) finding.
593
+ seen_keys: set = set()
594
+ attr_input: list[dict] = []
595
+ for r in all_finding_rows:
596
+ key = (r.get("repo"), r.get("file"), r.get("package"), r.get("version"))
597
+ if key in seen_keys:
598
+ continue
599
+ seen_keys.add(key)
600
+ attr_input.append({
601
+ "repo_full_name": r.get("repo"),
602
+ "file": r.get("file"),
603
+ "sha": per_repo_sha.get(r.get("repo")),
604
+ "ecosystem": r.get("ecosystem"),
605
+ "package": r.get("package"),
606
+ "version": r.get("version"),
607
+ })
608
+
609
+ manifests = len({(i["repo_full_name"], i["file"], i["sha"]) for i in attr_input})
610
+ state = {"last_emit_at": 0.0}
611
+
612
+ def _attr_progress(payload: dict, _state=state) -> None:
613
+ # Throttle the per-manifest firehose to ~1 line/sec for readable stdout.
614
+ if payload.get("phase") != "attribution_progress":
615
+ return
616
+ now = time.monotonic()
617
+ if now - _state["last_emit_at"] < 1.0:
618
+ return
619
+ _state["last_emit_at"] = now
620
+ log(f"[attribution] {payload.get('current', 0)}/{payload.get('total', 0)} "
621
+ f"manifests — {payload.get('repo', '')}:{payload.get('path', '')}")
622
+
623
+ enriched = enrich_attribution(attr_input, github, attr_cache, on_progress=_attr_progress)
624
+
625
+ attr_map = {
626
+ (e.get("repo_full_name"), e.get("file"), e.get("package"), e.get("version")): e.get("attribution")
627
+ for e in enriched
628
+ }
629
+ attributed = _merge_attribution(all_finding_rows, attr_map)
630
+ totals["attributed"] = attributed
631
+ log(f"[✓] Attribution complete: {manifests} manifests queried, {attributed} findings attributed")
632
+
633
+
634
+ def _resolve_targets(github: GitHubClient, args) -> tuple[list[str], list[str]]:
635
+ """Targeting precedence: --repos > --orgs > /user/orgs auto-discover."""
636
+ if args.repos:
637
+ return args.repos, []
638
+ if args.orgs:
639
+ return [], args.orgs
640
+ try:
641
+ return [], github.list_user_orgs()
642
+ except Exception as e:
643
+ log(f"[!] Could not auto-discover orgs via /user/orgs: {e}")
644
+ log("[!] Pass --orgs <org> [...] or --repos owner/name [...] to target explicitly.")
645
+ sys.exit(1)
646
+
647
+
648
+ def _print_run_header(repo_specs: list[str], orgs: list[str], rules: dict, args) -> None:
649
+ if repo_specs:
650
+ log(f"[+] Repos: {repo_specs}")
651
+ else:
652
+ log(f"[+] Orgs: {orgs}")
653
+ log(f"[+] Output: {args.output}")
654
+ log(f"[+] Cache: {args.cache_dir} (schema v{CACHE_SCHEMA}) Rules: {len(rules)}")
655
+ log(f"[+] Workers: {args.workers} Rate cap: {args.rps} req/s Burst: {args.burst}")
656
+ log(f"[+] OSV mode: {'ON' if args.osv else 'off'}"
657
+ + (f" TTL: {args.osv_ttl}s Rate cap: {args.osv_rps} req/s" if args.osv else ""))
658
+ if args.osv:
659
+ log(f"[+] KEV mode: {'off (--no-kev)' if args.no_kev else 'ON'} TTL: {args.kev_ttl}s")
660
+ log(f"[+] Exploit: {'off (--no-exploit-intel)' if args.no_exploit_intel else 'ON'} TTL: {args.exploit_ttl}s")
661
+ log(f"[+] Confusion: {'off (--no-confusion)' if args.no_confusion else 'ON'}"
662
+ + (f" Patterns: {args.internal_patterns}" if args.internal_patterns else ""))
663
+ log(f"[+] Typosquat: {'ON' if args.typosquat else 'off'}")
664
+ if not rules and not args.osv:
665
+ log("[!] No rules and no --osv. Inventory will be cached but no CSV rows will be written.")
666
+
667
+
668
+ def _print_run_footer(args, totals: dict) -> None:
669
+ log(f"\n[✓] Done. repos={totals['repos']}")
670
+ log(f" Rule rows: {totals['rule_hits']} (vulnerable: {totals['rule_vuln']})")
671
+ if args.osv:
672
+ log(f" OSV rows: {totals['osv_hits']}")
673
+ if not args.no_kev:
674
+ log(f" KEV: {totals['kev_hits']} finding(s) are known exploited vulnerabilities")
675
+ if not args.no_exploit_intel:
676
+ log(f" Exploit-DB: {totals['exploitdb_hits']} finding(s) have known public exploit code")
677
+ log(f" Metasploit: {totals['metasploit_hits']} finding(s) have a Metasploit module")
678
+ log(f" Has Exploit: {totals['exploit_hits']} finding(s) (KEV/ExploitDB/Metasploit)")
679
+ log(f" Fix avail: {totals['fix_available_hits']} finding(s) have a fix version available")
680
+ if not args.no_confusion:
681
+ log(f" Confusion: {totals['confusion_hits']} dependency-confusion finding(s)")
682
+ if args.typosquat:
683
+ log(f" Typosquat: {totals['typosquat_hits']} typosquatting finding(s)")
684
+ log(f" Output: {args.output}")
685
+
686
+
687
+ def _print_reachability_summary(reach_totals: dict) -> None:
688
+ sep = "─" * 36
689
+ log(f"\n{sep}")
690
+ log("Reachability Summary")
691
+ log(sep)
692
+ log(f"REACHABLE {reach_totals.get('REACHABLE', 0):4d} (fix these)")
693
+ log(f"UNUSED {reach_totals.get('UNUSED', 0):4d} (deprioritise)")
694
+ log(f"UNKNOWN {reach_totals.get('UNKNOWN', 0):4d} (verify manually)")
695
+ log(sep)
696
+
697
+
698
+ def _build_batches(github: GitHubClient, repo_specs: list[str],
699
+ orgs: list[str]) -> list[tuple[str, list[dict]]]:
700
+ """Group target repos by label for per-batch progress reporting."""
701
+ batches: list[tuple[str, list[dict]]] = []
702
+ if repo_specs:
703
+ repos: list[dict] = []
704
+ for spec in repo_specs:
705
+ spec = spec.strip().strip("/") # zsh/fish can prepend a slash on tab-complete
706
+ if "/" not in spec:
707
+ log(f"[!] Bad --repos value '{spec}' — expected owner/name. Skipping.")
708
+ continue
709
+ owner, name = spec.split("/", 1)
710
+ try:
711
+ repos.append(github.get_repo_meta(owner, name))
712
+ except Exception as e:
713
+ log(f"[!] Could not fetch repo {spec}: {e}")
714
+ if repos:
715
+ batches.append(("--repos targets", repos))
716
+ else:
717
+ for org in orgs:
718
+ log(f"\n===== ORG: {org} =====")
719
+ try:
720
+ batches.append((org, github.list_org_repos(org)))
721
+ except Exception as e:
722
+ log(f"[!] Failed to list repos for '{org}': {e}")
723
+ continue
724
+ return batches
725
+
726
+
727
+ def _scan_batch(
728
+ label: str, repos: list[dict], scanner: Scanner, cache: InventoryCache, args,
729
+ writer, inventory_lock: threading.Lock, per_repo_inventory: dict,
730
+ totals: dict, use_bar: bool,
731
+ per_repo_rule_rows: dict | None = None,
732
+ per_repo_sha: dict | None = None,
733
+ ) -> None:
734
+ """Submit a batch of repos to the thread pool and drain results."""
735
+ n = len(repos)
736
+ with ThreadPoolExecutor(max_workers=args.workers) as pool:
737
+ futures = {
738
+ pool.submit(
739
+ scanner.scan_repo,
740
+ repo,
741
+ f"[{idx}/{n}]",
742
+ args.full_rescan or not _repo_uses_cache(cache, repo["full_name"], args.full_rescan),
743
+ ): repo["full_name"]
744
+ for idx, repo in enumerate(repos, 1)
745
+ }
746
+
747
+ completed_iter = as_completed(futures)
748
+ bar = None
749
+ if use_bar:
750
+ bar = tqdm(
751
+ completed_iter,
752
+ total=len(futures),
753
+ desc=f"Scanning [{label}]",
754
+ unit="repo",
755
+ dynamic_ncols=True,
756
+ )
757
+ completed_iter = bar
758
+
759
+ done_count = 0
760
+ batch_inv = 0
761
+
762
+ for fut in completed_iter:
763
+ done_count += 1
764
+ full_name = futures[fut]
765
+ try:
766
+ result = fut.result()
767
+ except Exception as e:
768
+ log(f"[!] {full_name}: worker crashed: {e}")
769
+ continue
770
+
771
+ if result["skipped"] or result.get("error"):
772
+ if bar:
773
+ bar.set_postfix(rule_hits=totals["rule_hits"], vuln=totals["rule_vuln"], inv=batch_inv)
774
+ continue
775
+
776
+ rows = result["rows"]
777
+ vuln = sum(1 for r in rows if r.get("status") == "vulnerable")
778
+ batch_inv += len(result["inventory"])
779
+
780
+ # Custom Rule rows carry no CVE, so they never match the exploit
781
+ # catalogs (empty sets) — but they still need the has_* columns set
782
+ # uniformly ("false") and has_fix derived from fix_versions.
783
+ for r in rows:
784
+ _finalize_finding_row(r, set(), set())
785
+ totals["fix_available_hits"] += sum(1 for r in rows if r.get("has_fix") == "true")
786
+
787
+ totals["rule_hits"] += len(rows)
788
+ totals["rule_vuln"] += vuln
789
+
790
+ if per_repo_rule_rows is not None:
791
+ with inventory_lock:
792
+ per_repo_rule_rows[full_name] = rows
793
+ else:
794
+ writer.write_rows(rows)
795
+
796
+ # Inventory feeds the OSV pass, the confusion pass AND the typosquat
797
+ # pass — collect it when any is active.
798
+ if args.osv or not args.no_confusion or args.typosquat:
799
+ with inventory_lock:
800
+ per_repo_inventory[full_name] = result["inventory"]
801
+
802
+ if result["sha"]:
803
+ cache.update_index(full_name, result["sha"])
804
+ if per_repo_sha is not None:
805
+ with inventory_lock:
806
+ per_repo_sha[full_name] = result["sha"]
807
+
808
+ if bar:
809
+ bar.set_postfix(rule_hits=totals["rule_hits"], vuln=totals["rule_vuln"], inv=batch_inv)
810
+ elif args.summary_every > 0 and done_count % args.summary_every == 0:
811
+ log(f"[~] [{label}] {done_count}/{n} repos done — inv={batch_inv}, rule_hits={totals['rule_hits']}, vuln={totals['rule_vuln']}")
812
+
813
+ if bar:
814
+ bar.close()