dkatchr 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. dkatchr/__init__.py +3 -0
  2. dkatchr/__main__.py +6 -0
  3. dkatchr/cli/__init__.py +7 -0
  4. dkatchr/cli/args.py +84 -0
  5. dkatchr/cli/dry_run.py +68 -0
  6. dkatchr/cli/main.py +14 -0
  7. dkatchr/cli/run.py +814 -0
  8. dkatchr/clients/__init__.py +3 -0
  9. dkatchr/clients/epss.py +153 -0
  10. dkatchr/clients/exploitdb.py +155 -0
  11. dkatchr/clients/github.py +482 -0
  12. dkatchr/clients/kev.py +143 -0
  13. dkatchr/clients/metasploit.py +147 -0
  14. dkatchr/clients/nvd.py +121 -0
  15. dkatchr/clients/osv.py +191 -0
  16. dkatchr/clients/ratelimit.py +33 -0
  17. dkatchr/clients/registry_meta.py +280 -0
  18. dkatchr/clients/resolvers/__init__.py +335 -0
  19. dkatchr/clients/resolvers/_base.py +153 -0
  20. dkatchr/clients/resolvers/crates.py +51 -0
  21. dkatchr/clients/resolvers/maven.py +144 -0
  22. dkatchr/clients/resolvers/nuget.py +216 -0
  23. dkatchr/clients/resolvers/packagist.py +164 -0
  24. dkatchr/clients/resolvers/pypi.py +190 -0
  25. dkatchr/clients/resolvers/rubygems.py +132 -0
  26. dkatchr/clients/top_packages.py +302 -0
  27. dkatchr/config.py +428 -0
  28. dkatchr/core/__init__.py +3 -0
  29. dkatchr/core/attribution.py +464 -0
  30. dkatchr/core/confusion.py +334 -0
  31. dkatchr/core/inventory.py +104 -0
  32. dkatchr/core/osv_enrichment.py +358 -0
  33. dkatchr/core/reachability.py +793 -0
  34. dkatchr/core/rules.py +132 -0
  35. dkatchr/core/scanner.py +170 -0
  36. dkatchr/core/typosquat.py +294 -0
  37. dkatchr/logger.py +121 -0
  38. dkatchr/output/__init__.py +2 -0
  39. dkatchr/output/csv_writer.py +37 -0
  40. dkatchr/output/schema.py +83 -0
  41. dkatchr/parsers/__init__.py +74 -0
  42. dkatchr/parsers/_common.py +22 -0
  43. dkatchr/parsers/cargo.py +49 -0
  44. dkatchr/parsers/composer.py +86 -0
  45. dkatchr/parsers/go.py +42 -0
  46. dkatchr/parsers/maven.py +71 -0
  47. dkatchr/parsers/npm.py +278 -0
  48. dkatchr/parsers/nuget.py +71 -0
  49. dkatchr/parsers/pypi.py +214 -0
  50. dkatchr/parsers/ruby.py +119 -0
  51. dkatchr/storage/__init__.py +2 -0
  52. dkatchr/storage/attribution_cache.py +101 -0
  53. dkatchr/storage/inventory_cache.py +84 -0
  54. dkatchr/storage/reachability_cache.py +108 -0
  55. dkatchr-0.3.0.dist-info/METADATA +228 -0
  56. dkatchr-0.3.0.dist-info/RECORD +60 -0
  57. dkatchr-0.3.0.dist-info/WHEEL +5 -0
  58. dkatchr-0.3.0.dist-info/entry_points.txt +2 -0
  59. dkatchr-0.3.0.dist-info/licenses/LICENSE +21 -0
  60. dkatchr-0.3.0.dist-info/top_level.txt +1 -0
dkatchr/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """dKatchr — multi-ecosystem GitHub dependency scanner with OSV enrichment."""
2
+
3
+ __version__ = "0.3.0"
dkatchr/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ """Entry point: `python -m dkatchr`."""
2
+
3
+ from dkatchr.cli import main
4
+
5
+ if __name__ == "__main__":
6
+ main()
@@ -0,0 +1,7 @@
1
+ """CLI surface — argparse, run orchestration, dry-run reporting. This is the
2
+ ONLY package allowed to touch sys.argv, sys.exit, and the CSV file path.
3
+ Everything underneath is library code the web app reuses unchanged."""
4
+
5
+ from dkatchr.cli.main import main # re-export for `python -m dkatchr` and shims
6
+
7
+ __all__ = ["main"]
dkatchr/cli/args.py ADDED
@@ -0,0 +1,84 @@
1
+ """
2
+ argparse builder for the CLI.
3
+
4
+ EXTRACTED FROM: dkatchr/cli.py (build_arg_parser).
5
+ WHY: argparse is pure input parsing — keep it out of orchestration so adding
6
+ a flag doesn't risk breaking the runner. Single responsibility.
7
+ """
8
+
9
+ import argparse
10
+
11
+ from dkatchr.config import (
12
+ DEFAULT_CACHE_DIR,
13
+ DEFAULT_GITHUB_BURST,
14
+ DEFAULT_GITHUB_RPS,
15
+ DEFAULT_OSV_BURST,
16
+ DEFAULT_OSV_RPS,
17
+ DEFAULT_SUMMARY_EVERY,
18
+ DEFAULT_WORKERS,
19
+ EXPLOITDB_DEFAULT_TTL,
20
+ KEV_DEFAULT_TTL,
21
+ OSV_DEFAULT_TTL,
22
+ )
23
+
24
+
25
+ def build_arg_parser() -> argparse.ArgumentParser:
26
+ p = argparse.ArgumentParser(
27
+ prog="dkatchr",
28
+ description="Multi-ecosystem GitHub dependency scanner with caching + OSV CVE enrichment",
29
+ formatter_class=argparse.RawDescriptionHelpFormatter,
30
+ )
31
+ p.add_argument("-o", "--output", required=True, help="Output CSV file path")
32
+ p.add_argument("-c", "--config", help="JSON file with package rules (overrides inline DEFAULT_PACKAGE_CONFIG)")
33
+ p.add_argument("--cache-dir", default=DEFAULT_CACHE_DIR, help=f"Cache directory (default: {DEFAULT_CACHE_DIR})")
34
+ p.add_argument("--full-rescan", action="store_true", help="Ignore cache, re-scan all repos")
35
+ p.add_argument("--orgs", nargs="+", help="GitHub org names to scan (default: auto-discover via /user/orgs)")
36
+ p.add_argument("--repos", nargs="+", help="Specific repos to scan as owner/name (skips org listing entirely)")
37
+ p.add_argument("--workers", type=int, default=DEFAULT_WORKERS, help=f"Worker threads for per-repo parallelism (default: {DEFAULT_WORKERS})")
38
+ p.add_argument("--rps", type=float, default=DEFAULT_GITHUB_RPS, help=f"Global GitHub API rate cap, requests/sec (default: {DEFAULT_GITHUB_RPS} — sustainable vs 5000/hr budget)")
39
+ p.add_argument("--burst", type=float, default=DEFAULT_GITHUB_BURST, help=f"Token-bucket burst capacity (default: {DEFAULT_GITHUB_BURST})")
40
+ p.add_argument("--osv", action="store_true", help="Enrich findings with CVE data from OSV.dev (Google's open-source vuln DB)")
41
+ p.add_argument("--osv-ttl", type=int, default=OSV_DEFAULT_TTL, help=f"OSV (eco,pkg,ver)→IDs cache TTL in seconds (default: {OSV_DEFAULT_TTL} = 24h)")
42
+ p.add_argument("--osv-rps", type=float, default=DEFAULT_OSV_RPS, help=f"OSV API rate cap, requests/sec (default: {DEFAULT_OSV_RPS})")
43
+ p.add_argument("--osv-burst", type=float, default=DEFAULT_OSV_BURST, help=f"OSV token-bucket burst capacity (default: {DEFAULT_OSV_BURST})")
44
+ p.add_argument("--no-kev", action="store_true", help="Disable CISA KEV cross-reference (on by default when --osv is used)")
45
+ p.add_argument("--kev-ttl", type=int, default=KEV_DEFAULT_TTL, help=f"KEV catalog cache TTL in seconds (default: {KEV_DEFAULT_TTL} = 24h)")
46
+ p.add_argument("--no-exploit-intel", action="store_true",
47
+ help="Disable the public-exploit cross-reference (ExploitDB + Metasploit; "
48
+ "on by default when --osv is used). Drives the has_exploitdb/has_metasploit "
49
+ "CSV columns and the 'Has Exploit' summary.")
50
+ p.add_argument("--no-confusion", action="store_true",
51
+ help="Disable the dependency-confusion detection pass (ON by default). "
52
+ "Checks internal-looking package names (path/git/workspace deps, "
53
+ "internal namespace patterns, cross-repo mismatches) against the "
54
+ "public registries and writes source=CONFUSION rows.")
55
+ p.add_argument("--internal-patterns", default=None,
56
+ help="JSON file of internal namespace patterns for the confusion pass: "
57
+ "[{\"ecosystem\": \"npm\", \"pattern\": \"@acme/*\"}, ...]. "
58
+ "Optional — without it, only the manifest-internal (S1) and "
59
+ "cross-repo (S3) signals run.")
60
+ p.add_argument("--typosquat", action="store_true",
61
+ help="Enable typosquatting detection (OFF by default — this is a "
62
+ "POSITIVE opt-in flag, unlike --no-confusion). Measures edit "
63
+ "distance of each installed registry dependency against the "
64
+ "ecosystem's top public packages and, on a close match, confirms "
65
+ "via a registry-metadata gate (recently created OR few downloads) "
66
+ "before writing source=TYPOSQUAT rows. Coverage: "
67
+ "npm/PyPI/crates.io/RubyGems/Packagist.")
68
+ p.add_argument("--exploit-ttl", type=int, default=EXPLOITDB_DEFAULT_TTL,
69
+ help=f"ExploitDB + Metasploit catalog cache TTL in seconds (default: {EXPLOITDB_DEFAULT_TTL} = 24h)")
70
+ p.add_argument("--no-progress", action="store_true", help="Disable tqdm progress bar (forces verbose per-repo logging)")
71
+ p.add_argument("--summary-every", type=int, default=DEFAULT_SUMMARY_EVERY, help=f"In no-bar mode, print a running summary every N completed repos (default: {DEFAULT_SUMMARY_EVERY})")
72
+ p.add_argument("--dry-run", action="store_true", help="List repos that would be scanned + estimate cost/wall time, then exit")
73
+ p.add_argument("--reachability", action="store_true",
74
+ help="Run Level 1 reachability analysis after vulnerability detection. "
75
+ "Uses GitHub Code Search API (~1s per unique vulnerable package). "
76
+ "Requires GITHUB_TOKEN (already required for scanning).")
77
+ p.add_argument("--attribution", action="store_true",
78
+ help="Attribute each finding to the commit author who last touched that "
79
+ "package's manifest line ('Introduced By'). One git-blame call per "
80
+ "unique manifest file (cached by branch SHA). Requires GITHUB_TOKEN.")
81
+ p.add_argument("--log-file", default=None,
82
+ help="Also write all stderr log lines to this file (in addition to the terminal). "
83
+ "Useful for post-mortem inspection of a long scan.")
84
+ return p
dkatchr/cli/dry_run.py ADDED
@@ -0,0 +1,68 @@
1
+ """
2
+ Dry-run report — estimate API cost + wall time without making any scan calls.
3
+
4
+ EXTRACTED FROM: dkatchr/cli.py (_dry_run_report).
5
+ WHY: It's a self-contained reporting feature. Pulling it out keeps the
6
+ orchestration runner small and lets us evolve the estimate model without
7
+ touching the scan path.
8
+ """
9
+
10
+ from dkatchr.logger import log
11
+ from dkatchr.storage.inventory_cache import InventoryCache
12
+
13
+
14
+ def _repo_uses_cache(cache: InventoryCache, full_name: str, full_rescan: bool) -> bool:
15
+ """Cheap check — only consults the index, doesn't hit GitHub."""
16
+ if full_rescan:
17
+ return False
18
+ return bool(cache.get_index_sha(full_name))
19
+
20
+
21
+ def print_dry_run_report(batches, cache: InventoryCache, args) -> None:
22
+ """Estimate cost + wall time without making any scan API calls."""
23
+ all_repos = [r for _, batch in batches for r in batch]
24
+ n_total = len(all_repos)
25
+ n_archived = sum(1 for r in all_repos if r.get("archived"))
26
+ n_no_branch = sum(1 for r in all_repos if not r.get("default_branch"))
27
+ n_active = n_total - n_archived - n_no_branch
28
+ n_warm = sum(1 for r in all_repos if _repo_uses_cache(cache, r["full_name"], args.full_rescan))
29
+ n_cold = n_active - n_warm
30
+
31
+ cold_cost = n_cold * 5
32
+ warm_cost = n_warm * 1
33
+ total_cost = cold_cost + warm_cost
34
+ wall_seconds = total_cost / max(args.rps, 0.001)
35
+ budget_pct = total_cost / 5000 * 100
36
+
37
+ log(f"\n===== DRY RUN =====")
38
+ log(f" Repos discovered: {n_total}")
39
+ log(f" archived (skipped): {n_archived}")
40
+ log(f" no default branch: {n_no_branch}")
41
+ log(f" active to scan: {n_active}")
42
+ log(f" warm cache hits: {n_warm}")
43
+ log(f" cold (full scan): {n_cold}")
44
+ log(f"")
45
+ log(f" Estimated GitHub API calls: ~{total_cost} ({cold_cost} cold + {warm_cost} warm)")
46
+ log(f" Hourly budget usage: ~{budget_pct:.1f}% of 5000 req/hr")
47
+ log(f" Wall time at {args.rps:.2f} req/s: ~{wall_seconds/60:.1f} min ({wall_seconds/3600:.2f} h)")
48
+ if budget_pct > 100:
49
+ extra_hours = (budget_pct - 100) / 100 + 1
50
+ log(f" [!] Over hourly budget — will pause for rate-limit reset(s); add ~{extra_hours:.1f}h")
51
+ if args.osv:
52
+ log(f"")
53
+ log(f" OSV enrichment: enabled (separate budget, ~1-5 min added regardless of repo count)")
54
+
55
+ sample = all_repos[:10]
56
+ if sample:
57
+ log(f"")
58
+ log(f" First {len(sample)} repos:")
59
+ for r in sample:
60
+ tag = (
61
+ "[archived]" if r.get("archived")
62
+ else "[no-branch]" if not r.get("default_branch")
63
+ else "[cached]" if _repo_uses_cache(cache, r["full_name"], args.full_rescan)
64
+ else "[cold]"
65
+ )
66
+ log(f" {tag:<12} {r['full_name']}")
67
+ if n_total > len(sample):
68
+ log(f" ... and {n_total - len(sample)} more")
dkatchr/cli/main.py ADDED
@@ -0,0 +1,14 @@
1
+ """
2
+ CLI entry point — argparse + delegate to runner.
3
+
4
+ REPLACES: the old dkatchr/cli.py (316 lines, mixed concerns).
5
+ The new file is tiny on purpose — it only parses args and calls run_cli.
6
+ """
7
+
8
+ from dkatchr.cli.args import build_arg_parser
9
+ from dkatchr.cli.run import run_cli
10
+
11
+
12
+ def main() -> None:
13
+ args = build_arg_parser().parse_args()
14
+ run_cli(args)