dkatchr 0.3.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {dkatchr-0.3.2 → dkatchr-0.4.0}/PKG-INFO +9 -3
  2. {dkatchr-0.3.2 → dkatchr-0.4.0}/README.md +8 -2
  3. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/__init__.py +1 -1
  4. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/args.py +3 -0
  5. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/run.py +38 -3
  6. dkatchr-0.4.0/dkatchr/clients/dependency_meta.py +239 -0
  7. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/config.py +34 -1
  8. dkatchr-0.4.0/dkatchr/core/dependency_graph.py +311 -0
  9. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/inventory.py +11 -0
  10. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/reachability.py +106 -11
  11. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/output/schema.py +5 -0
  12. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/cargo.py +13 -0
  13. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/composer.py +15 -1
  14. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/npm.py +142 -5
  15. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/nuget.py +9 -0
  16. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/pypi.py +10 -0
  17. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/ruby.py +16 -0
  18. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/PKG-INFO +9 -3
  19. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/SOURCES.txt +7 -1
  20. {dkatchr-0.3.2 → dkatchr-0.4.0}/pyproject.toml +1 -1
  21. dkatchr-0.4.0/tests/test_dependency_graph.py +154 -0
  22. dkatchr-0.4.0/tests/test_dependency_meta.py +76 -0
  23. dkatchr-0.4.0/tests/test_parser_edges.py +93 -0
  24. dkatchr-0.4.0/tests/test_reachability_graph.py +103 -0
  25. {dkatchr-0.3.2 → dkatchr-0.4.0}/LICENSE +0 -0
  26. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/__main__.py +0 -0
  27. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/__init__.py +0 -0
  28. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/dry_run.py +0 -0
  29. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/main.py +0 -0
  30. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/__init__.py +0 -0
  31. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/epss.py +0 -0
  32. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/exploitdb.py +0 -0
  33. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/github.py +0 -0
  34. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/kev.py +0 -0
  35. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/metasploit.py +0 -0
  36. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/nvd.py +0 -0
  37. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/osv.py +0 -0
  38. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/ratelimit.py +0 -0
  39. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/registry_meta.py +0 -0
  40. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/__init__.py +0 -0
  41. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/_base.py +0 -0
  42. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/crates.py +0 -0
  43. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/maven.py +0 -0
  44. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/nuget.py +0 -0
  45. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/packagist.py +0 -0
  46. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/pypi.py +0 -0
  47. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/rubygems.py +0 -0
  48. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/top_packages.py +0 -0
  49. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/__init__.py +0 -0
  50. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/attribution.py +0 -0
  51. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/confusion.py +0 -0
  52. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/osv_enrichment.py +0 -0
  53. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/rules.py +0 -0
  54. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/scanner.py +0 -0
  55. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/typosquat.py +0 -0
  56. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/logger.py +0 -0
  57. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/output/__init__.py +0 -0
  58. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/output/csv_writer.py +0 -0
  59. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/__init__.py +0 -0
  60. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/_common.py +0 -0
  61. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/go.py +0 -0
  62. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/maven.py +0 -0
  63. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/__init__.py +0 -0
  64. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/attribution_cache.py +0 -0
  65. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/inventory_cache.py +0 -0
  66. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/reachability_cache.py +0 -0
  67. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/dependency_links.txt +0 -0
  68. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/entry_points.txt +0 -0
  69. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/requires.txt +0 -0
  70. {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/top_level.txt +0 -0
  71. {dkatchr-0.3.2 → dkatchr-0.4.0}/setup.cfg +0 -0
  72. {dkatchr-0.3.2 → dkatchr-0.4.0}/tests/test_nvd_client.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dkatchr
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: Multi-ecosystem GitHub dependency scanner with OSV enrichment
5
5
  Classifier: Programming Language :: Python :: 3
6
6
  Classifier: Programming Language :: Python :: 3.10
@@ -60,6 +60,7 @@ Most free SCA tools stop at "does this version have a CVE." dKatchr adds the lay
60
60
  - **Dependency confusion detection** - flags internal-looking package names (path/git/workspace dependencies, declared internal namespace patterns, or cross-repo mismatches) that also resolve on a public registry. On by default.
61
61
  - **Typosquatting detection** - Damerau-Levenshtein distance against per-ecosystem top-package lists, gated by a mandatory registry-metadata check (recent creation / low downloads) to suppress false positives. Opt-in (fetches large feed data).
62
62
  - **Level 1 reachability** - Aho-Corasick grep across a streamed repo tarball for import statements and ecosystem-idiomatic usage patterns (e.g. Bundler autoload conventions, `Rails::`, `JSON.parse`), so you know a flagged package is actually imported somewhere.
63
+ - **Dependency-graph (transitive) reachability** - a vulnerable package your code never imports can still be loaded by one it does (`certifi` via `requests`). With `--reachability`, dKatchr builds each repo's dependency graph from lockfile edges (poetry.lock, Cargo.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Gemfile.lock, composer.lock, packages.lock.json) and, for formats with no edges (requirements.txt incl. `pip freeze` output, Pipfile.lock, go.mod/go.sum), from registry metadata (PyPI wheel METADATA via PEP 658, the Go module proxy). A finding is REACHABLE when an imported package reaches it (the shortest chain lands in the `reachability_path` CSV column, e.g. `certifi <- requests`), UNUSED only when the graph *proves* nothing imported can load it, and UNKNOWN when edges or import checks are missing. Disable with `--no-dependency-graph`.
63
64
  - **Commit attribution ("Introduced By")** - one git-blame call per unique manifest file (GraphQL, cached by branch SHA) resolves each finding to the commit and author that introduced it.
64
65
  - **SHA-keyed caching everywhere** - inventory, reachability, and attribution are all cached per repo branch SHA, so re-scanning an unchanged branch costs nothing.
65
66
  - **Progress streaming** - every long-running phase (tarball download, per-repo scanning, confusion/typosquat lookups) reports structured progress, not silence.
@@ -205,6 +206,10 @@ rm -rf data/cache/osv data/cache/kev data/cache/epss
205
206
 
206
207
  # Force a fresh registry check for confusion/typosquat candidates
207
208
  rm -rf data/cache/registry_meta/
209
+
210
+ # Force fresh dependency metadata for the dependency graph (permanent per
211
+ # package version - published metadata never changes)
212
+ rm -rf data/cache/dependency_meta/
208
213
  ```
209
214
 
210
215
  ## Architecture
@@ -226,8 +231,9 @@ dkatchr/
226
231
 
227
232
  ## Known limitations
228
233
 
229
- - No formal automated test suite yet (contributions welcome - `dkatchr/core/` and `dkatchr/parsers/` are pure functions with no I/O, the natural place to start).
230
- - Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths.
234
+ - The test suite (`tests/`, run with `PYTHONPATH=. pytest tests`) covers the dependency graph, parser edges, the dependency-metadata and NVD clients; most of `dkatchr/core/` and `dkatchr/parsers/` still has none (contributions welcome - they are pure functions with no I/O).
235
+ - Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths. The dependency graph inherits this: a package loaded by NAME rather than imported (Django `INSTALLED_APPS` strings, pytest plugins, gunicorn worker classes, tools run from npm scripts) is invisible, so what it loads can be missed.
236
+ - Dependency graph: Maven/Gradle are not covered (transitive resolution needs parent POMs and version mediation) - their findings keep plain import presence. sdist-only PyPI releases and private Go modules have unknown edges, and a private git/url dependency's import names can't be resolved from a registry - verdicts touching either become UNKNOWN, never UNUSED. Rust build-dependencies / proc-macros are treated as runtime edges. The graph is name-level (all versions of a package share a node) and keeps optional/extra edges - both over-approximate, never hide a path.
231
237
  - Typosquat/confusion registry checks cover npm, PyPI, crates.io, RubyGems, and Packagist. Go, Maven, and NuGet aren't covered yet (different naming/ownership models per ecosystem).
232
238
  - PyPI download counts are unavailable upstream (the JSON API's count field is deprecated), so the typosquat metadata gate falls back to creation-date-only on PyPI.
233
239
 
@@ -38,6 +38,7 @@ Most free SCA tools stop at "does this version have a CVE." dKatchr adds the lay
38
38
  - **Dependency confusion detection** - flags internal-looking package names (path/git/workspace dependencies, declared internal namespace patterns, or cross-repo mismatches) that also resolve on a public registry. On by default.
39
39
  - **Typosquatting detection** - Damerau-Levenshtein distance against per-ecosystem top-package lists, gated by a mandatory registry-metadata check (recent creation / low downloads) to suppress false positives. Opt-in (fetches large feed data).
40
40
  - **Level 1 reachability** - Aho-Corasick grep across a streamed repo tarball for import statements and ecosystem-idiomatic usage patterns (e.g. Bundler autoload conventions, `Rails::`, `JSON.parse`), so you know a flagged package is actually imported somewhere.
41
+ - **Dependency-graph (transitive) reachability** - a vulnerable package your code never imports can still be loaded by one it does (`certifi` via `requests`). With `--reachability`, dKatchr builds each repo's dependency graph from lockfile edges (poetry.lock, Cargo.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Gemfile.lock, composer.lock, packages.lock.json) and, for formats with no edges (requirements.txt incl. `pip freeze` output, Pipfile.lock, go.mod/go.sum), from registry metadata (PyPI wheel METADATA via PEP 658, the Go module proxy). A finding is REACHABLE when an imported package reaches it (the shortest chain lands in the `reachability_path` CSV column, e.g. `certifi <- requests`), UNUSED only when the graph *proves* nothing imported can load it, and UNKNOWN when edges or import checks are missing. Disable with `--no-dependency-graph`.
41
42
  - **Commit attribution ("Introduced By")** - one git-blame call per unique manifest file (GraphQL, cached by branch SHA) resolves each finding to the commit and author that introduced it.
42
43
  - **SHA-keyed caching everywhere** - inventory, reachability, and attribution are all cached per repo branch SHA, so re-scanning an unchanged branch costs nothing.
43
44
  - **Progress streaming** - every long-running phase (tarball download, per-repo scanning, confusion/typosquat lookups) reports structured progress, not silence.
@@ -183,6 +184,10 @@ rm -rf data/cache/osv data/cache/kev data/cache/epss
183
184
 
184
185
  # Force a fresh registry check for confusion/typosquat candidates
185
186
  rm -rf data/cache/registry_meta/
187
+
188
+ # Force fresh dependency metadata for the dependency graph (permanent per
189
+ # package version - published metadata never changes)
190
+ rm -rf data/cache/dependency_meta/
186
191
  ```
187
192
 
188
193
  ## Architecture
@@ -204,8 +209,9 @@ dkatchr/
204
209
 
205
210
  ## Known limitations
206
211
 
207
- - No formal automated test suite yet (contributions welcome - `dkatchr/core/` and `dkatchr/parsers/` are pure functions with no I/O, the natural place to start).
208
- - Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths.
212
+ - The test suite (`tests/`, run with `PYTHONPATH=. pytest tests`) covers the dependency graph, parser edges, the dependency-metadata and NVD clients; most of `dkatchr/core/` and `dkatchr/parsers/` still has none (contributions welcome - they are pure functions with no I/O).
213
+ - Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths. The dependency graph inherits this: a package loaded by NAME rather than imported (Django `INSTALLED_APPS` strings, pytest plugins, gunicorn worker classes, tools run from npm scripts) is invisible, so what it loads can be missed.
214
+ - Dependency graph: Maven/Gradle are not covered (transitive resolution needs parent POMs and version mediation) - their findings keep plain import presence. sdist-only PyPI releases and private Go modules have unknown edges, and a private git/url dependency's import names can't be resolved from a registry - verdicts touching either become UNKNOWN, never UNUSED. Rust build-dependencies / proc-macros are treated as runtime edges. The graph is name-level (all versions of a package share a node) and keeps optional/extra edges - both over-approximate, never hide a path.
209
215
  - Typosquat/confusion registry checks cover npm, PyPI, crates.io, RubyGems, and Packagist. Go, Maven, and NuGet aren't covered yet (different naming/ownership models per ecosystem).
210
216
  - PyPI download counts are unavailable upstream (the JSON API's count field is deprecated), so the typosquat metadata gate falls back to creation-date-only on PyPI.
211
217
 
@@ -1,3 +1,3 @@
1
1
  """dKatchr — multi-ecosystem GitHub dependency scanner with OSV enrichment."""
2
2
 
3
- __version__ = "0.3.2"
3
+ __version__ = "0.4.0"
@@ -74,6 +74,9 @@ def build_arg_parser() -> argparse.ArgumentParser:
74
74
  help="Run Level 1 reachability analysis after vulnerability detection. "
75
75
  "Uses GitHub Code Search API (~1s per unique vulnerable package). "
76
76
  "Requires GITHUB_TOKEN (already required for scanning).")
77
+ p.add_argument("--no-dependency-graph", action="store_true",
78
+ help="With --reachability: skip dependency-graph (transitive) reachability and "
79
+ "use plain import presence only")
77
80
  p.add_argument("--attribution", action="store_true",
78
81
  help="Attribute each finding to the commit author who last touched that "
79
82
  "package's manifest line ('Introduced By'). One git-blame call per "
@@ -32,6 +32,9 @@ from dkatchr.core.attribution import enrich_attribution
32
32
  from dkatchr.core.confusion import detect_confusion, load_internal_patterns
33
33
  from dkatchr.core.typosquat import detect_typosquats
34
34
  from dkatchr.core.osv_enrichment import osv_enrich, osv_rows_for_repo
35
+ from dkatchr.clients.dependency_meta import DependencyMetaClient
36
+ from dkatchr.config import GRAPH_MAX_META_LOOKUPS
37
+ from dkatchr.core.dependency_graph import build_dependency_graph
35
38
  from dkatchr.core.reachability import enrich_reachability
36
39
  from dkatchr.core.rules import compile_rules, load_package_config
37
40
  from dkatchr.core.scanner import Scanner
@@ -218,6 +221,8 @@ def run_cli(args) -> None:
218
221
  log(f"\n===== REACHABILITY ANALYSIS =====")
219
222
  all_repos = set(per_repo_rule_rows) | set(per_repo_osv_rows)
220
223
  reach_cache = ReachabilityCache(cache_dir=args.cache_dir)
224
+ use_graph = not getattr(args, "no_dependency_graph", False)
225
+ dep_meta = DependencyMetaClient(cache_dir=args.cache_dir) if use_graph else None
221
226
  reach_totals: dict[str, int] = {
222
227
  "REACHABLE": 0, "UNUSED": 0, "UNKNOWN": 0, "packages": 0,
223
228
  }
@@ -286,17 +291,47 @@ def run_cli(args) -> None:
286
291
  log(f"[{_repo}] resolution FAILED "
287
292
  f"{payload.get('ecosystem','')}/{payload.get('package','')}: "
288
293
  f"{payload.get('reason','')}")
294
+ elif phase == "graph_probe":
295
+ log(f"[{_repo}] dependency graph: checking imports of "
296
+ f"{payload.get('probe_packages', 0)} package(s) that could load "
297
+ f"a vulnerable one")
298
+ elif phase == "graph_skipped":
299
+ log(f"[{_repo}] dependency graph skipped: "
300
+ f"{payload.get('probe_packages', 0)} packages to probe "
301
+ f"> limit {payload.get('limit')}")
302
+ elif phase == "graph_regrep":
303
+ log(f"[{_repo}] cached scan didn't cover the dependency graph — re-scanning")
289
304
  elif phase == "resolving_done":
290
305
  log(f"[{_repo}] import name resolution: "
291
306
  f"{payload.get('resolved',0)} resolved, "
292
307
  f"{payload.get('cached',0)} cached, "
293
308
  f"{payload.get('failed',0)} failed")
294
309
 
310
+ graph = None
311
+ if use_graph:
312
+ try:
313
+ def _graph_progress(p: dict, _repo=full_name) -> None:
314
+ log(f"[{_repo}] dependency metadata {p.get('current')}/{p.get('total')}")
315
+ graph = build_dependency_graph(
316
+ per_repo_inventory.get(full_name) or [], dep_meta.get_dependencies,
317
+ dep_meta.SUPPORTED_ECOSYSTEMS, GRAPH_MAX_META_LOOKUPS,
318
+ on_progress=_graph_progress)
319
+ log(f"[{full_name}] dependency graph: {len(graph.edges)} package(s), "
320
+ f"{len(graph.known)} with known dependencies"
321
+ + (" (lookup budget hit — partial)" if graph.truncated else ""))
322
+ except Exception as e: # graph failure = plain import presence
323
+ log(f"[!] [{full_name}] dependency graph failed: {e}")
324
+ graph = None
325
+
295
326
  enrich_reachability(
296
327
  all_rows, full_name, sha, github, reach_cache,
297
328
  on_progress=_cli_reach_progress,
298
329
  cache_dir=args.cache_dir,
330
+ dependency_graph=graph,
299
331
  )
332
+ for row in all_rows:
333
+ if isinstance(row.get("reachability_path"), dict):
334
+ row["reachability_path"] = json.dumps(row["reachability_path"])
300
335
 
301
336
  seen_pkgs: set[tuple[str, str]] = set()
302
337
  for row in all_rows:
@@ -793,9 +828,9 @@ def _scan_batch(
793
828
  else:
794
829
  writer.write_rows(rows)
795
830
 
796
- # Inventory feeds the OSV pass, the confusion pass AND the typosquat
797
- # pass — collect it when any is active.
798
- if args.osv or not args.no_confusion or args.typosquat:
831
+ # Inventory feeds the OSV pass, the confusion pass, the typosquat
832
+ # pass AND the dependency graph — collect it when any is active.
833
+ if args.osv or not args.no_confusion or args.typosquat or args.reachability:
799
834
  with inventory_lock:
800
835
  per_repo_inventory[full_name] = result["inventory"]
801
836
 
@@ -0,0 +1,239 @@
1
+ """
2
+ DependencyMetaClient — a package VERSION's direct dependencies, from its
3
+ registry, for manifest formats that record no dependency edges.
4
+
5
+ WHY: dependency-graph reachability needs edges. Lockfiles carry them
6
+ (`depends_on` on inventory rows); requirements.txt, Pipfile.lock and Go's
7
+ go.mod/go.sum do not. This client fills that gap from the authoritative source
8
+ — the registry's own metadata for that exact version — instead of any static
9
+ map ("no static data maps for domain knowledge").
10
+
11
+ Coverage:
12
+ * PyPI — `Requires-Dist` from the release's WHEEL core metadata, fetched via
13
+ PEP 658 ("<wheel-url>.metadata", a few KB) with a fallback to reading
14
+ *.dist-info/METADATA out of the wheel itself. The release JSON's
15
+ `info.requires_dist` is deliberately NOT used: it is None both for a
16
+ package with no dependencies and for one that never published the
17
+ metadata, so it cannot tell "none" from "unknown". A release with no wheel
18
+ (sdist-only) is UNKNOWN. Extra- and marker-gated requirements are KEPT:
19
+ an over-approximated edge can only over-flag, never hide a path.
20
+ * Go — the module proxy's go.mod for that module version (`require` lines,
21
+ // indirect included).
22
+
23
+ Return contract of get_dependencies(): a sorted list of (name, conditional)
24
+ pairs (possibly empty = "known, no dependencies"), or None = UNKNOWN
25
+ (unsupported ecosystem, sdist-only, not found, network failure). Callers must
26
+ never treat None as []. `conditional` marks a requirement that is only
27
+ installed when chosen — a PyPI `extra == …` or environment marker
28
+ (`sys_platform == 'win32'`), and every Go requirement (go.sum lists the whole
29
+ module graph, so a module missing from it is simply not in the build). The
30
+ graph drops a conditional edge whose target is not installed.
31
+
32
+ Caching: permanent per (ecosystem, package, version) at
33
+ {cache_dir}/dependency_meta/ — published metadata is immutable. Definitive
34
+ answers ([] / list / sdist-only unknown) are cached; transient failures are
35
+ NOT, so they retry next scan. Thread-safe; never raises.
36
+ """
37
+
38
+ import hashlib
39
+ import io
40
+ import json
41
+ import os
42
+ import re
43
+ import zipfile
44
+
45
+ import requests
46
+
47
+ from dkatchr.clients.ratelimit import TokenBucket
48
+ from dkatchr.config import (
49
+ DEPENDENCY_META_CACHE_SCHEMA,
50
+ DEPENDENCY_META_RPS,
51
+ DEPENDENCY_META_TIMEOUT,
52
+ DKATCHR_USER_AGENT,
53
+ GO_PROXY_LATEST_URL,
54
+ GO_PROXY_MOD_URL,
55
+ PYPI_PROJECT_JSON_URL,
56
+ PYPI_RELEASE_JSON_URL,
57
+ )
58
+ from dkatchr.logger import log
59
+
60
+ _MAX_WHEEL_BYTES = 50 * 1024 * 1024
61
+ # Requires-Dist: name [extras] (constraints) ; markers -> leading name
62
+ _REQ_NAME_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
63
+
64
+
65
+ def _go_escape(path: str) -> str:
66
+ """Module-proxy case encoding: 'Azure' -> '!azure'."""
67
+ return "".join("!" + c.lower() if c.isupper() else c for c in path)
68
+
69
+
70
+ def parse_requires_dist(metadata_text: str) -> list[tuple[str, bool]]:
71
+ """(name, conditional) from every `Requires-Dist:` header of a core-metadata
72
+ file (pure). conditional = the requirement carries a `;` marker (extra or
73
+ environment). A name listed both ways is unconditional."""
74
+ out: dict[str, bool] = {}
75
+ for line in metadata_text.splitlines():
76
+ if not line.lower().startswith("requires-dist:"):
77
+ continue
78
+ spec = line.split(":", 1)[1]
79
+ m = _REQ_NAME_RE.match(spec)
80
+ if m:
81
+ conditional = ";" in spec
82
+ out[m.group(1)] = out.get(m.group(1), True) and conditional
83
+ return sorted(out.items())
84
+
85
+
86
+ def parse_go_mod_requires(go_mod_text: str) -> list[str]:
87
+ """Module paths from a go.mod's `require` directives (pure)."""
88
+ mods = set()
89
+ for m in re.finditer(r"^require\s+(\S+)\s+v\S+", go_mod_text, re.MULTILINE):
90
+ mods.add(m.group(1))
91
+ for block in re.finditer(r"^require\s*\(([^)]*)\)", go_mod_text, re.MULTILINE | re.DOTALL):
92
+ for line in block.group(1).splitlines():
93
+ line = line.split("//", 1)[0].strip()
94
+ parts = line.split()
95
+ if len(parts) >= 2 and parts[1].startswith("v"):
96
+ mods.add(parts[0])
97
+ return sorted(mods)
98
+
99
+
100
+ class _Unknown(Exception):
101
+ """A definitive 'cannot know' (cacheable), e.g. sdist-only release."""
102
+
103
+
104
+ class DependencyMetaClient:
105
+ SUPPORTED_ECOSYSTEMS = frozenset({"PyPI", "Go"})
106
+
107
+ def __init__(self, cache_dir: str) -> None:
108
+ self.cache_dir = os.path.join(cache_dir, "dependency_meta")
109
+ os.makedirs(self.cache_dir, exist_ok=True)
110
+ self._buckets = {eco: TokenBucket(rate=rps, capacity=max(1.0, rps))
111
+ for eco, rps in DEPENDENCY_META_RPS.items()}
112
+ self._session = requests.Session()
113
+ self._session.headers["User-Agent"] = DKATCHR_USER_AGENT
114
+
115
+ # ---- cache ----------------------------------------------------------
116
+
117
+ def _path(self, eco: str, package: str, version: str) -> str:
118
+ key = hashlib.sha1(f"{eco}\0{package}\0{version}".encode()).hexdigest()
119
+ return os.path.join(self.cache_dir, f"{key}.json")
120
+
121
+ def _read(self, path: str):
122
+ try:
123
+ with open(path, encoding="utf-8") as fh:
124
+ data = json.load(fh)
125
+ except (OSError, ValueError):
126
+ return False, None
127
+ if not isinstance(data, dict) or data.get("schema") != DEPENDENCY_META_CACHE_SCHEMA:
128
+ return False, None
129
+ deps = data.get("deps")
130
+ if not isinstance(deps, list):
131
+ return True, None
132
+ return True, sorted((str(n), bool(c)) for n, c in deps)
133
+
134
+ def _write(self, path: str, deps: list[tuple[str, bool]] | None) -> None:
135
+ try:
136
+ tmp = path + ".tmp"
137
+ with open(tmp, "w", encoding="utf-8") as fh:
138
+ json.dump({"schema": DEPENDENCY_META_CACHE_SCHEMA,
139
+ "deps": None if deps is None else [list(d) for d in deps]}, fh)
140
+ os.replace(tmp, path)
141
+ except OSError as e:
142
+ log(f"[!] dependency-meta cache write {path}: {e}")
143
+
144
+ # ---- http -----------------------------------------------------------
145
+
146
+ def _get(self, eco: str, url: str, stream: bool = False):
147
+ self._buckets[eco].acquire()
148
+ return self._session.get(url, timeout=DEPENDENCY_META_TIMEOUT, stream=stream)
149
+
150
+ # ---- public ---------------------------------------------------------
151
+
152
+ def get_dependencies(self, ecosystem: str, package: str,
153
+ version: str | None) -> list[tuple[str, bool]] | None:
154
+ """version=None -> the registry's CURRENT version is resolved first
155
+ (one uncached request), then answered from the per-version cache."""
156
+ if ecosystem not in self.SUPPORTED_ECOSYSTEMS or not package:
157
+ return None
158
+ if not version:
159
+ try:
160
+ version = self._current_version(ecosystem, package)
161
+ except Exception as e: # noqa: BLE001 — unknown, retried next scan
162
+ log(f"[!] dependency-meta {ecosystem}/{package}: current version lookup failed: {e}")
163
+ return None
164
+ if not version:
165
+ return None
166
+ path = self._path(ecosystem, package, version)
167
+ hit, deps = self._read(path)
168
+ if hit:
169
+ return deps
170
+ try:
171
+ deps = (self._pypi(package, version) if ecosystem == "PyPI"
172
+ else self._go(package, version))
173
+ except _Unknown as e:
174
+ log(f"[dependency-meta] {ecosystem}/{package}@{version}: unknown ({e})")
175
+ self._write(path, None)
176
+ return None
177
+ except Exception as e: # noqa: BLE001 — transient: not cached, retried next scan
178
+ log(f"[!] dependency-meta {ecosystem}/{package}@{version}: {type(e).__name__}: {e}")
179
+ return None
180
+ self._write(path, deps)
181
+ return deps
182
+
183
+ # ---- per ecosystem --------------------------------------------------
184
+
185
+ def _current_version(self, ecosystem: str, package: str) -> str | None:
186
+ if ecosystem == "PyPI":
187
+ resp = self._get("PyPI", PYPI_PROJECT_JSON_URL.format(package=package))
188
+ if resp.status_code == 404:
189
+ return None
190
+ resp.raise_for_status()
191
+ return ((resp.json() or {}).get("info") or {}).get("version")
192
+ resp = self._get("Go", GO_PROXY_LATEST_URL.format(module=_go_escape(package)))
193
+ if resp.status_code in (404, 410):
194
+ return None
195
+ resp.raise_for_status()
196
+ return (resp.json() or {}).get("Version")
197
+
198
+ def _pypi(self, package: str, version: str) -> list[tuple[str, bool]]:
199
+ resp = self._get("PyPI", PYPI_RELEASE_JSON_URL.format(package=package, version=version))
200
+ if resp.status_code == 404:
201
+ raise _Unknown("release not on PyPI")
202
+ resp.raise_for_status()
203
+ files = (resp.json() or {}).get("urls") or []
204
+ wheels = [f for f in files if isinstance(f, dict) and f.get("packagetype") == "bdist_wheel"]
205
+ if not wheels:
206
+ raise _Unknown("no wheel (sdist-only release)")
207
+ # Prefer a pure-Python wheel (same choice the import-name resolver makes).
208
+ wheel = next((w for w in wheels if str(w.get("filename", "")).endswith("none-any.whl")), wheels[0])
209
+ url = wheel.get("url")
210
+ if not url:
211
+ raise _Unknown("wheel has no URL")
212
+ meta = self._get("PyPI", url + ".metadata")
213
+ if meta.status_code == 200:
214
+ return parse_requires_dist(meta.text)
215
+ # No PEP 658 metadata file: read METADATA out of the wheel itself.
216
+ blob = self._download(url)
217
+ with zipfile.ZipFile(io.BytesIO(blob)) as zf:
218
+ name = next((n for n in zf.namelist() if n.endswith(".dist-info/METADATA")), None)
219
+ if name is None:
220
+ raise _Unknown("wheel without dist-info/METADATA")
221
+ return parse_requires_dist(zf.read(name).decode("utf-8", "replace"))
222
+
223
+ def _download(self, url: str) -> bytes:
224
+ with self._get("PyPI", url, stream=True) as resp:
225
+ resp.raise_for_status()
226
+ buf = io.BytesIO()
227
+ for chunk in resp.iter_content(64 * 1024):
228
+ buf.write(chunk)
229
+ if buf.tell() > _MAX_WHEEL_BYTES:
230
+ raise _Unknown("wheel larger than the download guard")
231
+ return buf.getvalue()
232
+
233
+ def _go(self, module: str, version: str) -> list[tuple[str, bool]]:
234
+ url = GO_PROXY_MOD_URL.format(module=_go_escape(module), version=_go_escape(version))
235
+ resp = self._get("Go", url)
236
+ if resp.status_code in (404, 410):
237
+ raise _Unknown("module version not on the proxy (private module?)")
238
+ resp.raise_for_status()
239
+ return [(m, True) for m in parse_go_mod_requires(resp.text)]
@@ -52,7 +52,11 @@ ATTRIBUTION_CACHE_SCHEMA: int = 1
52
52
  # ('@scope/pkg@1.0.0':) because '@' is a reserved indicator, and the v9
53
53
  # regex now strips the quote — every scoped dep in a pnpm v9 lockfile was
54
54
  # previously dropped from inventory (live scan AND historical walk).
55
- CACHE_SCHEMA = 5
55
+ # v6: lockfile parsers attach `depends_on` (dependency edges) for
56
+ # dependency-graph reachability; ALSO fixes pnpm v5/v6 lockfiles, whose
57
+ # 2-space-indented keys never matched (they parsed to zero rows), keys with
58
+ # a peer suffix "(debug@4.3.4)", and truncated git-URL versions.
59
+ CACHE_SCHEMA = 6
56
60
 
57
61
  # Cache directory — anchored to CWD by default. Override via --cache-dir.
58
62
  # Lives under data/cache to match the web layer (web/settings.py CACHE_DIR) and
@@ -426,3 +430,32 @@ REACHABILITY_TEST_FILE_PATTERNS: tuple[str, ...] = (
426
430
  ".test.", ".spec.", # JS/TS: foo.test.ts, foo.spec.ts
427
431
  "conftest", # pytest conftest.py / shared fixtures
428
432
  )
433
+
434
+ # ---- Dependency-graph reachability ------------------------------------------
435
+ # A vulnerable package that your code never imports can still be loaded by a
436
+ # package that IS imported (certifi via requests). The graph comes from
437
+ # lockfile edges (`depends_on` on inventory rows) and, for formats that record
438
+ # none (requirements.txt, Pipfile.lock, go.mod/go.sum), from registry metadata
439
+ # via dkatchr/clients/dependency_meta.py. Pure propagation lives in
440
+ # dkatchr/core/dependency_graph.py.
441
+ #
442
+ # Registry endpoints for package-version dependency metadata (ENDPOINT config,
443
+ # not package knowledge). PyPI: the release JSON lists the version's files; a
444
+ # wheel's core metadata is served on its own at "<wheel-url>.metadata" (PEP 658)
445
+ # — a few KB instead of downloading the wheel. Go: the module proxy serves each
446
+ # module version's go.mod; uppercase path letters are escaped as "!" + lower.
447
+ PYPI_RELEASE_JSON_URL = "https://pypi.org/pypi/{package}/{version}/json"
448
+ PYPI_PROJECT_JSON_URL = "https://pypi.org/pypi/{package}/json" # current version (info.version)
449
+ GO_PROXY_MOD_URL = os.environ.get("GOPROXY_MOD_URL", "https://proxy.golang.org/{module}/@v/{version}.mod")
450
+ GO_PROXY_LATEST_URL = os.environ.get("GOPROXY_LATEST_URL", "https://proxy.golang.org/{module}/@latest")
451
+ DEPENDENCY_META_RPS: dict[str, float] = {"PyPI": 8.0, "Go": 8.0}
452
+ DEPENDENCY_META_TIMEOUT = 20 # seconds per request
453
+ # Permanent cache at {CACHE_DIR}/dependency_meta/ keyed (ecosystem, package,
454
+ # version): a published version's dependency list never changes. Bump the
455
+ # schema to invalidate after a parsing change.
456
+ DEPENDENCY_META_CACHE_SCHEMA = 2 # v2: entries are (name, conditional) pairs
457
+ # Per-repo bounds. More registry lookups than this (edge-less formats only)
458
+ # or more packages to probe for imports than this => the graph is skipped for
459
+ # that repo and reachability falls back to plain import presence.
460
+ GRAPH_MAX_META_LOOKUPS = int(os.environ.get("DKATCHR_GRAPH_MAX_META_LOOKUPS", "800"))
461
+ GRAPH_MAX_PROBE_PACKAGES = int(os.environ.get("DKATCHR_GRAPH_MAX_PROBE_PACKAGES", "600"))