dkatchr 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dkatchr-0.3.2 → dkatchr-0.4.0}/PKG-INFO +9 -3
- {dkatchr-0.3.2 → dkatchr-0.4.0}/README.md +8 -2
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/__init__.py +1 -1
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/args.py +3 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/run.py +38 -3
- dkatchr-0.4.0/dkatchr/clients/dependency_meta.py +239 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/config.py +34 -1
- dkatchr-0.4.0/dkatchr/core/dependency_graph.py +311 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/inventory.py +11 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/reachability.py +106 -11
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/output/schema.py +5 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/cargo.py +13 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/composer.py +15 -1
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/npm.py +142 -5
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/nuget.py +9 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/pypi.py +10 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/ruby.py +16 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/PKG-INFO +9 -3
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/SOURCES.txt +7 -1
- {dkatchr-0.3.2 → dkatchr-0.4.0}/pyproject.toml +1 -1
- dkatchr-0.4.0/tests/test_dependency_graph.py +154 -0
- dkatchr-0.4.0/tests/test_dependency_meta.py +76 -0
- dkatchr-0.4.0/tests/test_parser_edges.py +93 -0
- dkatchr-0.4.0/tests/test_reachability_graph.py +103 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/LICENSE +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/__main__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/__init__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/dry_run.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/cli/main.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/__init__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/epss.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/exploitdb.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/github.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/kev.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/metasploit.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/nvd.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/osv.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/ratelimit.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/registry_meta.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/__init__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/_base.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/crates.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/maven.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/nuget.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/packagist.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/pypi.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/resolvers/rubygems.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/clients/top_packages.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/__init__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/attribution.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/confusion.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/osv_enrichment.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/rules.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/scanner.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/core/typosquat.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/logger.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/output/__init__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/output/csv_writer.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/__init__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/_common.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/go.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/parsers/maven.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/__init__.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/attribution_cache.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/inventory_cache.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr/storage/reachability_cache.py +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/dependency_links.txt +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/entry_points.txt +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/requires.txt +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/dkatchr.egg-info/top_level.txt +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/setup.cfg +0 -0
- {dkatchr-0.3.2 → dkatchr-0.4.0}/tests/test_nvd_client.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dkatchr
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Multi-ecosystem GitHub dependency scanner with OSV enrichment
|
|
5
5
|
Classifier: Programming Language :: Python :: 3
|
|
6
6
|
Classifier: Programming Language :: Python :: 3.10
|
|
@@ -60,6 +60,7 @@ Most free SCA tools stop at "does this version have a CVE." dKatchr adds the lay
|
|
|
60
60
|
- **Dependency confusion detection** - flags internal-looking package names (path/git/workspace dependencies, declared internal namespace patterns, or cross-repo mismatches) that also resolve on a public registry. On by default.
|
|
61
61
|
- **Typosquatting detection** - Damerau-Levenshtein distance against per-ecosystem top-package lists, gated by a mandatory registry-metadata check (recent creation / low downloads) to suppress false positives. Opt-in (fetches large feed data).
|
|
62
62
|
- **Level 1 reachability** - Aho-Corasick grep across a streamed repo tarball for import statements and ecosystem-idiomatic usage patterns (e.g. Bundler autoload conventions, `Rails::`, `JSON.parse`), so you know a flagged package is actually imported somewhere.
|
|
63
|
+
- **Dependency-graph (transitive) reachability** - a vulnerable package your code never imports can still be loaded by one it does (`certifi` via `requests`). With `--reachability`, dKatchr builds each repo's dependency graph from lockfile edges (poetry.lock, Cargo.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Gemfile.lock, composer.lock, packages.lock.json) and, for formats with no edges (requirements.txt incl. `pip freeze` output, Pipfile.lock, go.mod/go.sum), from registry metadata (PyPI wheel METADATA via PEP 658, the Go module proxy). A finding is REACHABLE when an imported package reaches it (the shortest chain lands in the `reachability_path` CSV column, e.g. `certifi <- requests`), UNUSED only when the graph *proves* nothing imported can load it, and UNKNOWN when edges or import checks are missing. Disable with `--no-dependency-graph`.
|
|
63
64
|
- **Commit attribution ("Introduced By")** - one git-blame call per unique manifest file (GraphQL, cached by branch SHA) resolves each finding to the commit and author that introduced it.
|
|
64
65
|
- **SHA-keyed caching everywhere** - inventory, reachability, and attribution are all cached per repo branch SHA, so re-scanning an unchanged branch costs nothing.
|
|
65
66
|
- **Progress streaming** - every long-running phase (tarball download, per-repo scanning, confusion/typosquat lookups) reports structured progress, not silence.
|
|
@@ -205,6 +206,10 @@ rm -rf data/cache/osv data/cache/kev data/cache/epss
|
|
|
205
206
|
|
|
206
207
|
# Force a fresh registry check for confusion/typosquat candidates
|
|
207
208
|
rm -rf data/cache/registry_meta/
|
|
209
|
+
|
|
210
|
+
# Force fresh dependency metadata for the dependency graph (permanent per
|
|
211
|
+
# package version - published metadata never changes)
|
|
212
|
+
rm -rf data/cache/dependency_meta/
|
|
208
213
|
```
|
|
209
214
|
|
|
210
215
|
## Architecture
|
|
@@ -226,8 +231,9 @@ dkatchr/
|
|
|
226
231
|
|
|
227
232
|
## Known limitations
|
|
228
233
|
|
|
229
|
-
-
|
|
230
|
-
- Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths.
|
|
234
|
+
- The test suite (`tests/`, run with `PYTHONPATH=. pytest tests`) covers the dependency graph, parser edges, the dependency-metadata and NVD clients; most of `dkatchr/core/` and `dkatchr/parsers/` still has none (contributions welcome - they are pure functions with no I/O).
|
|
235
|
+
- Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths. The dependency graph inherits this: a package loaded by NAME rather than imported (Django `INSTALLED_APPS` strings, pytest plugins, gunicorn worker classes, tools run from npm scripts) is invisible, so what it loads can be missed.
|
|
236
|
+
- Dependency graph: Maven/Gradle are not covered (transitive resolution needs parent POMs and version mediation) - their findings keep plain import presence. sdist-only PyPI releases and private Go modules have unknown edges, and a private git/url dependency's import names can't be resolved from a registry - verdicts touching either become UNKNOWN, never UNUSED. Rust build-dependencies / proc-macros are treated as runtime edges. The graph is name-level (all versions of a package share a node) and keeps optional/extra edges - both over-approximate, never hide a path.
|
|
231
237
|
- Typosquat/confusion registry checks cover npm, PyPI, crates.io, RubyGems, and Packagist. Go, Maven, and NuGet aren't covered yet (different naming/ownership models per ecosystem).
|
|
232
238
|
- PyPI download counts are unavailable upstream (the JSON API's count field is deprecated), so the typosquat metadata gate falls back to creation-date-only on PyPI.
|
|
233
239
|
|
|
@@ -38,6 +38,7 @@ Most free SCA tools stop at "does this version have a CVE." dKatchr adds the lay
|
|
|
38
38
|
- **Dependency confusion detection** - flags internal-looking package names (path/git/workspace dependencies, declared internal namespace patterns, or cross-repo mismatches) that also resolve on a public registry. On by default.
|
|
39
39
|
- **Typosquatting detection** - Damerau-Levenshtein distance against per-ecosystem top-package lists, gated by a mandatory registry-metadata check (recent creation / low downloads) to suppress false positives. Opt-in (fetches large feed data).
|
|
40
40
|
- **Level 1 reachability** - Aho-Corasick grep across a streamed repo tarball for import statements and ecosystem-idiomatic usage patterns (e.g. Bundler autoload conventions, `Rails::`, `JSON.parse`), so you know a flagged package is actually imported somewhere.
|
|
41
|
+
- **Dependency-graph (transitive) reachability** - a vulnerable package your code never imports can still be loaded by one it does (`certifi` via `requests`). With `--reachability`, dKatchr builds each repo's dependency graph from lockfile edges (poetry.lock, Cargo.lock, package-lock.json, yarn.lock, pnpm-lock.yaml, Gemfile.lock, composer.lock, packages.lock.json) and, for formats with no edges (requirements.txt incl. `pip freeze` output, Pipfile.lock, go.mod/go.sum), from registry metadata (PyPI wheel METADATA via PEP 658, the Go module proxy). A finding is REACHABLE when an imported package reaches it (the shortest chain lands in the `reachability_path` CSV column, e.g. `certifi <- requests`), UNUSED only when the graph *proves* nothing imported can load it, and UNKNOWN when edges or import checks are missing. Disable with `--no-dependency-graph`.
|
|
41
42
|
- **Commit attribution ("Introduced By")** - one git-blame call per unique manifest file (GraphQL, cached by branch SHA) resolves each finding to the commit and author that introduced it.
|
|
42
43
|
- **SHA-keyed caching everywhere** - inventory, reachability, and attribution are all cached per repo branch SHA, so re-scanning an unchanged branch costs nothing.
|
|
43
44
|
- **Progress streaming** - every long-running phase (tarball download, per-repo scanning, confusion/typosquat lookups) reports structured progress, not silence.
|
|
@@ -183,6 +184,10 @@ rm -rf data/cache/osv data/cache/kev data/cache/epss
|
|
|
183
184
|
|
|
184
185
|
# Force a fresh registry check for confusion/typosquat candidates
|
|
185
186
|
rm -rf data/cache/registry_meta/
|
|
187
|
+
|
|
188
|
+
# Force fresh dependency metadata for the dependency graph (permanent per
|
|
189
|
+
# package version - published metadata never changes)
|
|
190
|
+
rm -rf data/cache/dependency_meta/
|
|
186
191
|
```
|
|
187
192
|
|
|
188
193
|
## Architecture
|
|
@@ -204,8 +209,9 @@ dkatchr/
|
|
|
204
209
|
|
|
205
210
|
## Known limitations
|
|
206
211
|
|
|
207
|
-
-
|
|
208
|
-
- Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths.
|
|
212
|
+
- The test suite (`tests/`, run with `PYTHONPATH=. pytest tests`) covers the dependency graph, parser edges, the dependency-metadata and NVD clients; most of `dkatchr/core/` and `dkatchr/parsers/` still has none (contributions welcome - they are pure functions with no I/O).
|
|
213
|
+
- Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths. The dependency graph inherits this: a package loaded by NAME rather than imported (Django `INSTALLED_APPS` strings, pytest plugins, gunicorn worker classes, tools run from npm scripts) is invisible, so what it loads can be missed.
|
|
214
|
+
- Dependency graph: Maven/Gradle are not covered (transitive resolution needs parent POMs and version mediation) - their findings keep plain import presence. sdist-only PyPI releases and private Go modules have unknown edges, and a private git/url dependency's import names can't be resolved from a registry - verdicts touching either become UNKNOWN, never UNUSED. Rust build-dependencies / proc-macros are treated as runtime edges. The graph is name-level (all versions of a package share a node) and keeps optional/extra edges - both over-approximate, never hide a path.
|
|
209
215
|
- Typosquat/confusion registry checks cover npm, PyPI, crates.io, RubyGems, and Packagist. Go, Maven, and NuGet aren't covered yet (different naming/ownership models per ecosystem).
|
|
210
216
|
- PyPI download counts are unavailable upstream (the JSON API's count field is deprecated), so the typosquat metadata gate falls back to creation-date-only on PyPI.
|
|
211
217
|
|
|
@@ -74,6 +74,9 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|
|
74
74
|
help="Run Level 1 reachability analysis after vulnerability detection. "
|
|
75
75
|
"Uses GitHub Code Search API (~1s per unique vulnerable package). "
|
|
76
76
|
"Requires GITHUB_TOKEN (already required for scanning).")
|
|
77
|
+
p.add_argument("--no-dependency-graph", action="store_true",
|
|
78
|
+
help="With --reachability: skip dependency-graph (transitive) reachability and "
|
|
79
|
+
"use plain import presence only")
|
|
77
80
|
p.add_argument("--attribution", action="store_true",
|
|
78
81
|
help="Attribute each finding to the commit author who last touched that "
|
|
79
82
|
"package's manifest line ('Introduced By'). One git-blame call per "
|
|
@@ -32,6 +32,9 @@ from dkatchr.core.attribution import enrich_attribution
|
|
|
32
32
|
from dkatchr.core.confusion import detect_confusion, load_internal_patterns
|
|
33
33
|
from dkatchr.core.typosquat import detect_typosquats
|
|
34
34
|
from dkatchr.core.osv_enrichment import osv_enrich, osv_rows_for_repo
|
|
35
|
+
from dkatchr.clients.dependency_meta import DependencyMetaClient
|
|
36
|
+
from dkatchr.config import GRAPH_MAX_META_LOOKUPS
|
|
37
|
+
from dkatchr.core.dependency_graph import build_dependency_graph
|
|
35
38
|
from dkatchr.core.reachability import enrich_reachability
|
|
36
39
|
from dkatchr.core.rules import compile_rules, load_package_config
|
|
37
40
|
from dkatchr.core.scanner import Scanner
|
|
@@ -218,6 +221,8 @@ def run_cli(args) -> None:
|
|
|
218
221
|
log(f"\n===== REACHABILITY ANALYSIS =====")
|
|
219
222
|
all_repos = set(per_repo_rule_rows) | set(per_repo_osv_rows)
|
|
220
223
|
reach_cache = ReachabilityCache(cache_dir=args.cache_dir)
|
|
224
|
+
use_graph = not getattr(args, "no_dependency_graph", False)
|
|
225
|
+
dep_meta = DependencyMetaClient(cache_dir=args.cache_dir) if use_graph else None
|
|
221
226
|
reach_totals: dict[str, int] = {
|
|
222
227
|
"REACHABLE": 0, "UNUSED": 0, "UNKNOWN": 0, "packages": 0,
|
|
223
228
|
}
|
|
@@ -286,17 +291,47 @@ def run_cli(args) -> None:
|
|
|
286
291
|
log(f"[{_repo}] resolution FAILED "
|
|
287
292
|
f"{payload.get('ecosystem','')}/{payload.get('package','')}: "
|
|
288
293
|
f"{payload.get('reason','')}")
|
|
294
|
+
elif phase == "graph_probe":
|
|
295
|
+
log(f"[{_repo}] dependency graph: checking imports of "
|
|
296
|
+
f"{payload.get('probe_packages', 0)} package(s) that could load "
|
|
297
|
+
f"a vulnerable one")
|
|
298
|
+
elif phase == "graph_skipped":
|
|
299
|
+
log(f"[{_repo}] dependency graph skipped: "
|
|
300
|
+
f"{payload.get('probe_packages', 0)} packages to probe "
|
|
301
|
+
f"> limit {payload.get('limit')}")
|
|
302
|
+
elif phase == "graph_regrep":
|
|
303
|
+
log(f"[{_repo}] cached scan didn't cover the dependency graph — re-scanning")
|
|
289
304
|
elif phase == "resolving_done":
|
|
290
305
|
log(f"[{_repo}] import name resolution: "
|
|
291
306
|
f"{payload.get('resolved',0)} resolved, "
|
|
292
307
|
f"{payload.get('cached',0)} cached, "
|
|
293
308
|
f"{payload.get('failed',0)} failed")
|
|
294
309
|
|
|
310
|
+
graph = None
|
|
311
|
+
if use_graph:
|
|
312
|
+
try:
|
|
313
|
+
def _graph_progress(p: dict, _repo=full_name) -> None:
|
|
314
|
+
log(f"[{_repo}] dependency metadata {p.get('current')}/{p.get('total')}")
|
|
315
|
+
graph = build_dependency_graph(
|
|
316
|
+
per_repo_inventory.get(full_name) or [], dep_meta.get_dependencies,
|
|
317
|
+
dep_meta.SUPPORTED_ECOSYSTEMS, GRAPH_MAX_META_LOOKUPS,
|
|
318
|
+
on_progress=_graph_progress)
|
|
319
|
+
log(f"[{full_name}] dependency graph: {len(graph.edges)} package(s), "
|
|
320
|
+
f"{len(graph.known)} with known dependencies"
|
|
321
|
+
+ (" (lookup budget hit — partial)" if graph.truncated else ""))
|
|
322
|
+
except Exception as e: # graph failure = plain import presence
|
|
323
|
+
log(f"[!] [{full_name}] dependency graph failed: {e}")
|
|
324
|
+
graph = None
|
|
325
|
+
|
|
295
326
|
enrich_reachability(
|
|
296
327
|
all_rows, full_name, sha, github, reach_cache,
|
|
297
328
|
on_progress=_cli_reach_progress,
|
|
298
329
|
cache_dir=args.cache_dir,
|
|
330
|
+
dependency_graph=graph,
|
|
299
331
|
)
|
|
332
|
+
for row in all_rows:
|
|
333
|
+
if isinstance(row.get("reachability_path"), dict):
|
|
334
|
+
row["reachability_path"] = json.dumps(row["reachability_path"])
|
|
300
335
|
|
|
301
336
|
seen_pkgs: set[tuple[str, str]] = set()
|
|
302
337
|
for row in all_rows:
|
|
@@ -793,9 +828,9 @@ def _scan_batch(
|
|
|
793
828
|
else:
|
|
794
829
|
writer.write_rows(rows)
|
|
795
830
|
|
|
796
|
-
# Inventory feeds the OSV pass, the confusion pass
|
|
797
|
-
# pass — collect it when any is active.
|
|
798
|
-
if args.osv or not args.no_confusion or args.typosquat:
|
|
831
|
+
# Inventory feeds the OSV pass, the confusion pass, the typosquat
|
|
832
|
+
# pass AND the dependency graph — collect it when any is active.
|
|
833
|
+
if args.osv or not args.no_confusion or args.typosquat or args.reachability:
|
|
799
834
|
with inventory_lock:
|
|
800
835
|
per_repo_inventory[full_name] = result["inventory"]
|
|
801
836
|
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DependencyMetaClient — a package VERSION's direct dependencies, from its
|
|
3
|
+
registry, for manifest formats that record no dependency edges.
|
|
4
|
+
|
|
5
|
+
WHY: dependency-graph reachability needs edges. Lockfiles carry them
|
|
6
|
+
(`depends_on` on inventory rows); requirements.txt, Pipfile.lock and Go's
|
|
7
|
+
go.mod/go.sum do not. This client fills that gap from the authoritative source
|
|
8
|
+
— the registry's own metadata for that exact version — instead of any static
|
|
9
|
+
map ("no static data maps for domain knowledge").
|
|
10
|
+
|
|
11
|
+
Coverage:
|
|
12
|
+
* PyPI — `Requires-Dist` from the release's WHEEL core metadata, fetched via
|
|
13
|
+
PEP 658 ("<wheel-url>.metadata", a few KB) with a fallback to reading
|
|
14
|
+
*.dist-info/METADATA out of the wheel itself. The release JSON's
|
|
15
|
+
`info.requires_dist` is deliberately NOT used: it is None both for a
|
|
16
|
+
package with no dependencies and for one that never published the
|
|
17
|
+
metadata, so it cannot tell "none" from "unknown". A release with no wheel
|
|
18
|
+
(sdist-only) is UNKNOWN. Extra- and marker-gated requirements are KEPT:
|
|
19
|
+
an over-approximated edge can only over-flag, never hide a path.
|
|
20
|
+
* Go — the module proxy's go.mod for that module version (`require` lines,
|
|
21
|
+
// indirect included).
|
|
22
|
+
|
|
23
|
+
Return contract of get_dependencies(): a sorted list of (name, conditional)
|
|
24
|
+
pairs (possibly empty = "known, no dependencies"), or None = UNKNOWN
|
|
25
|
+
(unsupported ecosystem, sdist-only, not found, network failure). Callers must
|
|
26
|
+
never treat None as []. `conditional` marks a requirement that is only
|
|
27
|
+
installed when chosen — a PyPI `extra == …` or environment marker
|
|
28
|
+
(`sys_platform == 'win32'`), and every Go requirement (go.sum lists the whole
|
|
29
|
+
module graph, so a module missing from it is simply not in the build). The
|
|
30
|
+
graph drops a conditional edge whose target is not installed.
|
|
31
|
+
|
|
32
|
+
Caching: permanent per (ecosystem, package, version) at
|
|
33
|
+
{cache_dir}/dependency_meta/ — published metadata is immutable. Definitive
|
|
34
|
+
answers ([] / list / sdist-only unknown) are cached; transient failures are
|
|
35
|
+
NOT, so they retry next scan. Thread-safe; never raises.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
import hashlib
|
|
39
|
+
import io
|
|
40
|
+
import json
|
|
41
|
+
import os
|
|
42
|
+
import re
|
|
43
|
+
import zipfile
|
|
44
|
+
|
|
45
|
+
import requests
|
|
46
|
+
|
|
47
|
+
from dkatchr.clients.ratelimit import TokenBucket
|
|
48
|
+
from dkatchr.config import (
|
|
49
|
+
DEPENDENCY_META_CACHE_SCHEMA,
|
|
50
|
+
DEPENDENCY_META_RPS,
|
|
51
|
+
DEPENDENCY_META_TIMEOUT,
|
|
52
|
+
DKATCHR_USER_AGENT,
|
|
53
|
+
GO_PROXY_LATEST_URL,
|
|
54
|
+
GO_PROXY_MOD_URL,
|
|
55
|
+
PYPI_PROJECT_JSON_URL,
|
|
56
|
+
PYPI_RELEASE_JSON_URL,
|
|
57
|
+
)
|
|
58
|
+
from dkatchr.logger import log
|
|
59
|
+
|
|
60
|
+
_MAX_WHEEL_BYTES = 50 * 1024 * 1024
|
|
61
|
+
# Requires-Dist: name [extras] (constraints) ; markers -> leading name
|
|
62
|
+
_REQ_NAME_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _go_escape(path: str) -> str:
|
|
66
|
+
"""Module-proxy case encoding: 'Azure' -> '!azure'."""
|
|
67
|
+
return "".join("!" + c.lower() if c.isupper() else c for c in path)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def parse_requires_dist(metadata_text: str) -> list[tuple[str, bool]]:
|
|
71
|
+
"""(name, conditional) from every `Requires-Dist:` header of a core-metadata
|
|
72
|
+
file (pure). conditional = the requirement carries a `;` marker (extra or
|
|
73
|
+
environment). A name listed both ways is unconditional."""
|
|
74
|
+
out: dict[str, bool] = {}
|
|
75
|
+
for line in metadata_text.splitlines():
|
|
76
|
+
if not line.lower().startswith("requires-dist:"):
|
|
77
|
+
continue
|
|
78
|
+
spec = line.split(":", 1)[1]
|
|
79
|
+
m = _REQ_NAME_RE.match(spec)
|
|
80
|
+
if m:
|
|
81
|
+
conditional = ";" in spec
|
|
82
|
+
out[m.group(1)] = out.get(m.group(1), True) and conditional
|
|
83
|
+
return sorted(out.items())
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def parse_go_mod_requires(go_mod_text: str) -> list[str]:
|
|
87
|
+
"""Module paths from a go.mod's `require` directives (pure)."""
|
|
88
|
+
mods = set()
|
|
89
|
+
for m in re.finditer(r"^require\s+(\S+)\s+v\S+", go_mod_text, re.MULTILINE):
|
|
90
|
+
mods.add(m.group(1))
|
|
91
|
+
for block in re.finditer(r"^require\s*\(([^)]*)\)", go_mod_text, re.MULTILINE | re.DOTALL):
|
|
92
|
+
for line in block.group(1).splitlines():
|
|
93
|
+
line = line.split("//", 1)[0].strip()
|
|
94
|
+
parts = line.split()
|
|
95
|
+
if len(parts) >= 2 and parts[1].startswith("v"):
|
|
96
|
+
mods.add(parts[0])
|
|
97
|
+
return sorted(mods)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class _Unknown(Exception):
|
|
101
|
+
"""A definitive 'cannot know' (cacheable), e.g. sdist-only release."""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class DependencyMetaClient:
|
|
105
|
+
SUPPORTED_ECOSYSTEMS = frozenset({"PyPI", "Go"})
|
|
106
|
+
|
|
107
|
+
def __init__(self, cache_dir: str) -> None:
|
|
108
|
+
self.cache_dir = os.path.join(cache_dir, "dependency_meta")
|
|
109
|
+
os.makedirs(self.cache_dir, exist_ok=True)
|
|
110
|
+
self._buckets = {eco: TokenBucket(rate=rps, capacity=max(1.0, rps))
|
|
111
|
+
for eco, rps in DEPENDENCY_META_RPS.items()}
|
|
112
|
+
self._session = requests.Session()
|
|
113
|
+
self._session.headers["User-Agent"] = DKATCHR_USER_AGENT
|
|
114
|
+
|
|
115
|
+
# ---- cache ----------------------------------------------------------
|
|
116
|
+
|
|
117
|
+
def _path(self, eco: str, package: str, version: str) -> str:
|
|
118
|
+
key = hashlib.sha1(f"{eco}\0{package}\0{version}".encode()).hexdigest()
|
|
119
|
+
return os.path.join(self.cache_dir, f"{key}.json")
|
|
120
|
+
|
|
121
|
+
def _read(self, path: str):
|
|
122
|
+
try:
|
|
123
|
+
with open(path, encoding="utf-8") as fh:
|
|
124
|
+
data = json.load(fh)
|
|
125
|
+
except (OSError, ValueError):
|
|
126
|
+
return False, None
|
|
127
|
+
if not isinstance(data, dict) or data.get("schema") != DEPENDENCY_META_CACHE_SCHEMA:
|
|
128
|
+
return False, None
|
|
129
|
+
deps = data.get("deps")
|
|
130
|
+
if not isinstance(deps, list):
|
|
131
|
+
return True, None
|
|
132
|
+
return True, sorted((str(n), bool(c)) for n, c in deps)
|
|
133
|
+
|
|
134
|
+
def _write(self, path: str, deps: list[tuple[str, bool]] | None) -> None:
|
|
135
|
+
try:
|
|
136
|
+
tmp = path + ".tmp"
|
|
137
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
138
|
+
json.dump({"schema": DEPENDENCY_META_CACHE_SCHEMA,
|
|
139
|
+
"deps": None if deps is None else [list(d) for d in deps]}, fh)
|
|
140
|
+
os.replace(tmp, path)
|
|
141
|
+
except OSError as e:
|
|
142
|
+
log(f"[!] dependency-meta cache write {path}: {e}")
|
|
143
|
+
|
|
144
|
+
# ---- http -----------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
def _get(self, eco: str, url: str, stream: bool = False):
|
|
147
|
+
self._buckets[eco].acquire()
|
|
148
|
+
return self._session.get(url, timeout=DEPENDENCY_META_TIMEOUT, stream=stream)
|
|
149
|
+
|
|
150
|
+
# ---- public ---------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
def get_dependencies(self, ecosystem: str, package: str,
|
|
153
|
+
version: str | None) -> list[tuple[str, bool]] | None:
|
|
154
|
+
"""version=None -> the registry's CURRENT version is resolved first
|
|
155
|
+
(one uncached request), then answered from the per-version cache."""
|
|
156
|
+
if ecosystem not in self.SUPPORTED_ECOSYSTEMS or not package:
|
|
157
|
+
return None
|
|
158
|
+
if not version:
|
|
159
|
+
try:
|
|
160
|
+
version = self._current_version(ecosystem, package)
|
|
161
|
+
except Exception as e: # noqa: BLE001 — unknown, retried next scan
|
|
162
|
+
log(f"[!] dependency-meta {ecosystem}/{package}: current version lookup failed: {e}")
|
|
163
|
+
return None
|
|
164
|
+
if not version:
|
|
165
|
+
return None
|
|
166
|
+
path = self._path(ecosystem, package, version)
|
|
167
|
+
hit, deps = self._read(path)
|
|
168
|
+
if hit:
|
|
169
|
+
return deps
|
|
170
|
+
try:
|
|
171
|
+
deps = (self._pypi(package, version) if ecosystem == "PyPI"
|
|
172
|
+
else self._go(package, version))
|
|
173
|
+
except _Unknown as e:
|
|
174
|
+
log(f"[dependency-meta] {ecosystem}/{package}@{version}: unknown ({e})")
|
|
175
|
+
self._write(path, None)
|
|
176
|
+
return None
|
|
177
|
+
except Exception as e: # noqa: BLE001 — transient: not cached, retried next scan
|
|
178
|
+
log(f"[!] dependency-meta {ecosystem}/{package}@{version}: {type(e).__name__}: {e}")
|
|
179
|
+
return None
|
|
180
|
+
self._write(path, deps)
|
|
181
|
+
return deps
|
|
182
|
+
|
|
183
|
+
# ---- per ecosystem --------------------------------------------------
|
|
184
|
+
|
|
185
|
+
def _current_version(self, ecosystem: str, package: str) -> str | None:
|
|
186
|
+
if ecosystem == "PyPI":
|
|
187
|
+
resp = self._get("PyPI", PYPI_PROJECT_JSON_URL.format(package=package))
|
|
188
|
+
if resp.status_code == 404:
|
|
189
|
+
return None
|
|
190
|
+
resp.raise_for_status()
|
|
191
|
+
return ((resp.json() or {}).get("info") or {}).get("version")
|
|
192
|
+
resp = self._get("Go", GO_PROXY_LATEST_URL.format(module=_go_escape(package)))
|
|
193
|
+
if resp.status_code in (404, 410):
|
|
194
|
+
return None
|
|
195
|
+
resp.raise_for_status()
|
|
196
|
+
return (resp.json() or {}).get("Version")
|
|
197
|
+
|
|
198
|
+
def _pypi(self, package: str, version: str) -> list[tuple[str, bool]]:
|
|
199
|
+
resp = self._get("PyPI", PYPI_RELEASE_JSON_URL.format(package=package, version=version))
|
|
200
|
+
if resp.status_code == 404:
|
|
201
|
+
raise _Unknown("release not on PyPI")
|
|
202
|
+
resp.raise_for_status()
|
|
203
|
+
files = (resp.json() or {}).get("urls") or []
|
|
204
|
+
wheels = [f for f in files if isinstance(f, dict) and f.get("packagetype") == "bdist_wheel"]
|
|
205
|
+
if not wheels:
|
|
206
|
+
raise _Unknown("no wheel (sdist-only release)")
|
|
207
|
+
# Prefer a pure-Python wheel (same choice the import-name resolver makes).
|
|
208
|
+
wheel = next((w for w in wheels if str(w.get("filename", "")).endswith("none-any.whl")), wheels[0])
|
|
209
|
+
url = wheel.get("url")
|
|
210
|
+
if not url:
|
|
211
|
+
raise _Unknown("wheel has no URL")
|
|
212
|
+
meta = self._get("PyPI", url + ".metadata")
|
|
213
|
+
if meta.status_code == 200:
|
|
214
|
+
return parse_requires_dist(meta.text)
|
|
215
|
+
# No PEP 658 metadata file: read METADATA out of the wheel itself.
|
|
216
|
+
blob = self._download(url)
|
|
217
|
+
with zipfile.ZipFile(io.BytesIO(blob)) as zf:
|
|
218
|
+
name = next((n for n in zf.namelist() if n.endswith(".dist-info/METADATA")), None)
|
|
219
|
+
if name is None:
|
|
220
|
+
raise _Unknown("wheel without dist-info/METADATA")
|
|
221
|
+
return parse_requires_dist(zf.read(name).decode("utf-8", "replace"))
|
|
222
|
+
|
|
223
|
+
def _download(self, url: str) -> bytes:
|
|
224
|
+
with self._get("PyPI", url, stream=True) as resp:
|
|
225
|
+
resp.raise_for_status()
|
|
226
|
+
buf = io.BytesIO()
|
|
227
|
+
for chunk in resp.iter_content(64 * 1024):
|
|
228
|
+
buf.write(chunk)
|
|
229
|
+
if buf.tell() > _MAX_WHEEL_BYTES:
|
|
230
|
+
raise _Unknown("wheel larger than the download guard")
|
|
231
|
+
return buf.getvalue()
|
|
232
|
+
|
|
233
|
+
def _go(self, module: str, version: str) -> list[tuple[str, bool]]:
|
|
234
|
+
url = GO_PROXY_MOD_URL.format(module=_go_escape(module), version=_go_escape(version))
|
|
235
|
+
resp = self._get("Go", url)
|
|
236
|
+
if resp.status_code in (404, 410):
|
|
237
|
+
raise _Unknown("module version not on the proxy (private module?)")
|
|
238
|
+
resp.raise_for_status()
|
|
239
|
+
return [(m, True) for m in parse_go_mod_requires(resp.text)]
|
|
@@ -52,7 +52,11 @@ ATTRIBUTION_CACHE_SCHEMA: int = 1
|
|
|
52
52
|
# ('@scope/pkg@1.0.0':) because '@' is a reserved indicator, and the v9
|
|
53
53
|
# regex now strips the quote — every scoped dep in a pnpm v9 lockfile was
|
|
54
54
|
# previously dropped from inventory (live scan AND historical walk).
|
|
55
|
-
|
|
55
|
+
# v6: lockfile parsers attach `depends_on` (dependency edges) for
|
|
56
|
+
# dependency-graph reachability; ALSO fixes pnpm v5/v6 lockfiles, whose
|
|
57
|
+
# 2-space-indented keys never matched (they parsed to zero rows), keys with
|
|
58
|
+
# a peer suffix "(debug@4.3.4)", and truncated git-URL versions.
|
|
59
|
+
CACHE_SCHEMA = 6
|
|
56
60
|
|
|
57
61
|
# Cache directory — anchored to CWD by default. Override via --cache-dir.
|
|
58
62
|
# Lives under data/cache to match the web layer (web/settings.py CACHE_DIR) and
|
|
@@ -426,3 +430,32 @@ REACHABILITY_TEST_FILE_PATTERNS: tuple[str, ...] = (
|
|
|
426
430
|
".test.", ".spec.", # JS/TS: foo.test.ts, foo.spec.ts
|
|
427
431
|
"conftest", # pytest conftest.py / shared fixtures
|
|
428
432
|
)
|
|
433
|
+
|
|
434
|
+
# ---- Dependency-graph reachability ------------------------------------------
|
|
435
|
+
# A vulnerable package that your code never imports can still be loaded by a
|
|
436
|
+
# package that IS imported (certifi via requests). The graph comes from
|
|
437
|
+
# lockfile edges (`depends_on` on inventory rows) and, for formats that record
|
|
438
|
+
# none (requirements.txt, Pipfile.lock, go.mod/go.sum), from registry metadata
|
|
439
|
+
# via dkatchr/clients/dependency_meta.py. Pure propagation lives in
|
|
440
|
+
# dkatchr/core/dependency_graph.py.
|
|
441
|
+
#
|
|
442
|
+
# Registry endpoints for package-version dependency metadata (ENDPOINT config,
|
|
443
|
+
# not package knowledge). PyPI: the release JSON lists the version's files; a
|
|
444
|
+
# wheel's core metadata is served on its own at "<wheel-url>.metadata" (PEP 658)
|
|
445
|
+
# — a few KB instead of downloading the wheel. Go: the module proxy serves each
|
|
446
|
+
# module version's go.mod; uppercase path letters are escaped as "!" + lower.
|
|
447
|
+
PYPI_RELEASE_JSON_URL = "https://pypi.org/pypi/{package}/{version}/json"
|
|
448
|
+
PYPI_PROJECT_JSON_URL = "https://pypi.org/pypi/{package}/json" # current version (info.version)
|
|
449
|
+
GO_PROXY_MOD_URL = os.environ.get("GOPROXY_MOD_URL", "https://proxy.golang.org/{module}/@v/{version}.mod")
|
|
450
|
+
GO_PROXY_LATEST_URL = os.environ.get("GOPROXY_LATEST_URL", "https://proxy.golang.org/{module}/@latest")
|
|
451
|
+
DEPENDENCY_META_RPS: dict[str, float] = {"PyPI": 8.0, "Go": 8.0}
|
|
452
|
+
DEPENDENCY_META_TIMEOUT = 20 # seconds per request
|
|
453
|
+
# Permanent cache at {CACHE_DIR}/dependency_meta/ keyed (ecosystem, package,
|
|
454
|
+
# version): a published version's dependency list never changes. Bump the
|
|
455
|
+
# schema to invalidate after a parsing change.
|
|
456
|
+
DEPENDENCY_META_CACHE_SCHEMA = 2 # v2: entries are (name, conditional) pairs
|
|
457
|
+
# Per-repo bounds. More registry lookups than this (edge-less formats only)
|
|
458
|
+
# or more packages to probe for imports than this => the graph is skipped for
|
|
459
|
+
# that repo and reachability falls back to plain import presence.
|
|
460
|
+
GRAPH_MAX_META_LOOKUPS = int(os.environ.get("DKATCHR_GRAPH_MAX_META_LOOKUPS", "800"))
|
|
461
|
+
GRAPH_MAX_PROBE_PACKAGES = int(os.environ.get("DKATCHR_GRAPH_MAX_PROBE_PACKAGES", "600"))
|