dkatchr 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dkatchr-0.3.0/LICENSE +21 -0
- dkatchr-0.3.0/PKG-INFO +228 -0
- dkatchr-0.3.0/README.md +212 -0
- dkatchr-0.3.0/dkatchr/__init__.py +3 -0
- dkatchr-0.3.0/dkatchr/__main__.py +6 -0
- dkatchr-0.3.0/dkatchr/cli/__init__.py +7 -0
- dkatchr-0.3.0/dkatchr/cli/args.py +84 -0
- dkatchr-0.3.0/dkatchr/cli/dry_run.py +68 -0
- dkatchr-0.3.0/dkatchr/cli/main.py +14 -0
- dkatchr-0.3.0/dkatchr/cli/run.py +814 -0
- dkatchr-0.3.0/dkatchr/clients/__init__.py +3 -0
- dkatchr-0.3.0/dkatchr/clients/epss.py +153 -0
- dkatchr-0.3.0/dkatchr/clients/exploitdb.py +155 -0
- dkatchr-0.3.0/dkatchr/clients/github.py +482 -0
- dkatchr-0.3.0/dkatchr/clients/kev.py +143 -0
- dkatchr-0.3.0/dkatchr/clients/metasploit.py +147 -0
- dkatchr-0.3.0/dkatchr/clients/nvd.py +121 -0
- dkatchr-0.3.0/dkatchr/clients/osv.py +191 -0
- dkatchr-0.3.0/dkatchr/clients/ratelimit.py +33 -0
- dkatchr-0.3.0/dkatchr/clients/registry_meta.py +280 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/__init__.py +335 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/_base.py +153 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/crates.py +51 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/maven.py +144 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/nuget.py +216 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/packagist.py +164 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/pypi.py +190 -0
- dkatchr-0.3.0/dkatchr/clients/resolvers/rubygems.py +132 -0
- dkatchr-0.3.0/dkatchr/clients/top_packages.py +302 -0
- dkatchr-0.3.0/dkatchr/config.py +428 -0
- dkatchr-0.3.0/dkatchr/core/__init__.py +3 -0
- dkatchr-0.3.0/dkatchr/core/attribution.py +464 -0
- dkatchr-0.3.0/dkatchr/core/confusion.py +334 -0
- dkatchr-0.3.0/dkatchr/core/inventory.py +104 -0
- dkatchr-0.3.0/dkatchr/core/osv_enrichment.py +358 -0
- dkatchr-0.3.0/dkatchr/core/reachability.py +793 -0
- dkatchr-0.3.0/dkatchr/core/rules.py +132 -0
- dkatchr-0.3.0/dkatchr/core/scanner.py +170 -0
- dkatchr-0.3.0/dkatchr/core/typosquat.py +294 -0
- dkatchr-0.3.0/dkatchr/logger.py +121 -0
- dkatchr-0.3.0/dkatchr/output/__init__.py +2 -0
- dkatchr-0.3.0/dkatchr/output/csv_writer.py +37 -0
- dkatchr-0.3.0/dkatchr/output/schema.py +83 -0
- dkatchr-0.3.0/dkatchr/parsers/__init__.py +74 -0
- dkatchr-0.3.0/dkatchr/parsers/_common.py +22 -0
- dkatchr-0.3.0/dkatchr/parsers/cargo.py +49 -0
- dkatchr-0.3.0/dkatchr/parsers/composer.py +86 -0
- dkatchr-0.3.0/dkatchr/parsers/go.py +42 -0
- dkatchr-0.3.0/dkatchr/parsers/maven.py +71 -0
- dkatchr-0.3.0/dkatchr/parsers/npm.py +278 -0
- dkatchr-0.3.0/dkatchr/parsers/nuget.py +71 -0
- dkatchr-0.3.0/dkatchr/parsers/pypi.py +214 -0
- dkatchr-0.3.0/dkatchr/parsers/ruby.py +119 -0
- dkatchr-0.3.0/dkatchr/storage/__init__.py +2 -0
- dkatchr-0.3.0/dkatchr/storage/attribution_cache.py +101 -0
- dkatchr-0.3.0/dkatchr/storage/inventory_cache.py +84 -0
- dkatchr-0.3.0/dkatchr/storage/reachability_cache.py +108 -0
- dkatchr-0.3.0/dkatchr.egg-info/PKG-INFO +228 -0
- dkatchr-0.3.0/dkatchr.egg-info/SOURCES.txt +63 -0
- dkatchr-0.3.0/dkatchr.egg-info/dependency_links.txt +1 -0
- dkatchr-0.3.0/dkatchr.egg-info/entry_points.txt +2 -0
- dkatchr-0.3.0/dkatchr.egg-info/requires.txt +7 -0
- dkatchr-0.3.0/dkatchr.egg-info/top_level.txt +1 -0
- dkatchr-0.3.0/pyproject.toml +25 -0
- dkatchr-0.3.0/setup.cfg +4 -0
dkatchr-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Trim Kadriu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dkatchr-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dkatchr
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Multi-ecosystem GitHub dependency scanner with OSV enrichment
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Dist: requests>=2.28.0
|
|
9
|
+
Requires-Dist: tqdm>=4.65.0
|
|
10
|
+
Requires-Dist: cvss>=3.0
|
|
11
|
+
Requires-Dist: pyahocorasick>=2.0.0
|
|
12
|
+
Requires-Dist: rapidfuzz>=3.0.0
|
|
13
|
+
Requires-Dist: dotnetfile>=0.2.0
|
|
14
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
<p align="center">
|
|
18
|
+
<picture>
|
|
19
|
+
<source media="(prefers-color-scheme: dark)" srcset="assets/logo-dark.png">
|
|
20
|
+
<source media="(prefers-color-scheme: light)" srcset="assets/logo-light.png">
|
|
21
|
+
<img alt="dKatchr" src="assets/logo-light.png" width="360">
|
|
22
|
+
</picture>
|
|
23
|
+
</p>
|
|
24
|
+
|
|
25
|
+
# dKatchr
|
|
26
|
+
|
|
27
|
+
Multi-ecosystem Software Composition Analysis (SCA) scanner for GitHub. Point it at one or more GitHub orgs (or a specific list of repos), and it discovers every dependency manifest across 8 package ecosystems, enriches findings against multiple vulnerability intelligence sources, and flags supply-chain risks that plain CVE scanning misses - dependency confusion, typosquatting, and unauthenticated internal-name exposure.
|
|
28
|
+
|
|
29
|
+
No CI pipeline changes, no agents, no per-repo setup. Connect a GitHub token, point at an org, get a CSV.
|
|
30
|
+
|
|
31
|
+
## Why this exists
|
|
32
|
+
|
|
33
|
+
Most free SCA tools stop at "does this version have a CVE." dKatchr adds the layers that actually decide whether you should care:
|
|
34
|
+
|
|
35
|
+
- **Exploitability, not just severity** - CVSS alone tells you how bad a bug *could* be. EPSS tells you the probability it's *actually* being exploited. CISA KEV tells you it already has been. dKatchr surfaces all three per finding, plus cross-references against ExploitDB and Metasploit for known public exploit code.
|
|
36
|
+
- **Supply-chain attack surface, not just known CVEs** - dependency confusion (an internal package name that also exists, unclaimed, on the public registry) and typosquatting (a close-name lookalike of a popular package) are both live attack vectors with no CVE assigned. dKatchr detects both.
|
|
37
|
+
- **Reachability** - a vulnerable package sitting unused in a lockfile is a different risk than one your code actually imports. Level 1 reachability checks import-presence across the source tree before you spend time triaging.
|
|
38
|
+
- **Attribution** - every finding can be traced back to the commit and author who introduced that dependency line, so "who do I ask about this" isn't a separate investigation.
|
|
39
|
+
|
|
40
|
+
## Features
|
|
41
|
+
|
|
42
|
+
- **8 package ecosystems**: npm, PyPI, Maven (incl. Gradle lockfiles), RubyGems, Go, NuGet, crates.io, Packagist (PHP/Composer) - see the full manifest table below.
|
|
43
|
+
- **OSV.dev enrichment** - batch CVE/advisory lookups against Google's open-source vulnerability database, with a two-tier cache to keep re-scans fast.
|
|
44
|
+
- **CISA KEV overlay** - flags findings with a documented history of active exploitation in the wild.
|
|
45
|
+
- **EPSS scoring** - attaches the daily-updated exploit-probability percentage from FIRST.org to every CVE-bearing finding.
|
|
46
|
+
- **Exploit intelligence** - cross-references ExploitDB and Metasploit module coverage per CVE, surfaced as a single "Has Exploit" signal.
|
|
47
|
+
- **Custom rules engine** - define your own package/version bans (exact match or regex) independent of OSV, with an optional reason code (`INTERNAL_VULNERABLE`, `ORG_BANNED`, `LICENSE_ISSUE`, `PRE_CVE`) for internal tracking.
|
|
48
|
+
- **Dependency confusion detection** - flags internal-looking package names (path/git/workspace dependencies, declared internal namespace patterns, or cross-repo mismatches) that also resolve on a public registry. On by default.
|
|
49
|
+
- **Typosquatting detection** - Damerau-Levenshtein distance against per-ecosystem top-package lists, gated by a mandatory registry-metadata check (recent creation / low downloads) to suppress false positives. Opt-in (fetches large feed data).
|
|
50
|
+
- **Level 1 reachability** - Aho-Corasick grep across a streamed repo tarball for import statements and ecosystem-idiomatic usage patterns (e.g. Bundler autoload conventions, `Rails::`, `JSON.parse`), so you know a flagged package is actually imported somewhere.
|
|
51
|
+
- **Commit attribution ("Introduced By")** - one git-blame call per unique manifest file (GraphQL, cached by branch SHA) resolves each finding to the commit and author that introduced it.
|
|
52
|
+
- **SHA-keyed caching everywhere** - inventory, reachability, and attribution are all cached per repo branch SHA, so re-scanning an unchanged branch costs nothing.
|
|
53
|
+
- **Progress streaming** - every long-running phase (tarball download, per-repo scanning, confusion/typosquat lookups) reports structured progress, not silence.
|
|
54
|
+
- **Parallel, fault-isolated scanning** - one malformed manifest or one dead repo never aborts the run; it's logged, marked, and the scan continues.
|
|
55
|
+
|
|
56
|
+
## Supported manifests
|
|
57
|
+
|
|
58
|
+
| Ecosystem | Files |
|
|
59
|
+
|---|---|
|
|
60
|
+
| npm | `package.json`, `package-lock.json`, `yarn.lock`, `pnpm-lock.yaml` |
|
|
61
|
+
| PyPI | `requirements.txt`, `Pipfile.lock`, `poetry.lock` |
|
|
62
|
+
| Maven | `pom.xml`, `gradle.lockfile` |
|
|
63
|
+
| RubyGems | `Gemfile`, `Gemfile.lock` |
|
|
64
|
+
| Go | `go.mod`, `go.sum` |
|
|
65
|
+
| NuGet | `packages.config`, `packages.lock.json`, `*.csproj`, `*.fsproj`, `*.vbproj` |
|
|
66
|
+
| crates.io | `Cargo.lock` |
|
|
67
|
+
| Packagist (PHP) | `composer.json`, `composer.lock` |
|
|
68
|
+
|
|
69
|
+
## Install
|
|
70
|
+
|
|
71
|
+
Requires Python 3.10+.
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install dkatchr
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Or, if you prefer CLI tools installed in their own isolated environment (recommended, avoids clashing with whatever's in your project virtualenvs):
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
pipx install dkatchr
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### Install from source (latest, unreleased changes)
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install git+https://github.com/dkatchr/dkatchr-core.git
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Or clone and install locally for development:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
git clone https://github.com/dkatchr/dkatchr-core.git
|
|
93
|
+
cd dkatchr-core
|
|
94
|
+
python3 -m venv venv && source venv/bin/activate
|
|
95
|
+
pip install -e .
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
`pyahocorasick` and `rapidfuzz` are C-extension dependencies. PyPI ships manylinux wheels for common platforms, so most installs get a prebuilt binary - if pip builds from source, you'll need a C toolchain (`gcc`/`clang`).
|
|
99
|
+
|
|
100
|
+
## Quick start
|
|
101
|
+
|
|
102
|
+
1. Create a GitHub Personal Access Token with read access to the repos/orgs you want to scan.
|
|
103
|
+
2. Copy the example env file and fill it in:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
cp dkatchr/.env.example .env
|
|
107
|
+
# edit .env, set GITHUB_TOKEN=ghp_...
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
3. Run a scan:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
dkatchr -o results.csv --orgs my-github-org --osv
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
(`python -m dkatchr` works identically if you prefer that form.)
|
|
117
|
+
|
|
118
|
+
That scans every repo in `my-github-org`, extracts all dependency manifests, enriches every finding against OSV + KEV + EPSS + exploit intel, and writes the results to `results.csv`. Dependency confusion detection runs by default; nothing else extra needs to be turned on to get a meaningful first scan.
|
|
119
|
+
|
|
120
|
+
### Common variations
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
# Scan specific repos instead of a whole org
|
|
124
|
+
python -m dkatchr -o results.csv --repos my-org/repo-one my-org/repo-two --osv
|
|
125
|
+
|
|
126
|
+
# See what would be scanned (repo count, cache hit/miss estimate) without scanning
|
|
127
|
+
python -m dkatchr -o results.csv --orgs my-github-org --dry-run
|
|
128
|
+
|
|
129
|
+
# Add reachability + commit attribution
|
|
130
|
+
python -m dkatchr -o results.csv --orgs my-github-org --osv --reachability --attribution
|
|
131
|
+
|
|
132
|
+
# Supply your own internal namespace patterns for confusion detection
|
|
133
|
+
python -m dkatchr -o results.csv --orgs my-github-org --internal-patterns patterns.json
|
|
134
|
+
|
|
135
|
+
# Turn on typosquat detection (off by default - fetches per-ecosystem top-package feeds)
|
|
136
|
+
python -m dkatchr -o results.csv --orgs my-github-org --typosquat
|
|
137
|
+
|
|
138
|
+
# Use custom package/version rules instead of (or alongside) OSV
|
|
139
|
+
python -m dkatchr -o results.csv --orgs my-github-org -c rules.json
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Run `python -m dkatchr --help` for the full flag reference - every flag is self-documented, including defaults and rate-limit behavior.
|
|
143
|
+
|
|
144
|
+
### Custom rules format
|
|
145
|
+
|
|
146
|
+
```json
|
|
147
|
+
{
|
|
148
|
+
"lodash_rule": {
|
|
149
|
+
"description": "Track lodash, flag versions below 4.17.21",
|
|
150
|
+
"reason": "INTERNAL_VULNERABLE",
|
|
151
|
+
"npm_names": ["lodash"],
|
|
152
|
+
"vulnerable_versions": ["4.17.20", "4.17.19"],
|
|
153
|
+
"vulnerable_version_patterns": ["^[123]\\."]
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Supported name fields: `npm_names`, `pypi_names`, `maven_names`, `rubygems_names`, `go_names`, `nuget_names`, `cargo_names`, `composer_names`.
|
|
159
|
+
|
|
160
|
+
### Internal namespace patterns (for confusion detection)
|
|
161
|
+
|
|
162
|
+
```json
|
|
163
|
+
[
|
|
164
|
+
{ "ecosystem": "npm", "pattern": "@my-org/*" },
|
|
165
|
+
{ "ecosystem": "PyPI", "pattern": "myorg-*" }
|
|
166
|
+
]
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Configuration
|
|
170
|
+
|
|
171
|
+
All tunables (rate limits, cache TTLs, worker counts, reachability limits) are environment variables - see `dkatchr/.env.example` for the full list with defaults and explanations. Nothing is hardcoded that you'd need to patch the source to change.
|
|
172
|
+
|
|
173
|
+
Key ones to know:
|
|
174
|
+
|
|
175
|
+
| Variable | Purpose |
|
|
176
|
+
|---|---|
|
|
177
|
+
| `GITHUB_TOKEN` | Required for anything beyond a handful of unauthenticated requests. |
|
|
178
|
+
| `DKATCHR_GITHUB_RPS` / `DKATCHR_GITHUB_BURST` | GitHub API rate cap (default tuned for the 5,000 req/hr authenticated budget). |
|
|
179
|
+
| `DKATCHR_OSV_RPS` / `DKATCHR_OSV_BURST` | OSV.dev API rate cap. |
|
|
180
|
+
| `DKATCHR_WORKERS` | Per-repo parallelism. |
|
|
181
|
+
| `DKATCHR_OSV_TTL` / `DKATCHR_KEV_TTL` | Cache freshness windows for the respective data sources. |
|
|
182
|
+
|
|
183
|
+
## Caching
|
|
184
|
+
|
|
185
|
+
Every external data source (OSV, KEV, EPSS, ExploitDB, Metasploit, registry metadata, top-package feeds, per-repo inventory) is cached to disk under `--cache-dir` (default `./data/cache`). If you suspect stale data or change parser logic locally:
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
# Force a full re-scan of everything
|
|
189
|
+
rm -rf data/cache/
|
|
190
|
+
|
|
191
|
+
# Force just a fresh CVE dataset
|
|
192
|
+
rm -rf data/cache/osv data/cache/kev data/cache/epss
|
|
193
|
+
|
|
194
|
+
# Force a fresh registry check for confusion/typosquat candidates
|
|
195
|
+
rm -rf data/cache/registry_meta/
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
## Architecture
|
|
199
|
+
|
|
200
|
+
```
|
|
201
|
+
dkatchr/
|
|
202
|
+
├── cli/ # argparse glue + orchestration (ThreadPoolExecutor for per-repo parallelism)
|
|
203
|
+
├── clients/ # one client per external system: GitHub, OSV, KEV, EPSS, ExploitDB,
|
|
204
|
+
│ # Metasploit, registry metadata, top-package feeds - each owns its own
|
|
205
|
+
│ # rate limiter and cache
|
|
206
|
+
├── core/ # pure logic, no I/O: scanning orchestration, rule matching, confusion
|
|
207
|
+
│ # detection, typosquat detection, reachability grep, attribution resolution
|
|
208
|
+
├── parsers/ # one module per ecosystem, each returning a plain list of dependency dicts
|
|
209
|
+
├── output/ # CSV schema + writer
|
|
210
|
+
└── storage/ # per-repo SHA-keyed JSON caches (inventory, reachability, attribution)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
`core/` has zero knowledge of argparse, CSV, or threading - it's the part safe to reuse if you want to embed dKatchr's detection logic in something other than the bundled CLI.
|
|
214
|
+
|
|
215
|
+
## Known limitations
|
|
216
|
+
|
|
217
|
+
- No formal automated test suite yet (contributions welcome - `dkatchr/core/` and `dkatchr/parsers/` are pure functions with no I/O, the natural place to start).
|
|
218
|
+
- Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths.
|
|
219
|
+
- Typosquat/confusion registry checks cover npm, PyPI, crates.io, RubyGems, and Packagist. Go, Maven, and NuGet aren't covered yet (different naming/ownership models per ecosystem).
|
|
220
|
+
- PyPI download counts are unavailable upstream (the JSON API's count field is deprecated), so the typosquat metadata gate falls back to creation-date-only on PyPI.
|
|
221
|
+
|
|
222
|
+
## Contributing
|
|
223
|
+
|
|
224
|
+
Issues and PRs welcome. If you're adding a new ecosystem, see the docstring at the top of `dkatchr/parsers/__init__.py` for the four-step registration process. If you're touching `core/`, keep it pure - no argparse, no direct file I/O beyond what's passed in, no threading assumptions; parallelism belongs in `cli/`.
|
|
225
|
+
|
|
226
|
+
## License
|
|
227
|
+
|
|
228
|
+
MIT - see [LICENSE](LICENSE).
|
dkatchr-0.3.0/README.md
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="assets/logo-dark.png">
|
|
4
|
+
<source media="(prefers-color-scheme: light)" srcset="assets/logo-light.png">
|
|
5
|
+
<img alt="dKatchr" src="assets/logo-light.png" width="360">
|
|
6
|
+
</picture>
|
|
7
|
+
</p>
|
|
8
|
+
|
|
9
|
+
# dKatchr
|
|
10
|
+
|
|
11
|
+
Multi-ecosystem Software Composition Analysis (SCA) scanner for GitHub. Point it at one or more GitHub orgs (or a specific list of repos), and it discovers every dependency manifest across 8 package ecosystems, enriches findings against multiple vulnerability intelligence sources, and flags supply-chain risks that plain CVE scanning misses - dependency confusion, typosquatting, and unauthenticated internal-name exposure.
|
|
12
|
+
|
|
13
|
+
No CI pipeline changes, no agents, no per-repo setup. Connect a GitHub token, point at an org, get a CSV.
|
|
14
|
+
|
|
15
|
+
## Why this exists
|
|
16
|
+
|
|
17
|
+
Most free SCA tools stop at "does this version have a CVE." dKatchr adds the layers that actually decide whether you should care:
|
|
18
|
+
|
|
19
|
+
- **Exploitability, not just severity** - CVSS alone tells you how bad a bug *could* be. EPSS tells you the probability it's *actually* being exploited. CISA KEV tells you it already has been. dKatchr surfaces all three per finding, plus cross-references against ExploitDB and Metasploit for known public exploit code.
|
|
20
|
+
- **Supply-chain attack surface, not just known CVEs** - dependency confusion (an internal package name that also exists, unclaimed, on the public registry) and typosquatting (a close-name lookalike of a popular package) are both live attack vectors with no CVE assigned. dKatchr detects both.
|
|
21
|
+
- **Reachability** - a vulnerable package sitting unused in a lockfile is a different risk than one your code actually imports. Level 1 reachability checks import-presence across the source tree before you spend time triaging.
|
|
22
|
+
- **Attribution** - every finding can be traced back to the commit and author who introduced that dependency line, so "who do I ask about this" isn't a separate investigation.
|
|
23
|
+
|
|
24
|
+
## Features
|
|
25
|
+
|
|
26
|
+
- **8 package ecosystems**: npm, PyPI, Maven (incl. Gradle lockfiles), RubyGems, Go, NuGet, crates.io, Packagist (PHP/Composer) - see the full manifest table below.
|
|
27
|
+
- **OSV.dev enrichment** - batch CVE/advisory lookups against Google's open-source vulnerability database, with a two-tier cache to keep re-scans fast.
|
|
28
|
+
- **CISA KEV overlay** - flags findings with a documented history of active exploitation in the wild.
|
|
29
|
+
- **EPSS scoring** - attaches the daily-updated exploit-probability percentage from FIRST.org to every CVE-bearing finding.
|
|
30
|
+
- **Exploit intelligence** - cross-references ExploitDB and Metasploit module coverage per CVE, surfaced as a single "Has Exploit" signal.
|
|
31
|
+
- **Custom rules engine** - define your own package/version bans (exact match or regex) independent of OSV, with an optional reason code (`INTERNAL_VULNERABLE`, `ORG_BANNED`, `LICENSE_ISSUE`, `PRE_CVE`) for internal tracking.
|
|
32
|
+
- **Dependency confusion detection** - flags internal-looking package names (path/git/workspace dependencies, declared internal namespace patterns, or cross-repo mismatches) that also resolve on a public registry. On by default.
|
|
33
|
+
- **Typosquatting detection** - Damerau-Levenshtein distance against per-ecosystem top-package lists, gated by a mandatory registry-metadata check (recent creation / low downloads) to suppress false positives. Opt-in (fetches large feed data).
|
|
34
|
+
- **Level 1 reachability** - Aho-Corasick grep across a streamed repo tarball for import statements and ecosystem-idiomatic usage patterns (e.g. Bundler autoload conventions, `Rails::`, `JSON.parse`), so you know a flagged package is actually imported somewhere.
|
|
35
|
+
- **Commit attribution ("Introduced By")** - one git-blame call per unique manifest file (GraphQL, cached by branch SHA) resolves each finding to the commit and author that introduced it.
|
|
36
|
+
- **SHA-keyed caching everywhere** - inventory, reachability, and attribution are all cached per repo branch SHA, so re-scanning an unchanged branch costs nothing.
|
|
37
|
+
- **Progress streaming** - every long-running phase (tarball download, per-repo scanning, confusion/typosquat lookups) reports structured progress, not silence.
|
|
38
|
+
- **Parallel, fault-isolated scanning** - one malformed manifest or one dead repo never aborts the run; it's logged, marked, and the scan continues.
|
|
39
|
+
|
|
40
|
+
## Supported manifests
|
|
41
|
+
|
|
42
|
+
| Ecosystem | Files |
|
|
43
|
+
|---|---|
|
|
44
|
+
| npm | `package.json`, `package-lock.json`, `yarn.lock`, `pnpm-lock.yaml` |
|
|
45
|
+
| PyPI | `requirements.txt`, `Pipfile.lock`, `poetry.lock` |
|
|
46
|
+
| Maven | `pom.xml`, `gradle.lockfile` |
|
|
47
|
+
| RubyGems | `Gemfile`, `Gemfile.lock` |
|
|
48
|
+
| Go | `go.mod`, `go.sum` |
|
|
49
|
+
| NuGet | `packages.config`, `packages.lock.json`, `*.csproj`, `*.fsproj`, `*.vbproj` |
|
|
50
|
+
| crates.io | `Cargo.lock` |
|
|
51
|
+
| Packagist (PHP) | `composer.json`, `composer.lock` |
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
Requires Python 3.10+.
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install dkatchr
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Or, if you prefer CLI tools installed in their own isolated environment (recommended, avoids clashing with whatever's in your project virtualenvs):
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pipx install dkatchr
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### Install from source (latest, unreleased changes)
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install git+https://github.com/dkatchr/dkatchr-core.git
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Or clone and install locally for development:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
git clone https://github.com/dkatchr/dkatchr-core.git
|
|
77
|
+
cd dkatchr-core
|
|
78
|
+
python3 -m venv venv && source venv/bin/activate
|
|
79
|
+
pip install -e .
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`pyahocorasick` and `rapidfuzz` are C-extension dependencies. PyPI ships manylinux wheels for common platforms, so most installs get a prebuilt binary - if pip builds from source, you'll need a C toolchain (`gcc`/`clang`).
|
|
83
|
+
|
|
84
|
+
## Quick start
|
|
85
|
+
|
|
86
|
+
1. Create a GitHub Personal Access Token with read access to the repos/orgs you want to scan.
|
|
87
|
+
2. Copy the example env file and fill it in:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
cp dkatchr/.env.example .env
|
|
91
|
+
# edit .env, set GITHUB_TOKEN=ghp_...
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
3. Run a scan:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
dkatchr -o results.csv --orgs my-github-org --osv
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
(`python -m dkatchr` works identically if you prefer that form.)
|
|
101
|
+
|
|
102
|
+
That scans every repo in `my-github-org`, extracts all dependency manifests, enriches every finding against OSV + KEV + EPSS + exploit intel, and writes the results to `results.csv`. Dependency confusion detection runs by default; nothing else extra needs to be turned on to get a meaningful first scan.
|
|
103
|
+
|
|
104
|
+
### Common variations
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
# Scan specific repos instead of a whole org
|
|
108
|
+
python -m dkatchr -o results.csv --repos my-org/repo-one my-org/repo-two --osv
|
|
109
|
+
|
|
110
|
+
# See what would be scanned (repo count, cache hit/miss estimate) without scanning
|
|
111
|
+
python -m dkatchr -o results.csv --orgs my-github-org --dry-run
|
|
112
|
+
|
|
113
|
+
# Add reachability + commit attribution
|
|
114
|
+
python -m dkatchr -o results.csv --orgs my-github-org --osv --reachability --attribution
|
|
115
|
+
|
|
116
|
+
# Supply your own internal namespace patterns for confusion detection
|
|
117
|
+
python -m dkatchr -o results.csv --orgs my-github-org --internal-patterns patterns.json
|
|
118
|
+
|
|
119
|
+
# Turn on typosquat detection (off by default - fetches per-ecosystem top-package feeds)
|
|
120
|
+
python -m dkatchr -o results.csv --orgs my-github-org --typosquat
|
|
121
|
+
|
|
122
|
+
# Use custom package/version rules instead of (or alongside) OSV
|
|
123
|
+
python -m dkatchr -o results.csv --orgs my-github-org -c rules.json
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Run `python -m dkatchr --help` for the full flag reference - every flag is self-documented, including defaults and rate-limit behavior.
|
|
127
|
+
|
|
128
|
+
### Custom rules format
|
|
129
|
+
|
|
130
|
+
```json
|
|
131
|
+
{
|
|
132
|
+
"lodash_rule": {
|
|
133
|
+
"description": "Track lodash, flag versions below 4.17.21",
|
|
134
|
+
"reason": "INTERNAL_VULNERABLE",
|
|
135
|
+
"npm_names": ["lodash"],
|
|
136
|
+
"vulnerable_versions": ["4.17.20", "4.17.19"],
|
|
137
|
+
"vulnerable_version_patterns": ["^[123]\\."]
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Supported name fields: `npm_names`, `pypi_names`, `maven_names`, `rubygems_names`, `go_names`, `nuget_names`, `cargo_names`, `composer_names`.
|
|
143
|
+
|
|
144
|
+
### Internal namespace patterns (for confusion detection)
|
|
145
|
+
|
|
146
|
+
```json
|
|
147
|
+
[
|
|
148
|
+
{ "ecosystem": "npm", "pattern": "@my-org/*" },
|
|
149
|
+
{ "ecosystem": "PyPI", "pattern": "myorg-*" }
|
|
150
|
+
]
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Configuration
|
|
154
|
+
|
|
155
|
+
All tunables (rate limits, cache TTLs, worker counts, reachability limits) are environment variables - see `dkatchr/.env.example` for the full list with defaults and explanations. Nothing is hardcoded that you'd need to patch the source to change.
|
|
156
|
+
|
|
157
|
+
Key ones to know:
|
|
158
|
+
|
|
159
|
+
| Variable | Purpose |
|
|
160
|
+
|---|---|
|
|
161
|
+
| `GITHUB_TOKEN` | Required for anything beyond a handful of unauthenticated requests. |
|
|
162
|
+
| `DKATCHR_GITHUB_RPS` / `DKATCHR_GITHUB_BURST` | GitHub API rate cap (default tuned for the 5,000 req/hr authenticated budget). |
|
|
163
|
+
| `DKATCHR_OSV_RPS` / `DKATCHR_OSV_BURST` | OSV.dev API rate cap. |
|
|
164
|
+
| `DKATCHR_WORKERS` | Per-repo parallelism. |
|
|
165
|
+
| `DKATCHR_OSV_TTL` / `DKATCHR_KEV_TTL` | Cache freshness windows for the respective data sources. |
|
|
166
|
+
|
|
167
|
+
## Caching
|
|
168
|
+
|
|
169
|
+
Every external data source (OSV, KEV, EPSS, ExploitDB, Metasploit, registry metadata, top-package feeds, per-repo inventory) is cached to disk under `--cache-dir` (default `./data/cache`). If you suspect stale data or change parser logic locally:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
# Force a full re-scan of everything
|
|
173
|
+
rm -rf data/cache/
|
|
174
|
+
|
|
175
|
+
# Force just a fresh CVE dataset
|
|
176
|
+
rm -rf data/cache/osv data/cache/kev data/cache/epss
|
|
177
|
+
|
|
178
|
+
# Force a fresh registry check for confusion/typosquat candidates
|
|
179
|
+
rm -rf data/cache/registry_meta/
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
## Architecture
|
|
183
|
+
|
|
184
|
+
```
|
|
185
|
+
dkatchr/
|
|
186
|
+
├── cli/ # argparse glue + orchestration (ThreadPoolExecutor for per-repo parallelism)
|
|
187
|
+
├── clients/ # one client per external system: GitHub, OSV, KEV, EPSS, ExploitDB,
|
|
188
|
+
│ # Metasploit, registry metadata, top-package feeds - each owns its own
|
|
189
|
+
│ # rate limiter and cache
|
|
190
|
+
├── core/ # pure logic, no I/O: scanning orchestration, rule matching, confusion
|
|
191
|
+
│ # detection, typosquat detection, reachability grep, attribution resolution
|
|
192
|
+
├── parsers/ # one module per ecosystem, each returning a plain list of dependency dicts
|
|
193
|
+
├── output/ # CSV schema + writer
|
|
194
|
+
└── storage/ # per-repo SHA-keyed JSON caches (inventory, reachability, attribution)
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
`core/` has zero knowledge of argparse, CSV, or threading - it's the part safe to reuse if you want to embed dKatchr's detection logic in something other than the bundled CLI.
|
|
198
|
+
|
|
199
|
+
## Known limitations
|
|
200
|
+
|
|
201
|
+
- No formal automated test suite yet (contributions welcome - `dkatchr/core/` and `dkatchr/parsers/` are pure functions with no I/O, the natural place to start).
|
|
202
|
+
- Reachability is import-presence + usage-pattern detection, not full call-graph analysis - it won't catch reflection-based or dynamically-constructed import paths.
|
|
203
|
+
- Typosquat/confusion registry checks cover npm, PyPI, crates.io, RubyGems, and Packagist. Go, Maven, and NuGet aren't covered yet (different naming/ownership models per ecosystem).
|
|
204
|
+
- PyPI download counts are unavailable upstream (the JSON API's count field is deprecated), so the typosquat metadata gate falls back to creation-date-only on PyPI.
|
|
205
|
+
|
|
206
|
+
## Contributing
|
|
207
|
+
|
|
208
|
+
Issues and PRs welcome. If you're adding a new ecosystem, see the docstring at the top of `dkatchr/parsers/__init__.py` for the four-step registration process. If you're touching `core/`, keep it pure - no argparse, no direct file I/O beyond what's passed in, no threading assumptions; parallelism belongs in `cli/`.
|
|
209
|
+
|
|
210
|
+
## License
|
|
211
|
+
|
|
212
|
+
MIT - see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""CLI surface — argparse, run orchestration, dry-run reporting. This is the
|
|
2
|
+
ONLY package allowed to touch sys.argv, sys.exit, and the CSV file path.
|
|
3
|
+
Everything underneath is library code the web app reuses unchanged."""
|
|
4
|
+
|
|
5
|
+
from dkatchr.cli.main import main # re-export for `python -m dkatchr` and shims
|
|
6
|
+
|
|
7
|
+
__all__ = ["main"]
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""
|
|
2
|
+
argparse builder for the CLI.
|
|
3
|
+
|
|
4
|
+
EXTRACTED FROM: dkatchr/cli.py (build_arg_parser).
|
|
5
|
+
WHY: argparse is pure input parsing — keep it out of orchestration so adding
|
|
6
|
+
a flag doesn't risk breaking the runner. Single responsibility.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
|
|
11
|
+
from dkatchr.config import (
|
|
12
|
+
DEFAULT_CACHE_DIR,
|
|
13
|
+
DEFAULT_GITHUB_BURST,
|
|
14
|
+
DEFAULT_GITHUB_RPS,
|
|
15
|
+
DEFAULT_OSV_BURST,
|
|
16
|
+
DEFAULT_OSV_RPS,
|
|
17
|
+
DEFAULT_SUMMARY_EVERY,
|
|
18
|
+
DEFAULT_WORKERS,
|
|
19
|
+
EXPLOITDB_DEFAULT_TTL,
|
|
20
|
+
KEV_DEFAULT_TTL,
|
|
21
|
+
OSV_DEFAULT_TTL,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def build_arg_parser() -> argparse.ArgumentParser:
|
|
26
|
+
p = argparse.ArgumentParser(
|
|
27
|
+
prog="dkatchr",
|
|
28
|
+
description="Multi-ecosystem GitHub dependency scanner with caching + OSV CVE enrichment",
|
|
29
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
30
|
+
)
|
|
31
|
+
p.add_argument("-o", "--output", required=True, help="Output CSV file path")
|
|
32
|
+
p.add_argument("-c", "--config", help="JSON file with package rules (overrides inline DEFAULT_PACKAGE_CONFIG)")
|
|
33
|
+
p.add_argument("--cache-dir", default=DEFAULT_CACHE_DIR, help=f"Cache directory (default: {DEFAULT_CACHE_DIR})")
|
|
34
|
+
p.add_argument("--full-rescan", action="store_true", help="Ignore cache, re-scan all repos")
|
|
35
|
+
p.add_argument("--orgs", nargs="+", help="GitHub org names to scan (default: auto-discover via /user/orgs)")
|
|
36
|
+
p.add_argument("--repos", nargs="+", help="Specific repos to scan as owner/name (skips org listing entirely)")
|
|
37
|
+
p.add_argument("--workers", type=int, default=DEFAULT_WORKERS, help=f"Worker threads for per-repo parallelism (default: {DEFAULT_WORKERS})")
|
|
38
|
+
p.add_argument("--rps", type=float, default=DEFAULT_GITHUB_RPS, help=f"Global GitHub API rate cap, requests/sec (default: {DEFAULT_GITHUB_RPS} — sustainable vs 5000/hr budget)")
|
|
39
|
+
p.add_argument("--burst", type=float, default=DEFAULT_GITHUB_BURST, help=f"Token-bucket burst capacity (default: {DEFAULT_GITHUB_BURST})")
|
|
40
|
+
p.add_argument("--osv", action="store_true", help="Enrich findings with CVE data from OSV.dev (Google's open-source vuln DB)")
|
|
41
|
+
p.add_argument("--osv-ttl", type=int, default=OSV_DEFAULT_TTL, help=f"OSV (eco,pkg,ver)→IDs cache TTL in seconds (default: {OSV_DEFAULT_TTL} = 24h)")
|
|
42
|
+
p.add_argument("--osv-rps", type=float, default=DEFAULT_OSV_RPS, help=f"OSV API rate cap, requests/sec (default: {DEFAULT_OSV_RPS})")
|
|
43
|
+
p.add_argument("--osv-burst", type=float, default=DEFAULT_OSV_BURST, help=f"OSV token-bucket burst capacity (default: {DEFAULT_OSV_BURST})")
|
|
44
|
+
p.add_argument("--no-kev", action="store_true", help="Disable CISA KEV cross-reference (on by default when --osv is used)")
|
|
45
|
+
p.add_argument("--kev-ttl", type=int, default=KEV_DEFAULT_TTL, help=f"KEV catalog cache TTL in seconds (default: {KEV_DEFAULT_TTL} = 24h)")
|
|
46
|
+
p.add_argument("--no-exploit-intel", action="store_true",
|
|
47
|
+
help="Disable the public-exploit cross-reference (ExploitDB + Metasploit; "
|
|
48
|
+
"on by default when --osv is used). Drives the has_exploitdb/has_metasploit "
|
|
49
|
+
"CSV columns and the 'Has Exploit' summary.")
|
|
50
|
+
p.add_argument("--no-confusion", action="store_true",
|
|
51
|
+
help="Disable the dependency-confusion detection pass (ON by default). "
|
|
52
|
+
"Checks internal-looking package names (path/git/workspace deps, "
|
|
53
|
+
"internal namespace patterns, cross-repo mismatches) against the "
|
|
54
|
+
"public registries and writes source=CONFUSION rows.")
|
|
55
|
+
p.add_argument("--internal-patterns", default=None,
|
|
56
|
+
help="JSON file of internal namespace patterns for the confusion pass: "
|
|
57
|
+
"[{\"ecosystem\": \"npm\", \"pattern\": \"@acme/*\"}, ...]. "
|
|
58
|
+
"Optional — without it, only the manifest-internal (S1) and "
|
|
59
|
+
"cross-repo (S3) signals run.")
|
|
60
|
+
p.add_argument("--typosquat", action="store_true",
|
|
61
|
+
help="Enable typosquatting detection (OFF by default — this is a "
|
|
62
|
+
"POSITIVE opt-in flag, unlike --no-confusion). Measures edit "
|
|
63
|
+
"distance of each installed registry dependency against the "
|
|
64
|
+
"ecosystem's top public packages and, on a close match, confirms "
|
|
65
|
+
"via a registry-metadata gate (recently created OR few downloads) "
|
|
66
|
+
"before writing source=TYPOSQUAT rows. Coverage: "
|
|
67
|
+
"npm/PyPI/crates.io/RubyGems/Packagist.")
|
|
68
|
+
p.add_argument("--exploit-ttl", type=int, default=EXPLOITDB_DEFAULT_TTL,
|
|
69
|
+
help=f"ExploitDB + Metasploit catalog cache TTL in seconds (default: {EXPLOITDB_DEFAULT_TTL} = 24h)")
|
|
70
|
+
p.add_argument("--no-progress", action="store_true", help="Disable tqdm progress bar (forces verbose per-repo logging)")
|
|
71
|
+
p.add_argument("--summary-every", type=int, default=DEFAULT_SUMMARY_EVERY, help=f"In no-bar mode, print a running summary every N completed repos (default: {DEFAULT_SUMMARY_EVERY})")
|
|
72
|
+
p.add_argument("--dry-run", action="store_true", help="List repos that would be scanned + estimate cost/wall time, then exit")
|
|
73
|
+
p.add_argument("--reachability", action="store_true",
|
|
74
|
+
help="Run Level 1 reachability analysis after vulnerability detection. "
|
|
75
|
+
"Uses GitHub Code Search API (~1s per unique vulnerable package). "
|
|
76
|
+
"Requires GITHUB_TOKEN (already required for scanning).")
|
|
77
|
+
p.add_argument("--attribution", action="store_true",
|
|
78
|
+
help="Attribute each finding to the commit author who last touched that "
|
|
79
|
+
"package's manifest line ('Introduced By'). One git-blame call per "
|
|
80
|
+
"unique manifest file (cached by branch SHA). Requires GITHUB_TOKEN.")
|
|
81
|
+
p.add_argument("--log-file", default=None,
|
|
82
|
+
help="Also write all stderr log lines to this file (in addition to the terminal). "
|
|
83
|
+
"Useful for post-mortem inspection of a long scan.")
|
|
84
|
+
return p
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Dry-run report — estimate API cost + wall time without making any scan calls.
|
|
3
|
+
|
|
4
|
+
EXTRACTED FROM: dkatchr/cli.py (_dry_run_report).
|
|
5
|
+
WHY: It's a self-contained reporting feature. Pulling it out keeps the
|
|
6
|
+
orchestration runner small and lets us evolve the estimate model without
|
|
7
|
+
touching the scan path.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from dkatchr.logger import log
|
|
11
|
+
from dkatchr.storage.inventory_cache import InventoryCache
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _repo_uses_cache(cache: InventoryCache, full_name: str, full_rescan: bool) -> bool:
|
|
15
|
+
"""Cheap check — only consults the index, doesn't hit GitHub."""
|
|
16
|
+
if full_rescan:
|
|
17
|
+
return False
|
|
18
|
+
return bool(cache.get_index_sha(full_name))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def print_dry_run_report(batches, cache: InventoryCache, args) -> None:
|
|
22
|
+
"""Estimate cost + wall time without making any scan API calls."""
|
|
23
|
+
all_repos = [r for _, batch in batches for r in batch]
|
|
24
|
+
n_total = len(all_repos)
|
|
25
|
+
n_archived = sum(1 for r in all_repos if r.get("archived"))
|
|
26
|
+
n_no_branch = sum(1 for r in all_repos if not r.get("default_branch"))
|
|
27
|
+
n_active = n_total - n_archived - n_no_branch
|
|
28
|
+
n_warm = sum(1 for r in all_repos if _repo_uses_cache(cache, r["full_name"], args.full_rescan))
|
|
29
|
+
n_cold = n_active - n_warm
|
|
30
|
+
|
|
31
|
+
cold_cost = n_cold * 5
|
|
32
|
+
warm_cost = n_warm * 1
|
|
33
|
+
total_cost = cold_cost + warm_cost
|
|
34
|
+
wall_seconds = total_cost / max(args.rps, 0.001)
|
|
35
|
+
budget_pct = total_cost / 5000 * 100
|
|
36
|
+
|
|
37
|
+
log(f"\n===== DRY RUN =====")
|
|
38
|
+
log(f" Repos discovered: {n_total}")
|
|
39
|
+
log(f" archived (skipped): {n_archived}")
|
|
40
|
+
log(f" no default branch: {n_no_branch}")
|
|
41
|
+
log(f" active to scan: {n_active}")
|
|
42
|
+
log(f" warm cache hits: {n_warm}")
|
|
43
|
+
log(f" cold (full scan): {n_cold}")
|
|
44
|
+
log(f"")
|
|
45
|
+
log(f" Estimated GitHub API calls: ~{total_cost} ({cold_cost} cold + {warm_cost} warm)")
|
|
46
|
+
log(f" Hourly budget usage: ~{budget_pct:.1f}% of 5000 req/hr")
|
|
47
|
+
log(f" Wall time at {args.rps:.2f} req/s: ~{wall_seconds/60:.1f} min ({wall_seconds/3600:.2f} h)")
|
|
48
|
+
if budget_pct > 100:
|
|
49
|
+
extra_hours = (budget_pct - 100) / 100 + 1
|
|
50
|
+
log(f" [!] Over hourly budget — will pause for rate-limit reset(s); add ~{extra_hours:.1f}h")
|
|
51
|
+
if args.osv:
|
|
52
|
+
log(f"")
|
|
53
|
+
log(f" OSV enrichment: enabled (separate budget, ~1-5 min added regardless of repo count)")
|
|
54
|
+
|
|
55
|
+
sample = all_repos[:10]
|
|
56
|
+
if sample:
|
|
57
|
+
log(f"")
|
|
58
|
+
log(f" First {len(sample)} repos:")
|
|
59
|
+
for r in sample:
|
|
60
|
+
tag = (
|
|
61
|
+
"[archived]" if r.get("archived")
|
|
62
|
+
else "[no-branch]" if not r.get("default_branch")
|
|
63
|
+
else "[cached]" if _repo_uses_cache(cache, r["full_name"], args.full_rescan)
|
|
64
|
+
else "[cold]"
|
|
65
|
+
)
|
|
66
|
+
log(f" {tag:<12} {r['full_name']}")
|
|
67
|
+
if n_total > len(sample):
|
|
68
|
+
log(f" ... and {n_total - len(sample)} more")
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CLI entry point — argparse + delegate to runner.
|
|
3
|
+
|
|
4
|
+
REPLACES: the old dkatchr/cli.py (316 lines, mixed concerns).
|
|
5
|
+
The new file is tiny on purpose — it only parses args and calls run_cli.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dkatchr.cli.args import build_arg_parser
|
|
9
|
+
from dkatchr.cli.run import run_cli
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def main() -> None:
|
|
13
|
+
args = build_arg_parser().parse_args()
|
|
14
|
+
run_cli(args)
|