ghostpkg 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/CHANGELOG.md +52 -1
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/PKG-INFO +28 -7
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/README.en.md +27 -6
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/README.md +15 -5
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/ghostpkg/__init__.py +1 -1
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/ghostpkg/assess.py +53 -12
- ghostpkg-0.4.0/ghostpkg/cache.py +165 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/ghostpkg/cli.py +30 -4
- ghostpkg-0.4.0/ghostpkg/data.py +820 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/pyproject.toml +1 -1
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/tests/test_assess.py +63 -0
- ghostpkg-0.4.0/tests/test_cache.py +158 -0
- ghostpkg-0.2.0/ghostpkg/data.py +0 -399
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/.github/ISSUE_TEMPLATE/bug.yml +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/.github/ISSUE_TEMPLATE/false-positive.yml +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/.github/ISSUE_TEMPLATE/missed-package.yml +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/.github/workflows/ci.yml +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/.gitignore +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/CONTRIBUTING.md +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/LICENSE +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/SECURITY.md +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/assets/banner.html +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/assets/banner.png +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/assets/demo.gif +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/assets/make_demo.py +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/ghostpkg/__main__.py +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/ghostpkg/manifests.py +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/ghostpkg/registries.py +0 -0
- {ghostpkg-0.2.0 → ghostpkg-0.4.0}/tests/test_manifests.py +0 -0
|
@@ -6,6 +6,55 @@ this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.4.0] - 2026-09-01
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
- **On-disk cache for registry lookups.** Scanning a 150-package manifest drops
|
|
13
|
+
from 4.7s to 0.4s on a warm cache. Previously every scan cost one request per
|
|
14
|
+
dependency, every run, which is slow in CI and rude to the registry.
|
|
15
|
+
- `--no-cache` to bypass it, and `ghostpkg clear-cache` to delete it.
|
|
16
|
+
- `GHOSTPKG_CACHE_DIR` to override the location. The default is
|
|
17
|
+
`%LOCALAPPDATA%\ghostpkg` on Windows, `~/Library/Caches/ghostpkg` on macOS and
|
|
18
|
+
`$XDG_CACHE_HOME/ghostpkg` elsewhere, worked out without a dependency.
|
|
19
|
+
|
|
20
|
+
### Notes on the cache design
|
|
21
|
+
- **Time-to-live depends on the answer, and "does not exist" is held for only an
|
|
22
|
+
hour.** A free name can be registered at any moment — that is the whole attack
|
|
23
|
+
— so a negative result must not be trusted for long. Young packages are held
|
|
24
|
+
six hours, established ones a day.
|
|
25
|
+
- Cache failures are never fatal. A corrupt file, a wrong schema, a malformed
|
|
26
|
+
entry or an unwritable directory all degrade to no cache rather than breaking
|
|
27
|
+
a run. There are tests for each of those.
|
|
28
|
+
- Written atomically via a temporary file and `os.replace`, once per run, so a
|
|
29
|
+
killed process cannot leave a half-written cache behind.
|
|
30
|
+
|
|
31
|
+
## [0.3.0] - 2026-09-01
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
- **npm typo detection did not work at all.** Lookalike names were compared
|
|
35
|
+
against the 2,000 most-downloaded *PyPI* projects regardless of ecosystem, so
|
|
36
|
+
`expresss` was never flagged as a typo of `express` -- `express` was not in
|
|
37
|
+
the list being compared against.
|
|
38
|
+
|
|
39
|
+
### Added
|
|
40
|
+
- A list of the 2,000 most-downloaded npm packages, built from the registry's
|
|
41
|
+
own download-count API. `nearest_popular()` now picks the list matching the
|
|
42
|
+
ecosystem.
|
|
43
|
+
- Scoped npm names are compared on the part after the slash, since that is what
|
|
44
|
+
a squat targets: `@evil/expresss` is flagged, `@types/node` and `@babel/core`
|
|
45
|
+
are not.
|
|
46
|
+
|
|
47
|
+
### Changed
|
|
48
|
+
- Typo distance is now Damerau-Levenshtein: swapping two adjacent characters
|
|
49
|
+
counts as one edit rather than two. Transposition is the commonest typosquat
|
|
50
|
+
shape, and under plain Levenshtein `recat`, `lodahs` and `webpakc` all scored
|
|
51
|
+
two edits, which put them outside the budget for names that short. All three
|
|
52
|
+
are caught now, and the false-positive rate across both 2,000-name lists is
|
|
53
|
+
still zero -- there are tests asserting exactly that.
|
|
54
|
+
- Names shorter than five characters are no longer compared at all. Below that
|
|
55
|
+
the name space is too dense for edit distance to mean anything: `core` sits
|
|
56
|
+
one edit from `cors`.
|
|
57
|
+
|
|
9
58
|
## [0.2.0] - 2026-09-01
|
|
10
59
|
|
|
11
60
|
### Fixed
|
|
@@ -60,6 +109,8 @@ First release.
|
|
|
60
109
|
- No corpus of hallucinated package names is shipped, following the decision of
|
|
61
110
|
the USENIX'25 authors not to publish theirs.
|
|
62
111
|
|
|
63
|
-
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.
|
|
112
|
+
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.4.0...HEAD
|
|
113
|
+
[0.4.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.4.0
|
|
114
|
+
[0.3.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.3.0
|
|
64
115
|
[0.2.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.2.0
|
|
65
116
|
[0.1.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ghostpkg
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Catch package names that do not exist before you install them
|
|
5
5
|
Project-URL: Homepage, https://github.com/m1rwana12/ghostpkg
|
|
6
6
|
Project-URL: Issues, https://github.com/m1rwana12/ghostpkg/issues
|
|
@@ -180,6 +180,7 @@ and report TOML keys as package names.
|
|
|
180
180
|
| `--strict` | Promote warnings to blocks |
|
|
181
181
|
| `--json` | Machine-readable output for scripts and CI |
|
|
182
182
|
| `-q`, `--quiet` | Hide packages that passed |
|
|
183
|
+
| `--no-cache` | Neither read nor write the cache |
|
|
183
184
|
| `--version` | Print the version |
|
|
184
185
|
|
|
185
186
|
### Exit codes
|
|
@@ -190,6 +191,25 @@ and report TOML keys as package names.
|
|
|
190
191
|
| `1` | At least one package blocked |
|
|
191
192
|
| `2` | Usage error, unreadable manifest, or the registry was unreachable |
|
|
192
193
|
|
|
194
|
+
### Caching
|
|
195
|
+
|
|
196
|
+
Lookups are cached on disk, so re-scanning a 150-package manifest takes 0.4s
|
|
197
|
+
instead of 4.7s.
|
|
198
|
+
|
|
199
|
+
Time-to-live depends on the answer, and the negative case is the one that
|
|
200
|
+
matters: **"does not exist" is held for only an hour**, because a free name can
|
|
201
|
+
be registered at any moment and that is the entire attack. Young packages are
|
|
202
|
+
held six hours, established ones a day.
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
ghostpkg scan requirements.txt --no-cache # bypass it
|
|
206
|
+
ghostpkg clear-cache # delete it
|
|
207
|
+
GHOSTPKG_CACHE_DIR=/tmp/gp ghostpkg check x # move it
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
A corrupt, unreadable or unwritable cache degrades to no cache rather than
|
|
211
|
+
breaking your run.
|
|
212
|
+
|
|
193
213
|
### JSON output
|
|
194
214
|
|
|
195
215
|
```console
|
|
@@ -217,7 +237,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
|
|
|
217
237
|
| First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
|
|
218
238
|
| Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
|
|
219
239
|
| No repository or homepage link | 🟡 Warning | Real projects almost always link to source. |
|
|
220
|
-
| 1–2
|
|
240
|
+
| 1–2 edits from a popular name, **and** recently published | 🟡 Warning | Classic typosquat shape. A swap of adjacent characters counts as one edit, because `recat`/`react` is what squatters actually publish. Age matters: an old lookalike is just a package with a similar name. |
|
|
221
241
|
|
|
222
242
|
Warnings are advisory by default. Nothing but non-existence blocks unless you pass
|
|
223
243
|
`--strict`.
|
|
@@ -353,11 +373,12 @@ earns its keep. Use both.
|
|
|
353
373
|
> that stand between you and it, and they are advisory. Improving this is the main
|
|
354
374
|
> open problem — see [issues](https://github.com/M1rwana12/ghostpkg/issues).
|
|
355
375
|
|
|
356
|
-
- Typo detection compares against the 2,000 most-downloaded
|
|
357
|
-
on a less popular package won't be flagged as a lookalike.
|
|
358
|
-
-
|
|
359
|
-
|
|
360
|
-
-
|
|
376
|
+
- Typo detection compares against the 2,000 most-downloaded projects in each
|
|
377
|
+
ecosystem, so a squat on a less popular package won't be flagged as a lookalike.
|
|
378
|
+
- Names shorter than five characters are not compared at all: below that the name
|
|
379
|
+
space is too dense for edit distance to mean anything.
|
|
380
|
+
- The cache lives on disk. `ghostpkg clear-cache` removes it and
|
|
381
|
+
`GHOSTPKG_CACHE_DIR` moves it.
|
|
361
382
|
- Registry outages surface as exit code `2` rather than a silent pass — deliberately,
|
|
362
383
|
but it does mean a flaky network fails your build.
|
|
363
384
|
|
|
@@ -133,6 +133,7 @@ and report TOML keys as package names.
|
|
|
133
133
|
| `--strict` | Promote warnings to blocks |
|
|
134
134
|
| `--json` | Machine-readable output for scripts and CI |
|
|
135
135
|
| `-q`, `--quiet` | Hide packages that passed |
|
|
136
|
+
| `--no-cache` | Neither read nor write the cache |
|
|
136
137
|
| `--version` | Print the version |
|
|
137
138
|
|
|
138
139
|
### Exit codes
|
|
@@ -143,6 +144,25 @@ and report TOML keys as package names.
|
|
|
143
144
|
| `1` | At least one package blocked |
|
|
144
145
|
| `2` | Usage error, unreadable manifest, or the registry was unreachable |
|
|
145
146
|
|
|
147
|
+
### Caching
|
|
148
|
+
|
|
149
|
+
Lookups are cached on disk, so re-scanning a 150-package manifest takes 0.4s
|
|
150
|
+
instead of 4.7s.
|
|
151
|
+
|
|
152
|
+
Time-to-live depends on the answer, and the negative case is the one that
|
|
153
|
+
matters: **"does not exist" is held for only an hour**, because a free name can
|
|
154
|
+
be registered at any moment and that is the entire attack. Young packages are
|
|
155
|
+
held six hours, established ones a day.
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
ghostpkg scan requirements.txt --no-cache # bypass it
|
|
159
|
+
ghostpkg clear-cache # delete it
|
|
160
|
+
GHOSTPKG_CACHE_DIR=/tmp/gp ghostpkg check x # move it
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
A corrupt, unreadable or unwritable cache degrades to no cache rather than
|
|
164
|
+
breaking your run.
|
|
165
|
+
|
|
146
166
|
### JSON output
|
|
147
167
|
|
|
148
168
|
```console
|
|
@@ -170,7 +190,7 @@ $ ghostpkg check somepkgthatisnotreal9911 --json
|
|
|
170
190
|
| First published < 1 year ago | 🟡 Warning | Weaker version of the same signal. |
|
|
171
191
|
| Only one release | 🟡 Warning | Squats are usually published once and abandoned. |
|
|
172
192
|
| No repository or homepage link | 🟡 Warning | Real projects almost always link to source. |
|
|
173
|
-
| 1–2
|
|
193
|
+
| 1–2 edits from a popular name, **and** recently published | 🟡 Warning | Classic typosquat shape. A swap of adjacent characters counts as one edit, because `recat`/`react` is what squatters actually publish. Age matters: an old lookalike is just a package with a similar name. |
|
|
174
194
|
|
|
175
195
|
Warnings are advisory by default. Nothing but non-existence blocks unless you pass
|
|
176
196
|
`--strict`.
|
|
@@ -306,11 +326,12 @@ earns its keep. Use both.
|
|
|
306
326
|
> that stand between you and it, and they are advisory. Improving this is the main
|
|
307
327
|
> open problem — see [issues](https://github.com/M1rwana12/ghostpkg/issues).
|
|
308
328
|
|
|
309
|
-
- Typo detection compares against the 2,000 most-downloaded
|
|
310
|
-
on a less popular package won't be flagged as a lookalike.
|
|
311
|
-
-
|
|
312
|
-
|
|
313
|
-
-
|
|
329
|
+
- Typo detection compares against the 2,000 most-downloaded projects in each
|
|
330
|
+
ecosystem, so a squat on a less popular package won't be flagged as a lookalike.
|
|
331
|
+
- Names shorter than five characters are not compared at all: below that the name
|
|
332
|
+
space is too dense for edit distance to mean anything.
|
|
333
|
+
- The cache lives on disk. `ghostpkg clear-cache` removes it and
|
|
334
|
+
`GHOSTPKG_CACHE_DIR` moves it.
|
|
314
335
|
- Registry outages surface as exit code `2` rather than a silent pass — deliberately,
|
|
315
336
|
but it does mean a flaky network fails your build.
|
|
316
337
|
|
|
@@ -86,14 +86,23 @@ ghostpkg scan package.json
|
|
|
86
86
|
|
|
87
87
|
# машинозчитуваний вивід
|
|
88
88
|
ghostpkg check somepkg --json
|
|
89
|
+
|
|
90
|
+
# видалити кеш
|
|
91
|
+
ghostpkg clear-cache
|
|
89
92
|
```
|
|
90
93
|
|
|
94
|
+
Результати запитів кешуються на диск: повторна перевірка манифесту зі 150
|
|
95
|
+
залежностей займає 0,4 с замість 4,7 с. **Відповідь «пакета не існує»
|
|
96
|
+
зберігається лише годину** — вільне ім'я можуть зареєструвати будь-якої
|
|
97
|
+
миті, і саме в цьому вся атака.
|
|
98
|
+
|
|
91
99
|
| Прапорець | Призначення |
|
|
92
100
|
|---|---|
|
|
93
101
|
| `-e`, `--ecosystem` | `pypi` (типово) або `npm` |
|
|
94
102
|
| `--strict` | Підвищує попередження до блокувань |
|
|
95
103
|
| `--json` | Вивід у JSON для скриптів і CI |
|
|
96
104
|
| `-q`, `--quiet` | Ховає пакети, які пройшли перевірку |
|
|
105
|
+
| `--no-cache` | Не читати й не писати кеш |
|
|
97
106
|
|
|
98
107
|
`scan` розпізнає `requirements*.txt`, `pyproject.toml` (PEP 621 і Poetry)
|
|
99
108
|
та `package.json`. Невідомий формат він **відхиляє з помилкою, а не вгадує**.
|
|
@@ -163,11 +172,12 @@ $ ghostpkg check react-router-dom-utils -e npm
|
|
|
163
172
|
> попередження, і вони дорадчі. Це головна відкрита проблема —
|
|
164
173
|
> див. [issues](https://github.com/M1rwana12/ghostpkg/issues).
|
|
165
174
|
|
|
166
|
-
- Виявлення опечаток порівнює з 2 000 найпопулярніших проєктів
|
|
167
|
-
під менш популярний пакет як схожа назва не позначиться.
|
|
168
|
-
-
|
|
169
|
-
|
|
170
|
-
-
|
|
175
|
+
- Виявлення опечаток порівнює з 2 000 найпопулярніших проєктів кожної екосистеми,
|
|
176
|
+
тому підробка під менш популярний пакет як схожа назва не позначиться.
|
|
177
|
+
- Імена коротші за 5 символів не порівнюються: там простір назв надто щільний,
|
|
178
|
+
щоб відстань редагування щось означала.
|
|
179
|
+
- Кеш зберігається локально. `ghostpkg clear-cache` видаляє його,
|
|
180
|
+
`GHOSTPKG_CACHE_DIR` змінює розташування.
|
|
171
181
|
|
|
172
182
|
---
|
|
173
183
|
|
|
@@ -18,7 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
from dataclasses import dataclass, field
|
|
19
19
|
from enum import Enum
|
|
20
20
|
|
|
21
|
-
from .data import TOP_PYPI
|
|
21
|
+
from .data import TOP_NPM, TOP_PYPI
|
|
22
22
|
from .registries import PackageFacts
|
|
23
23
|
|
|
24
24
|
YOUNG_DAYS = 90
|
|
@@ -45,25 +45,44 @@ class Finding:
|
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
def edit_distance(left: str, right: str, cutoff: int = 3) -> int:
|
|
48
|
-
"""Levenshtein distance, abandoning early once it exceeds `cutoff`.
|
|
48
|
+
"""Damerau-Levenshtein distance, abandoning early once it exceeds `cutoff`.
|
|
49
|
+
|
|
50
|
+
Swapping two adjacent characters counts as one edit, not two. That matters
|
|
51
|
+
here more than it looks: transposition is the commonest typosquat shape --
|
|
52
|
+
`recat` for `react`, `lodahs` for `lodash`, `webpakc` for `webpack`. Plain
|
|
53
|
+
Levenshtein scores all three as two edits, which put them outside the budget
|
|
54
|
+
for names of that length and let every one of them through.
|
|
55
|
+
"""
|
|
49
56
|
if left == right:
|
|
50
57
|
return 0
|
|
51
58
|
if abs(len(left) - len(right)) > cutoff:
|
|
52
59
|
return cutoff + 1
|
|
53
60
|
|
|
61
|
+
before_previous: list[int] = []
|
|
54
62
|
previous = list(range(len(right) + 1))
|
|
55
63
|
for i, a in enumerate(left, 1):
|
|
56
64
|
current = [i]
|
|
57
65
|
for j, b in enumerate(right, 1):
|
|
58
|
-
current
|
|
59
|
-
|
|
60
|
-
|
|
66
|
+
cost = min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (a != b))
|
|
67
|
+
if (
|
|
68
|
+
i > 1
|
|
69
|
+
and j > 1
|
|
70
|
+
and a == right[j - 2]
|
|
71
|
+
and left[i - 2] == b
|
|
72
|
+
):
|
|
73
|
+
cost = min(cost, before_previous[j - 2] + 1)
|
|
74
|
+
current.append(cost)
|
|
61
75
|
if min(current) > cutoff:
|
|
62
76
|
return cutoff + 1
|
|
63
|
-
previous = current
|
|
77
|
+
before_previous, previous = previous, current
|
|
64
78
|
return previous[-1]
|
|
65
79
|
|
|
66
80
|
|
|
81
|
+
POPULAR: dict[str, frozenset[str]] = {"pypi": TOP_PYPI, "npm": TOP_NPM}
|
|
82
|
+
|
|
83
|
+
MIN_COMPARABLE_LENGTH = 5
|
|
84
|
+
|
|
85
|
+
|
|
67
86
|
def _typo_budget(name: str) -> int:
|
|
68
87
|
"""How many edits still count as a plausible typo of a popular name.
|
|
69
88
|
|
|
@@ -74,18 +93,40 @@ def _typo_budget(name: str) -> int:
|
|
|
74
93
|
return 2 if len(name) >= 10 else 1
|
|
75
94
|
|
|
76
95
|
|
|
77
|
-
def
|
|
96
|
+
def _comparable(name: str, ecosystem: str) -> str:
|
|
97
|
+
"""The part of a name worth comparing.
|
|
98
|
+
|
|
99
|
+
An npm squat on a scoped package targets the part after the slash, since
|
|
100
|
+
the scope is usually owned by whoever it names.
|
|
101
|
+
"""
|
|
102
|
+
lowered = name.lower()
|
|
103
|
+
if ecosystem == "npm" and lowered.startswith("@") and "/" in lowered:
|
|
104
|
+
return lowered.rsplit("/", 1)[1]
|
|
105
|
+
return lowered
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def nearest_popular(name: str, ecosystem: str = "pypi") -> tuple[str, int] | None:
|
|
78
109
|
"""Closest popular package name within the typo budget, if any."""
|
|
110
|
+
popular = POPULAR.get(ecosystem)
|
|
111
|
+
if not popular:
|
|
112
|
+
return None
|
|
113
|
+
|
|
79
114
|
lowered = name.lower()
|
|
80
|
-
|
|
115
|
+
target = _comparable(name, ecosystem)
|
|
116
|
+
|
|
117
|
+
if lowered in popular or target in popular:
|
|
118
|
+
return None
|
|
119
|
+
# Below this length the name space is too dense for edit distance to mean
|
|
120
|
+
# anything: 'core' sits one edit from 'cors'.
|
|
121
|
+
if len(target) < MIN_COMPARABLE_LENGTH:
|
|
81
122
|
return None
|
|
82
123
|
|
|
83
|
-
budget = _typo_budget(
|
|
124
|
+
budget = _typo_budget(target)
|
|
84
125
|
best: tuple[str, int] | None = None
|
|
85
126
|
for candidate in popular:
|
|
86
|
-
if abs(len(candidate) - len(
|
|
127
|
+
if abs(len(candidate) - len(target)) > budget:
|
|
87
128
|
continue
|
|
88
|
-
distance = edit_distance(
|
|
129
|
+
distance = edit_distance(target, candidate, cutoff=budget)
|
|
89
130
|
if 0 < distance <= budget and (best is None or distance < best[1]):
|
|
90
131
|
best = (candidate, distance)
|
|
91
132
|
if distance == 1:
|
|
@@ -114,7 +155,7 @@ def assess(facts: PackageFacts, strict: bool = False) -> Finding:
|
|
|
114
155
|
is_young = facts.age_days is not None and facts.age_days < NEW_DAYS
|
|
115
156
|
|
|
116
157
|
if is_young:
|
|
117
|
-
neighbour = nearest_popular(facts.name)
|
|
158
|
+
neighbour = nearest_popular(facts.name, facts.ecosystem)
|
|
118
159
|
if neighbour is not None:
|
|
119
160
|
popular_name, distance = neighbour
|
|
120
161
|
reasons.append(
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""On-disk cache for registry lookups.
|
|
2
|
+
|
|
3
|
+
Scanning a manifest costs one request per dependency, every time. A 200-line
|
|
4
|
+
requirements.txt in CI is 200 requests on every run, which is slow for the user
|
|
5
|
+
and rude to the registry.
|
|
6
|
+
|
|
7
|
+
Time-to-live depends on what was cached, and the important case is the negative
|
|
8
|
+
one. A name that does not exist today can be registered tomorrow -- that is the
|
|
9
|
+
entire attack this tool exists to catch -- so "does not exist" is held only
|
|
10
|
+
briefly. Established packages change slowly and are held far longer.
|
|
11
|
+
|
|
12
|
+
Cache failures are never fatal. An unwritable or corrupt cache degrades to no
|
|
13
|
+
cache at all rather than breaking a build.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
import tempfile
|
|
22
|
+
import time
|
|
23
|
+
from dataclasses import asdict
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from .registries import PackageFacts
|
|
27
|
+
|
|
28
|
+
SCHEMA = 1
|
|
29
|
+
|
|
30
|
+
TTL_MISSING = 60 * 60 # 1 hour -- a free name can be taken any time
|
|
31
|
+
TTL_YOUNG = 6 * 60 * 60 # 6 hours -- facts still moving
|
|
32
|
+
TTL_ESTABLISHED = 24 * 60 * 60 # 1 day -- changes slowly
|
|
33
|
+
|
|
34
|
+
ESTABLISHED_DAYS = 365
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def default_dir() -> Path:
|
|
38
|
+
"""Per-platform cache directory, worked out by hand.
|
|
39
|
+
|
|
40
|
+
A dependency for this would contradict the zero-dependency rule for the
|
|
41
|
+
sake of three lines.
|
|
42
|
+
"""
|
|
43
|
+
override = os.environ.get("GHOSTPKG_CACHE_DIR")
|
|
44
|
+
if override:
|
|
45
|
+
return Path(override)
|
|
46
|
+
if sys.platform == "win32":
|
|
47
|
+
base = os.environ.get("LOCALAPPDATA") or os.path.expanduser("~")
|
|
48
|
+
return Path(base) / "ghostpkg"
|
|
49
|
+
if sys.platform == "darwin":
|
|
50
|
+
return Path(os.path.expanduser("~/Library/Caches")) / "ghostpkg"
|
|
51
|
+
base = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")
|
|
52
|
+
return Path(base) / "ghostpkg"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def ttl_for(facts: PackageFacts) -> int:
|
|
56
|
+
if not facts.exists:
|
|
57
|
+
return TTL_MISSING
|
|
58
|
+
if facts.age_days is not None and facts.age_days >= ESTABLISHED_DAYS:
|
|
59
|
+
return TTL_ESTABLISHED
|
|
60
|
+
return TTL_YOUNG
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Cache:
|
|
64
|
+
"""Read once, write once. Threads only read, so no locking is needed."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, directory: Path | None = None, enabled: bool = True) -> None:
|
|
67
|
+
self.enabled = enabled
|
|
68
|
+
self.directory = directory or default_dir()
|
|
69
|
+
self._entries: dict[str, dict] = {}
|
|
70
|
+
self._dirty = False
|
|
71
|
+
self.hits = 0
|
|
72
|
+
self.misses = 0
|
|
73
|
+
if self.enabled:
|
|
74
|
+
self._load()
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def path(self) -> Path:
|
|
78
|
+
return self.directory / "registry.json"
|
|
79
|
+
|
|
80
|
+
def _load(self) -> None:
|
|
81
|
+
try:
|
|
82
|
+
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
|
83
|
+
except (OSError, ValueError):
|
|
84
|
+
return
|
|
85
|
+
if not isinstance(raw, dict) or raw.get("schema") != SCHEMA:
|
|
86
|
+
return
|
|
87
|
+
entries = raw.get("entries")
|
|
88
|
+
if isinstance(entries, dict):
|
|
89
|
+
self._entries = entries
|
|
90
|
+
|
|
91
|
+
@staticmethod
|
|
92
|
+
def _key(ecosystem: str, name: str) -> str:
|
|
93
|
+
return f"{ecosystem}:{name.lower()}"
|
|
94
|
+
|
|
95
|
+
def get(self, ecosystem: str, name: str) -> PackageFacts | None:
|
|
96
|
+
if not self.enabled:
|
|
97
|
+
return None
|
|
98
|
+
entry = self._entries.get(self._key(ecosystem, name))
|
|
99
|
+
if not isinstance(entry, dict):
|
|
100
|
+
self.misses += 1
|
|
101
|
+
return None
|
|
102
|
+
stored_at = entry.get("at")
|
|
103
|
+
ttl = entry.get("ttl")
|
|
104
|
+
facts = entry.get("facts")
|
|
105
|
+
if not isinstance(stored_at, (int, float)) or not isinstance(ttl, (int, float)):
|
|
106
|
+
self.misses += 1
|
|
107
|
+
return None
|
|
108
|
+
if time.time() - stored_at > ttl:
|
|
109
|
+
self.misses += 1
|
|
110
|
+
return None
|
|
111
|
+
try:
|
|
112
|
+
revived = PackageFacts(**facts)
|
|
113
|
+
except (TypeError, ValueError):
|
|
114
|
+
self.misses += 1
|
|
115
|
+
return None
|
|
116
|
+
self.hits += 1
|
|
117
|
+
# keep the caller's spelling of the name rather than the cached one
|
|
118
|
+
if revived.name != name:
|
|
119
|
+
revived = PackageFacts(**{**asdict(revived), "name": name})
|
|
120
|
+
return revived
|
|
121
|
+
|
|
122
|
+
def put(self, facts: PackageFacts) -> None:
|
|
123
|
+
if not self.enabled:
|
|
124
|
+
return
|
|
125
|
+
self._entries[self._key(facts.ecosystem, facts.name)] = {
|
|
126
|
+
"at": time.time(),
|
|
127
|
+
"ttl": ttl_for(facts),
|
|
128
|
+
"facts": asdict(facts),
|
|
129
|
+
}
|
|
130
|
+
self._dirty = True
|
|
131
|
+
|
|
132
|
+
def _prune(self) -> None:
|
|
133
|
+
now = time.time()
|
|
134
|
+
self._entries = {
|
|
135
|
+
key: entry
|
|
136
|
+
for key, entry in self._entries.items()
|
|
137
|
+
if isinstance(entry, dict)
|
|
138
|
+
and isinstance(entry.get("at"), (int, float))
|
|
139
|
+
and now - entry["at"] <= entry.get("ttl", 0)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
def save(self) -> None:
|
|
143
|
+
"""Write atomically. Any failure is silent -- a cache is a convenience."""
|
|
144
|
+
if not self.enabled or not self._dirty:
|
|
145
|
+
return
|
|
146
|
+
self._prune()
|
|
147
|
+
payload = {"schema": SCHEMA, "entries": self._entries}
|
|
148
|
+
try:
|
|
149
|
+
self.directory.mkdir(parents=True, exist_ok=True)
|
|
150
|
+
handle, temporary = tempfile.mkstemp(dir=str(self.directory), suffix=".tmp")
|
|
151
|
+
with os.fdopen(handle, "w", encoding="utf-8") as file:
|
|
152
|
+
json.dump(payload, file)
|
|
153
|
+
os.replace(temporary, self.path)
|
|
154
|
+
except OSError:
|
|
155
|
+
try:
|
|
156
|
+
os.unlink(temporary) # noqa: F821
|
|
157
|
+
except (OSError, NameError):
|
|
158
|
+
pass
|
|
159
|
+
|
|
160
|
+
def clear(self) -> bool:
|
|
161
|
+
try:
|
|
162
|
+
self.path.unlink()
|
|
163
|
+
return True
|
|
164
|
+
except OSError:
|
|
165
|
+
return False
|
|
@@ -11,6 +11,7 @@ from pathlib import Path
|
|
|
11
11
|
|
|
12
12
|
from . import __version__
|
|
13
13
|
from .assess import Finding, Verdict, assess
|
|
14
|
+
from .cache import Cache
|
|
14
15
|
from .manifests import UnsupportedManifest, load_manifest
|
|
15
16
|
from .registries import RegistryError, fetch
|
|
16
17
|
|
|
@@ -89,12 +90,25 @@ def summarise(findings: list[Finding], palette: Palette) -> None:
|
|
|
89
90
|
print(palette.green(f" all {len(findings)} packages look fine"))
|
|
90
91
|
|
|
91
92
|
|
|
92
|
-
def evaluate(
|
|
93
|
+
def evaluate(
|
|
94
|
+
names: list[str],
|
|
95
|
+
ecosystem: str,
|
|
96
|
+
strict: bool,
|
|
97
|
+
cache: Cache | None = None,
|
|
98
|
+
) -> list[Finding]:
|
|
93
99
|
def one(name: str) -> Finding:
|
|
94
|
-
|
|
100
|
+
facts = cache.get(ecosystem, name) if cache else None
|
|
101
|
+
if facts is None:
|
|
102
|
+
facts = fetch(name, ecosystem)
|
|
103
|
+
if cache:
|
|
104
|
+
cache.put(facts)
|
|
105
|
+
return assess(facts, strict=strict)
|
|
95
106
|
|
|
96
107
|
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
|
97
|
-
|
|
108
|
+
findings = list(pool.map(one, names))
|
|
109
|
+
if cache:
|
|
110
|
+
cache.save()
|
|
111
|
+
return findings
|
|
98
112
|
|
|
99
113
|
|
|
100
114
|
def build_parser() -> argparse.ArgumentParser:
|
|
@@ -124,6 +138,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
124
138
|
)
|
|
125
139
|
command.add_argument("--json", action="store_true", help="machine-readable output")
|
|
126
140
|
command.add_argument("-q", "--quiet", action="store_true", help="hide passing packages")
|
|
141
|
+
command.add_argument(
|
|
142
|
+
"--no-cache", action="store_true", help="ignore and do not write the cache"
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
sub.add_parser("clear-cache", help="delete the cached registry lookups")
|
|
127
146
|
|
|
128
147
|
return parser
|
|
129
148
|
|
|
@@ -131,6 +150,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
131
150
|
def main(argv: list[str] | None = None) -> int:
|
|
132
151
|
args = build_parser().parse_args(argv)
|
|
133
152
|
|
|
153
|
+
if args.command == "clear-cache":
|
|
154
|
+
cache = Cache(enabled=True)
|
|
155
|
+
removed = cache.clear()
|
|
156
|
+
print(f"ghostpkg: {'removed ' + str(cache.path) if removed else 'nothing to remove'}")
|
|
157
|
+
return EXIT_OK
|
|
158
|
+
|
|
134
159
|
if args.command == "scan":
|
|
135
160
|
if not args.path.exists():
|
|
136
161
|
print(f"ghostpkg: no such file: {args.path}", file=sys.stderr)
|
|
@@ -149,8 +174,9 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
149
174
|
else:
|
|
150
175
|
names, ecosystem = args.names, args.ecosystem
|
|
151
176
|
|
|
177
|
+
cache = Cache(enabled=not args.no_cache)
|
|
152
178
|
try:
|
|
153
|
-
findings = evaluate(names, ecosystem, args.strict)
|
|
179
|
+
findings = evaluate(names, ecosystem, args.strict, cache)
|
|
154
180
|
except RegistryError as exc:
|
|
155
181
|
print(f"ghostpkg: {exc}", file=sys.stderr)
|
|
156
182
|
return EXIT_ERROR
|