ghostpkg 0.19.2__tar.gz → 0.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/CHANGELOG.md +80 -1
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/PKG-INFO +25 -12
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/README.en.md +24 -11
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/__init__.py +1 -1
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/cli.py +28 -2
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/jslocks.py +187 -156
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/manifests.py +766 -684
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/prose.py +210 -204
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/registries.py +45 -3
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/pyproject.toml +1 -1
- ghostpkg-0.20.0/tests/test_field_findings.py +184 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_policy.py +57 -0
- ghostpkg-0.20.0/tests/test_review_findings.py +163 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.github/ISSUE_TEMPLATE/bug.yml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.github/ISSUE_TEMPLATE/false-positive.yml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.github/ISSUE_TEMPLATE/missed-package.yml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.github/ghostpkg-ignore.json +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.github/workflows/ci.yml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.github/workflows/publish.yml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.gitignore +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/.pre-commit-hooks.yaml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/2.0 +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/CONTRIBUTING.md +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/LICENSE +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/README.md +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/SECURITY.md +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/action.yml +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/assets/banner.html +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/assets/banner.png +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/assets/demo.gif +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/assets/make_demo.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/__main__.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/assess.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/cache.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/data.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/discover.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/inspection.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/policy.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/report.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/rules.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/ghostpkg/scanner.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_action.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_assess.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_cache.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_discover.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_hostile_input.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_inspection.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_jslocks.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_lockfiles.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_manifests.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_prose.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_registries.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_regressions.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_report.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_stated_sources.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_versions.py +0 -0
- {ghostpkg-0.19.2 → ghostpkg-0.20.0}/tests/test_withdrawn.py +0 -0
|
@@ -6,6 +6,83 @@ this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.20.0] - 2026-09-03
|
|
10
|
+
|
|
11
|
+
Four defects found by scanning **10,109 packages** across `vercel/next.js`,
|
|
12
|
+
`home-assistant/core` and `getsentry/sentry`. Those three repositories produced
|
|
13
|
+
53 blocks and **every one of them was false**. None of these shapes occur in a
|
|
14
|
+
small synthetic manifest, which is why 571 tests and a 35-check acceptance pass
|
|
15
|
+
had missed all four.
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
- **`ghostpkg check -e npm "@scope/name"` reported success having checked
|
|
19
|
+
nothing.** The names went through the PEP 508 requirements parser, whose
|
|
20
|
+
pattern demands an alphanumeric first character, so every scoped npm name was
|
|
21
|
+
dropped in silence -- the run printed "all 0 packages look fine" and exited
|
|
22
|
+
0. On the command this tool is named for, and for a quarter of the npm
|
|
23
|
+
namespace. A false all-clear is worse than any false positive, and this is
|
|
24
|
+
the most serious defect the project has shipped.
|
|
25
|
+
- **A pinned version was compared as text rather than as a version.**
|
|
26
|
+
`aiopurpleair==2025.08.1` was blocked as non-existent while `pip download`
|
|
27
|
+
installed it happily: PyPI stores the canonical `2025.8.1`. Comparison now
|
|
28
|
+
normalises both sides -- leading zeros, a leading `v`, `-`/`_` separators and
|
|
29
|
+
the pre-release spellings. Three such pins sit in unmodified Home Assistant
|
|
30
|
+
requirements, and a false block breaks a build.
|
|
31
|
+
- **A package the checkout provides itself is no longer looked up.** All three
|
|
32
|
+
names blocked in a 6,335-package scan of `vercel/next.js` were the
|
|
33
|
+
repository's own packages, `@next/font` among them. A monorepo depends on
|
|
34
|
+
itself, and not always through `workspace:*` -- an exact pin is just as
|
|
35
|
+
common. Names declared by any `package.json` or `pyproject.toml` in the scan
|
|
36
|
+
are now excluded, which is the same rule as `workspace:` stated differently.
|
|
37
|
+
- **`MANIFEST.in` is no longer read as a requirements file.** `.in` is the
|
|
38
|
+
pip-tools convention and also the extension of a packaging directives file.
|
|
39
|
+
Read as requirements it reported `graft` as a package that exists -- there is
|
|
40
|
+
a real project of that name -- and blocked `recursive-exclude`.
|
|
41
|
+
|
|
42
|
+
### Documentation
|
|
43
|
+
- The `--json` example showed the bare array from before 0.14.0. It is a
|
|
44
|
+
`{schema, tool, summary, findings}` envelope, and the documented example now
|
|
45
|
+
matches, including `source` and `line`.
|
|
46
|
+
|
|
47
|
+
605 tests.
|
|
48
|
+
|
|
49
|
+
## [0.19.3] - 2026-09-03
|
|
50
|
+
|
|
51
|
+
Found by reviewing the published 0.19.2 rather than the working tree. Two of
|
|
52
|
+
the four are false blocks, which this project treats as its worst failure, and
|
|
53
|
+
both have the same cause: `jslocks.py` was written after the test file that
|
|
54
|
+
exists to defend the rule "a dependency naming its own source is not the
|
|
55
|
+
registry's business", so `yarn.lock` and `pnpm-lock.yaml` were never covered by
|
|
56
|
+
it.
|
|
57
|
+
|
|
58
|
+
### Fixed
|
|
59
|
+
- **A yarn `owner/repo#ref` dependency was blocked.** GitHub shorthand was
|
|
60
|
+
handled for `package.json` and missed in `yarn.lock`, so
|
|
61
|
+
`internal-lib@acme/internal-lib#v1.2.3` was looked up on npmjs, found absent,
|
|
62
|
+
and reported as a package that does not exist.
|
|
63
|
+
- **A pnpm git or URL dependency produced a nonsense package name.** The
|
|
64
|
+
protocol is only a prefix in lockfile v5; from v6 it follows `name@`, so the
|
|
65
|
+
guard never fired. `github.com/acme/forked/abc123` was read as a package
|
|
66
|
+
called `github.com/acme/forked`, and
|
|
67
|
+
`foo@https://codeload.github.com/...` as one called
|
|
68
|
+
`foo@https://codeload.github.com/acme/foo`. Both were then blocked.
|
|
69
|
+
- **`--timeout` did nothing.** `def _get_json(url, timeout=TIMEOUT)` bound the
|
|
70
|
+
module global once, at import; the CLI assigned `registries.TIMEOUT` and
|
|
71
|
+
nothing ever re-read it, so every request used 15 seconds regardless.
|
|
72
|
+
- **`.windsurfrules` was found and never scanned.** The directory search
|
|
73
|
+
offered it up, the parser refused it, and the CLI ignores an unreadable
|
|
74
|
+
*discovered* file on purpose -- so nothing was printed. There is now a test
|
|
75
|
+
asserting that every file the search returns can actually be parsed.
|
|
76
|
+
|
|
77
|
+
### Added
|
|
78
|
+
- End-to-end tests for suppression. The ignore file was loaded and applied in
|
|
79
|
+
`main()` only, and nothing asserted its effect on a verdict or an exit code:
|
|
80
|
+
the entire policy call could be replaced with `used = []` and all 539 tests
|
|
81
|
+
still passed. Three tests now cover the real path, and they fail against that
|
|
82
|
+
stub.
|
|
83
|
+
|
|
84
|
+
571 tests.
|
|
85
|
+
|
|
9
86
|
## [0.19.2] - 2026-09-02
|
|
10
87
|
|
|
11
88
|
Found by throwing hostile input at the parsers rather than by reading them.
|
|
@@ -733,7 +810,9 @@ First release.
|
|
|
733
810
|
- No corpus of hallucinated package names is shipped, following the decision of
|
|
734
811
|
the USENIX'25 authors not to publish theirs.
|
|
735
812
|
|
|
736
|
-
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.
|
|
813
|
+
[Unreleased]: https://github.com/M1rwana12/ghostpkg/compare/v0.20.0...HEAD
|
|
814
|
+
[0.20.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.20.0
|
|
815
|
+
[0.19.3]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.19.3
|
|
737
816
|
[0.19.2]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.19.2
|
|
738
817
|
[0.19.1]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.19.1
|
|
739
818
|
[0.19.0]: https://github.com/M1rwana12/ghostpkg/releases/tag/v0.19.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ghostpkg
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.20.0
|
|
4
4
|
Summary: Catch package names that do not exist before you install them
|
|
5
5
|
Project-URL: Homepage, https://github.com/m1rwana12/ghostpkg
|
|
6
6
|
Project-URL: Issues, https://github.com/m1rwana12/ghostpkg/issues
|
|
@@ -340,19 +340,32 @@ malformed file **stops the run** rather than quietly leaving you unprotected.
|
|
|
340
340
|
### JSON output
|
|
341
341
|
|
|
342
342
|
```console
|
|
343
|
-
$ ghostpkg
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
"
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
343
|
+
$ ghostpkg scan requirements.txt --json
|
|
344
|
+
{
|
|
345
|
+
"schema": 1,
|
|
346
|
+
"tool": { "name": "ghostpkg", "version": "0.20.0" },
|
|
347
|
+
"summary": { "checked": 2, "blocked": 1, "warned": 0, "errored": 0 },
|
|
348
|
+
"findings": [
|
|
349
|
+
{
|
|
350
|
+
"name": "somepkgthatisnotreal9911",
|
|
351
|
+
"ecosystem": "pypi",
|
|
352
|
+
"verdict": "BLOCK",
|
|
353
|
+
"source": "requirements.txt",
|
|
354
|
+
"line": 2,
|
|
355
|
+
"exists": false,
|
|
356
|
+
"latest_version": null,
|
|
357
|
+
"reasons": [
|
|
358
|
+
{ "rule": "GP001", "text": "does not exist on pypi" }
|
|
359
|
+
]
|
|
360
|
+
}
|
|
361
|
+
]
|
|
362
|
+
}
|
|
354
363
|
```
|
|
355
364
|
|
|
365
|
+
`schema` is there so a consumer can tell what it is reading; `source` and
|
|
366
|
+
`line` say where the name was written. `--format github` emits workflow
|
|
367
|
+
commands instead, and `--format text` is the default.
|
|
368
|
+
|
|
356
369
|
---
|
|
357
370
|
|
|
358
371
|
## Every signal it checks
|
|
@@ -292,19 +292,32 @@ malformed file **stops the run** rather than quietly leaving you unprotected.
|
|
|
292
292
|
### JSON output
|
|
293
293
|
|
|
294
294
|
```console
|
|
295
|
-
$ ghostpkg
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
"
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
295
|
+
$ ghostpkg scan requirements.txt --json
|
|
296
|
+
{
|
|
297
|
+
"schema": 1,
|
|
298
|
+
"tool": { "name": "ghostpkg", "version": "0.20.0" },
|
|
299
|
+
"summary": { "checked": 2, "blocked": 1, "warned": 0, "errored": 0 },
|
|
300
|
+
"findings": [
|
|
301
|
+
{
|
|
302
|
+
"name": "somepkgthatisnotreal9911",
|
|
303
|
+
"ecosystem": "pypi",
|
|
304
|
+
"verdict": "BLOCK",
|
|
305
|
+
"source": "requirements.txt",
|
|
306
|
+
"line": 2,
|
|
307
|
+
"exists": false,
|
|
308
|
+
"latest_version": null,
|
|
309
|
+
"reasons": [
|
|
310
|
+
{ "rule": "GP001", "text": "does not exist on pypi" }
|
|
311
|
+
]
|
|
312
|
+
}
|
|
313
|
+
]
|
|
314
|
+
}
|
|
306
315
|
```
|
|
307
316
|
|
|
317
|
+
`schema` is there so a consumer can tell what it is reading; `source` and
|
|
318
|
+
`line` say where the name was written. `--format github` emits workflow
|
|
319
|
+
commands instead, and `--format text` is the default.
|
|
320
|
+
|
|
308
321
|
---
|
|
309
322
|
|
|
310
323
|
## Every signal it checks
|
|
@@ -15,7 +15,13 @@ from . import __version__, registries
|
|
|
15
15
|
from .assess import Finding
|
|
16
16
|
from .cache import Cache
|
|
17
17
|
from .discover import discover
|
|
18
|
-
from .manifests import
|
|
18
|
+
from .manifests import (
|
|
19
|
+
UnsupportedManifest,
|
|
20
|
+
declared_name,
|
|
21
|
+
load_manifest,
|
|
22
|
+
parse_npm_names,
|
|
23
|
+
parse_requirements,
|
|
24
|
+
)
|
|
19
25
|
from .policy import PolicyError, apply as apply_policy, load as load_policy
|
|
20
26
|
from .report import Palette, as_github, as_json, render, summarise, use_colour
|
|
21
27
|
from .scanner import DEFAULT_WORKERS, evaluate
|
|
@@ -134,6 +140,16 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
134
140
|
print(f"ghostpkg: no manifests found in {where}", file=sys.stderr)
|
|
135
141
|
return EXIT_NOTHING_SCANNED
|
|
136
142
|
|
|
143
|
+
# Names the checkout provides itself. A monorepo depends on its own
|
|
144
|
+
# packages, and not always through `workspace:*` -- an exact pin is
|
|
145
|
+
# just as common, and looking those up on the public registry
|
|
146
|
+
# blocked every one of them.
|
|
147
|
+
local: set[str] = set()
|
|
148
|
+
for path in paths:
|
|
149
|
+
own = declared_name(path)
|
|
150
|
+
if own:
|
|
151
|
+
local.add(own.lower())
|
|
152
|
+
|
|
137
153
|
for path in paths:
|
|
138
154
|
try:
|
|
139
155
|
found, ecosystem = load_manifest(path)
|
|
@@ -151,6 +167,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
151
167
|
return EXIT_ERROR
|
|
152
168
|
continue
|
|
153
169
|
for requirement in found:
|
|
170
|
+
if requirement.name.lower() in local:
|
|
171
|
+
continue # provided by this checkout
|
|
154
172
|
by_ecosystem.setdefault(
|
|
155
173
|
requirement.ecosystem or ecosystem, []
|
|
156
174
|
).append(requirement)
|
|
@@ -162,7 +180,15 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
162
180
|
return EXIT_NOTHING_SCANNED
|
|
163
181
|
else:
|
|
164
182
|
# Accept a pin on the command line too: `ghostpkg check requests==2.31.0`.
|
|
165
|
-
|
|
183
|
+
# npm names go through their own reader, because a scoped name is not
|
|
184
|
+
# expressible in PEP 508 and the requirements parser dropped every one.
|
|
185
|
+
if args.ecosystem == "npm":
|
|
186
|
+
by_ecosystem[args.ecosystem] = parse_npm_names(args.names)
|
|
187
|
+
else:
|
|
188
|
+
by_ecosystem[args.ecosystem] = parse_requirements("\n".join(args.names))
|
|
189
|
+
if not by_ecosystem[args.ecosystem]:
|
|
190
|
+
print(f"ghostpkg: no package names in {' '.join(args.names)}", file=sys.stderr)
|
|
191
|
+
return EXIT_NOTHING_SCANNED
|
|
166
192
|
|
|
167
193
|
try:
|
|
168
194
|
policy, policy_path = load_policy(args.config)
|
|
@@ -1,156 +1,187 @@
|
|
|
1
|
-
"""`pnpm-lock.yaml` and `yarn.lock`, read as text.
|
|
2
|
-
|
|
3
|
-
Measured across twenty popular JavaScript repositories, `package-lock.json` --
|
|
4
|
-
the only npm lockfile ghostpkg could read -- was present in two of them.
|
|
5
|
-
`pnpm-lock.yaml` was in ten and `yarn.lock` in six, so four out of five projects
|
|
6
|
-
had a lockfile the tool refused. Lockfiles are the list that matters, because CI
|
|
7
|
-
installs from them and they name transitive dependencies a manifest never
|
|
8
|
-
mentions.
|
|
9
|
-
|
|
10
|
-
Both formats are read line by line rather than with a YAML library, for the same
|
|
11
|
-
reason `pyproject.toml` has a hand-written fallback: a supply-chain tool that
|
|
12
|
-
installs a dependency tree of its own undermines its own argument. Neither
|
|
13
|
-
format needs general YAML -- the part worth reading is a list of keys.
|
|
14
|
-
|
|
15
|
-
Every entry states its own source, and this module applies the same rule the
|
|
16
|
-
rest of the parsers do: a name resolved from a workspace, a directory, a patch
|
|
17
|
-
or a git host is not a registry name, so the registry has no say. Both formats
|
|
18
|
-
carry that inline, and both were confirmed to do so in real files:
|
|
19
|
-
`"eslint-plugin-react-internal@link:./scripts/eslint-rules"` in React's classic
|
|
20
|
-
lockfile, `"@babel/benchmark@workspace:..."` in Babel's berry one.
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
from __future__ import annotations
|
|
24
|
-
|
|
25
|
-
import re
|
|
26
|
-
|
|
27
|
-
from .manifests import
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
#:
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
#
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
continue
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
1
|
+
"""`pnpm-lock.yaml` and `yarn.lock`, read as text.
|
|
2
|
+
|
|
3
|
+
Measured across twenty popular JavaScript repositories, `package-lock.json` --
|
|
4
|
+
the only npm lockfile ghostpkg could read -- was present in two of them.
|
|
5
|
+
`pnpm-lock.yaml` was in ten and `yarn.lock` in six, so four out of five projects
|
|
6
|
+
had a lockfile the tool refused. Lockfiles are the list that matters, because CI
|
|
7
|
+
installs from them and they name transitive dependencies a manifest never
|
|
8
|
+
mentions.
|
|
9
|
+
|
|
10
|
+
Both formats are read line by line rather than with a YAML library, for the same
|
|
11
|
+
reason `pyproject.toml` has a hand-written fallback: a supply-chain tool that
|
|
12
|
+
installs a dependency tree of its own undermines its own argument. Neither
|
|
13
|
+
format needs general YAML -- the part worth reading is a list of keys.
|
|
14
|
+
|
|
15
|
+
Every entry states its own source, and this module applies the same rule the
|
|
16
|
+
rest of the parsers do: a name resolved from a workspace, a directory, a patch
|
|
17
|
+
or a git host is not a registry name, so the registry has no say. Both formats
|
|
18
|
+
carry that inline, and both were confirmed to do so in real files:
|
|
19
|
+
`"eslint-plugin-react-internal@link:./scripts/eslint-rules"` in React's classic
|
|
20
|
+
lockfile, `"@babel/benchmark@workspace:..."` in Babel's berry one.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import re
|
|
26
|
+
|
|
27
|
+
from .manifests import (
|
|
28
|
+
GITHUB_SHORTHAND,
|
|
29
|
+
Requirement,
|
|
30
|
+
npm_alias_target,
|
|
31
|
+
strip_bom,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
#: A yarn descriptor protocol that resolves somewhere other than the registry.
|
|
35
|
+
#: `exec:` and `patch:` are berry-only; the rest appear in both versions.
|
|
36
|
+
YARN_LOCAL = (
|
|
37
|
+
"workspace:", "link:", "portal:", "file:", "patch:", "exec:",
|
|
38
|
+
"git+", "git:", "http://", "https://",
|
|
39
|
+
"github:", "gitlab:", "bitbucket:",
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
#: pnpm writes the same protocols into its keys, minus yarn's own inventions.
|
|
43
|
+
PNPM_LOCAL = ("file:", "link:", "http://", "https://", "git+", "git:")
|
|
44
|
+
|
|
45
|
+
#: In a pnpm key the version always starts with a digit, which is what tells a
|
|
46
|
+
#: scoped name apart from its version: `@babel/parser@7.29.3` splits at the
|
|
47
|
+
#: second `@`, not the first. Peer suffixes -- `(react@18.2.0)` in v9,
|
|
48
|
+
#: `_react@18.2.0` in v5 -- come after it and must not be split on.
|
|
49
|
+
PNPM_VERSION_AT = re.compile(r"(?<=.)@(?=\d)")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _is_host_path(key: str) -> bool:
|
|
53
|
+
"""`github.com/acme/forked/abc123` -- a v5 git dependency, keyed by host.
|
|
54
|
+
|
|
55
|
+
The slash is what gives it away. An unscoped npm name never contains one,
|
|
56
|
+
and a scoped name starts with `@`, so a key with a slash and no leading `@`
|
|
57
|
+
is a URL path rather than a package.
|
|
58
|
+
|
|
59
|
+
The dot alone is not enough, and testing it alone was wrong: dots are legal
|
|
60
|
+
and common in real names. Requiring only a dot threw away `big.js`,
|
|
61
|
+
`array.prototype.concat` and 289 of Svelte's 435 packages.
|
|
62
|
+
"""
|
|
63
|
+
if key.startswith("@") or "/" not in key:
|
|
64
|
+
return False
|
|
65
|
+
return "." in key.split("/", 1)[0]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _yarn_descriptor(text: str) -> "tuple[str, str] | None":
|
|
69
|
+
"""`(name, range)` from `name@range`, keeping a leading scope `@`.
|
|
70
|
+
|
|
71
|
+
The split is at the *first* `@` past position zero. Taking the last one
|
|
72
|
+
reads `@babel-baseline/cli@npm:@babel/cli@7.27.1` -- a real entry in
|
|
73
|
+
Babel's lockfile -- as a package called
|
|
74
|
+
`@babel-baseline/cli@npm:@babel/cli`.
|
|
75
|
+
"""
|
|
76
|
+
text = text.strip().strip('"').strip("'")
|
|
77
|
+
at = text.find("@", 1)
|
|
78
|
+
if at < 1:
|
|
79
|
+
return None
|
|
80
|
+
return text[:at], text[at + 1 :]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def parse_yarn_lock(text: str, source: str | None = None) -> list[Requirement]:
|
|
84
|
+
"""Dependencies from a `yarn.lock`, classic (v1) or berry (v2+).
|
|
85
|
+
|
|
86
|
+
Both write entries as a key at column zero holding one or more
|
|
87
|
+
comma-separated descriptors. Classic writes the raw range (`lodash@^4.17.19`),
|
|
88
|
+
berry prefixes the protocol (`lodash@npm:^4.17.19`), and berry also uses the
|
|
89
|
+
descriptor to alias: `@babel-baseline/cli@npm:@babel/cli@7.27.1` installs
|
|
90
|
+
`@babel/cli`, which is the name worth checking.
|
|
91
|
+
"""
|
|
92
|
+
found: list[Requirement] = []
|
|
93
|
+
seen: set[str] = set()
|
|
94
|
+
|
|
95
|
+
for raw in strip_bom(text).splitlines():
|
|
96
|
+
if not raw or raw[0] in " \t#" or not raw.rstrip().endswith(":"):
|
|
97
|
+
continue
|
|
98
|
+
line = raw.rstrip()[:-1].strip()
|
|
99
|
+
if not line or line.startswith("__metadata"):
|
|
100
|
+
continue
|
|
101
|
+
|
|
102
|
+
for part in line.split(","):
|
|
103
|
+
parsed = _yarn_descriptor(part)
|
|
104
|
+
if parsed is None:
|
|
105
|
+
continue
|
|
106
|
+
name, spec = parsed
|
|
107
|
+
if spec.startswith(YARN_LOCAL) or GITHUB_SHORTHAND.match(spec):
|
|
108
|
+
# A protocol, or `owner/repo#ref` shorthand. Both name their
|
|
109
|
+
# own source. The shorthand was handled for package.json and
|
|
110
|
+
# missed here, so a private repository dependency was looked
|
|
111
|
+
# up on npmjs and blocked.
|
|
112
|
+
continue
|
|
113
|
+
if spec.startswith("npm:"):
|
|
114
|
+
alias = npm_alias_target(spec)
|
|
115
|
+
if alias is not None:
|
|
116
|
+
name = alias
|
|
117
|
+
if name and name not in seen:
|
|
118
|
+
seen.add(name)
|
|
119
|
+
found.append(Requirement(name=name, source=source))
|
|
120
|
+
|
|
121
|
+
return found
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def parse_pnpm_lock(text: str, source: str | None = None) -> list[Requirement]:
|
|
125
|
+
"""Dependencies from a `pnpm-lock.yaml`.
|
|
126
|
+
|
|
127
|
+
Only the `packages:` block is read: it is the resolved set, and unlike
|
|
128
|
+
`dependencies:` it names transitive packages too. Three key shapes exist
|
|
129
|
+
across lockfile versions, all seen in the wild:
|
|
130
|
+
|
|
131
|
+
/react/18.2.0: v5
|
|
132
|
+
/react@18.2.0: v6
|
|
133
|
+
'react@18.2.0': v9
|
|
134
|
+
|
|
135
|
+
v9 appends peer resolutions in parentheses and v5 after an underscore, so
|
|
136
|
+
the split is made at the `@` that starts the version rather than the last
|
|
137
|
+
one in the line.
|
|
138
|
+
"""
|
|
139
|
+
found: list[Requirement] = []
|
|
140
|
+
seen: set[str] = set()
|
|
141
|
+
in_packages = False
|
|
142
|
+
|
|
143
|
+
for raw in strip_bom(text).splitlines():
|
|
144
|
+
stripped = raw.strip()
|
|
145
|
+
if not stripped or stripped.startswith("#"):
|
|
146
|
+
# A blank line does not end the block. Treating one as a
|
|
147
|
+
# column-zero key closed `packages:` at the first gap between
|
|
148
|
+
# entries, and every real lockfile has one there -- the parser
|
|
149
|
+
# returned nothing at all for all three files tested.
|
|
150
|
+
continue
|
|
151
|
+
if not raw[:1].isspace():
|
|
152
|
+
# A column-zero key ends whatever block preceded it.
|
|
153
|
+
in_packages = stripped == "packages:"
|
|
154
|
+
continue
|
|
155
|
+
if not in_packages or not stripped.endswith(":"):
|
|
156
|
+
continue
|
|
157
|
+
# Keys sit one level in; anything deeper belongs to an entry's body.
|
|
158
|
+
if len(raw) - len(raw.lstrip()) != 2:
|
|
159
|
+
continue
|
|
160
|
+
|
|
161
|
+
key = stripped[:-1].strip().strip('"').strip("'")
|
|
162
|
+
if not key:
|
|
163
|
+
continue
|
|
164
|
+
key = key.split("(", 1)[0] # v9 peer resolutions
|
|
165
|
+
if key.startswith("/"):
|
|
166
|
+
key = key[1:]
|
|
167
|
+
# The protocol is only a prefix in v5. From v6 it follows `name@`, so
|
|
168
|
+
# testing the start of the key never fired and the parser fell through
|
|
169
|
+
# to its slash split -- emitting names like `github.com/acme/forked`,
|
|
170
|
+
# which were then looked up on npmjs and blocked.
|
|
171
|
+
if key.startswith(PNPM_LOCAL) or "://" in key or _is_host_path(key):
|
|
172
|
+
continue
|
|
173
|
+
|
|
174
|
+
match = PNPM_VERSION_AT.search(key)
|
|
175
|
+
if match:
|
|
176
|
+
name = key[: match.start()]
|
|
177
|
+
elif "/" in key.lstrip("@"):
|
|
178
|
+
# v5 keys the version with a slash: @babel/code-frame/7.12.11
|
|
179
|
+
name = key.rsplit("/", 1)[0]
|
|
180
|
+
else:
|
|
181
|
+
continue
|
|
182
|
+
|
|
183
|
+
if name and name not in seen:
|
|
184
|
+
seen.add(name)
|
|
185
|
+
found.append(Requirement(name=name, source=source))
|
|
186
|
+
|
|
187
|
+
return found
|