penthera 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +179 -0
- package/README.md +128 -663
- package/bin/scan.js +19 -0
- package/docs/commands/README.md +23 -0
- package/docs/commands/action.md +21 -0
- package/docs/commands/agents.md +38 -0
- package/docs/commands/deps.md +115 -0
- package/docs/commands/fix.md +53 -0
- package/docs/commands/hook.md +69 -0
- package/docs/commands/install.md +53 -0
- package/docs/commands/scan.md +100 -0
- package/docs/commands/skill.md +58 -0
- package/docs/compare/README.md +25 -0
- package/docs/compare/github.md +60 -0
- package/docs/compare/semgrep.md +54 -0
- package/docs/compare/snyk.md +62 -0
- package/docs/compare/socket.md +78 -0
- package/docs/compare/zap.md +59 -0
- package/docs/dashboard.md +244 -0
- package/docs/evidence.md +8 -4
- package/docs/examples/github-actions-deps.yml +4 -4
- package/docs/examples/github-actions-scan.yml +4 -4
- package/docs/index.md +9 -0
- package/docs/internal-audit.md +83 -0
- package/docs/marketing.md +151 -0
- package/docs/plan.md +4 -3
- package/docs/privacy.md +38 -0
- package/docs/references.bib +10 -0
- package/docs/references.md +1 -0
- package/docs/research/socket-positioning-notes.md +183 -0
- package/docs/use-cases/README.md +14 -0
- package/docs/use-cases/agencies.md +71 -0
- package/docs/use-cases/ai-coding-teams.md +81 -0
- package/docs/use-cases/eu-sme-compliance.md +68 -0
- package/docs/use-cases/open-source-maintainers.md +75 -0
- package/docs/use-cases/solo-founder.md +65 -0
- package/lib/blackbox/email-dns.js +5 -0
- package/lib/compliance/questionnaire.js +4 -2
- package/lib/deps/capabilities.js +49 -12
- package/lib/deps/index.js +290 -30
- package/lib/deps/lockfile.js +25 -2
- package/lib/deps/popular-pypi.json +1 -0
- package/lib/deps/python/capabilities.js +447 -0
- package/lib/deps/python/index.js +234 -0
- package/lib/deps/python/lockfiles.js +359 -0
- package/lib/deps/python/source.js +396 -0
- package/lib/deps/python/toml.js +219 -0
- package/lib/deps/report.js +37 -6
- package/lib/deps/source.js +149 -24
- package/lib/deps/typosquat.js +24 -13
- package/lib/fix/apply.js +83 -19
- package/lib/hook/index.js +41 -8
- package/lib/install/index.js +59 -18
- package/lib/install/policy.js +17 -5
- package/lib/memory/repo-suppress.js +11 -3
- package/lib/references.json +77 -0
- package/lib/remediation/index.js +9 -0
- package/lib/retirejs.js +69 -17
- package/lib/rules/regex-guard.js +368 -0
- package/lib/templates.js +13 -6
- package/lib/tls.js +7 -0
- package/lib/whitebox/frameworks.js +8 -5
- package/lib/whitebox/secrets.js +20 -7
- package/package.json +7 -3
- package/skills/penthera/SKILL.md +23 -5
- package/src/cli/deps.js +60 -5
- package/src/cli/evidence.js +14 -4
- package/src/cli/install.js +14 -5
- package/src/cli/report-outputs.js +25 -2
- package/src/cli/ui.js +89 -0
- package/src/cli.js +19 -7
- package/src/report/audience.js +10 -10
- package/src/report/evidence.js +33 -14
- package/src/report/footer.js +11 -0
- package/src/report/markdown.js +6 -3
- package/src/report/md-escape.js +16 -0
- package/src/reporter.js +1 -0
- package/src/scan-url.js +5 -1
- package/src/ui/api/data.js +309 -0
- package/src/ui/api/demo.js +464 -0
- package/src/ui/api/routes.js +134 -0
- package/src/ui/api/runs.js +243 -0
- package/src/ui/api/summarize.js +451 -0
- package/src/ui/security.js +153 -0
- package/src/ui/server.js +186 -0
- package/src/ui/static/app.css +723 -0
- package/src/ui/static/favicon.svg +1 -0
- package/src/ui/static/index.html +57 -0
- package/src/ui/static/js/api.js +29 -0
- package/src/ui/static/js/app.js +138 -0
- package/src/ui/static/js/charts.js +207 -0
- package/src/ui/static/js/copy.js +79 -0
- package/src/ui/static/js/dom.js +120 -0
- package/src/ui/static/js/format.js +42 -0
- package/src/ui/static/js/theme.js +10 -0
- package/src/ui/static/js/typer.js +99 -0
- package/src/ui/static/js/views/ai.js +68 -0
- package/src/ui/static/js/views/deps.js +96 -0
- package/src/ui/static/js/views/evidence.js +115 -0
- package/src/ui/static/js/views/findings.js +165 -0
- package/src/ui/static/js/views/history.js +92 -0
- package/src/ui/static/js/views/overview.js +184 -0
- package/src/ui/static/js/views/run.js +349 -0
- package/src/ui/static/js/views/settings.js +67 -0
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a Python package's files come from, offline.
|
|
3
|
+
*
|
|
4
|
+
* - An installed distribution in a virtualenv: site-packages/*.dist-info
|
|
5
|
+
* names the package (METADATA) and lists its files with hashes (RECORD).
|
|
6
|
+
* No pip, no interpreter: the files are read as data.
|
|
7
|
+
* - pip's HTTP cache (http-v2/.../*.body): every wheel or sdist pip downloaded,
|
|
8
|
+
* stored as raw bytes. Matched to a lockfile by sha256, so what is read is
|
|
9
|
+
* exactly the artifact the lockfile pinned.
|
|
10
|
+
* - uv's cache (wheels-vN/pypi/<name>/): unpacked wheels. Accepted when uv
|
|
11
|
+
* recorded a hash the lockfile also lists, or when uv fetched it from the
|
|
12
|
+
* lockfile's own files.pythonhosted.org URL (those paths are
|
|
13
|
+
* content-addressed), and the unpacked files still match their RECORD.
|
|
14
|
+
* - A local path dependency: the folder itself.
|
|
15
|
+
*
|
|
16
|
+
* Sources share the files()/read()/size() interface of lib/deps/source.js.
|
|
17
|
+
*/
|
|
18
|
+
import { readFileSync, readdirSync, existsSync, statSync, lstatSync, realpathSync } from "node:fs";
|
|
19
|
+
import { join, relative, resolve, sep } from "node:path";
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
21
|
+
import { inflateRawSync, gunzipSync } from "node:zlib";
|
|
22
|
+
import { homedir } from "node:os";
|
|
23
|
+
import { DirSource, MapSource, untar } from "../source.js";
|
|
24
|
+
import { normalizeName } from "./lockfiles.js";
|
|
25
|
+
import { recordHash } from "./capabilities.js";
|
|
26
|
+
|
|
27
|
+
const MAX_FILES = 5000;
|
|
28
|
+
// Not part of a local package's source: environments, caches, build output.
|
|
29
|
+
const LOCAL_SKIP = new Set([".venv", "venv", ".tox", ".nox", "__pycache__", ".mypy_cache", ".pytest_cache", "build", "dist", ".eggs"]);
|
|
30
|
+
const MAX_ARCHIVE_BYTES = 512 * 1024 * 1024;
|
|
31
|
+
const MAX_UNPACKED_BYTES = 256 * 1024 * 1024;
|
|
32
|
+
|
|
33
|
+
// ── Virtualenvs ───────────────────────────────────────────────────────────
|
|
34
|
+
|
|
35
|
+
/** site-packages folders inside a virtualenv (posix lib/pythonX.Y, Windows Lib). */
|
|
36
|
+
export function sitePackagesDirs(venv) {
|
|
37
|
+
const out = [];
|
|
38
|
+
if (/site-packages[\\/]?$/.test(venv) && existsSync(venv)) return [resolve(venv)];
|
|
39
|
+
for (const lib of ["lib", "lib64", "Lib"]) {
|
|
40
|
+
const d = join(venv, lib);
|
|
41
|
+
let entries;
|
|
42
|
+
try { entries = readdirSync(d); } catch { continue; }
|
|
43
|
+
if (entries.includes("site-packages")) out.push(join(d, "site-packages"));
|
|
44
|
+
for (const e of entries.sort()) {
|
|
45
|
+
if (/^(?:python|pypy)\d/.test(e) && existsSync(join(d, e, "site-packages"))) out.push(join(d, e, "site-packages"));
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// lib and Lib are one folder on case-insensitive file systems: dedupe by inode.
|
|
49
|
+
const seen = new Set();
|
|
50
|
+
return out.map((p) => { try { return realpathSync(p); } catch { return p; } }).filter((p) => {
|
|
51
|
+
let key = p;
|
|
52
|
+
try { const st = statSync(p); key = `${st.dev}:${st.ino}`; } catch { /* keep path */ }
|
|
53
|
+
if (seen.has(key)) return false;
|
|
54
|
+
seen.add(key);
|
|
55
|
+
return true;
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The project's virtualenv: --venv, else .venv or venv beside the project. */
|
|
60
|
+
export function findVenv(root, explicit) {
|
|
61
|
+
if (explicit) {
|
|
62
|
+
const p = resolve(root, explicit);
|
|
63
|
+
if (!existsSync(p)) throw new Error(`--venv path not found: ${explicit}`);
|
|
64
|
+
if (!sitePackagesDirs(p).length) throw new Error(`No site-packages under ${explicit}; pass the virtualenv folder (the one with pyvenv.cfg).`);
|
|
65
|
+
return p;
|
|
66
|
+
}
|
|
67
|
+
for (const name of [".venv", "venv"]) {
|
|
68
|
+
const p = join(root, name);
|
|
69
|
+
if (existsSync(p) && sitePackagesDirs(p).length) return p;
|
|
70
|
+
}
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Parse RFC 822-style METADATA / PKG-INFO headers (first block only). */
|
|
75
|
+
export function parseMetadata(text) {
|
|
76
|
+
const head = String(text).replace(/\r\n/g, "\n").split("\n\n")[0];
|
|
77
|
+
const out = {};
|
|
78
|
+
for (const line of head.split("\n")) {
|
|
79
|
+
const m = line.match(/^([\w-]+):\s?(.*)$/);
|
|
80
|
+
if (m && !(m[1] in out)) out[m[1]] = m[2].trim();
|
|
81
|
+
}
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Parse a RECORD file (CSV: path,hash,size). Paths may be quoted. */
|
|
86
|
+
export function parseRecord(text) {
|
|
87
|
+
const out = [];
|
|
88
|
+
for (const line of String(text).split(/\r?\n/)) {
|
|
89
|
+
if (!line.trim()) continue;
|
|
90
|
+
let path;
|
|
91
|
+
let rest;
|
|
92
|
+
if (line.startsWith('"')) {
|
|
93
|
+
const end = line.indexOf('",', 1);
|
|
94
|
+
path = line.slice(1, end === -1 ? undefined : end).replace(/""/g, '"');
|
|
95
|
+
rest = end === -1 ? "" : line.slice(end + 2);
|
|
96
|
+
} else {
|
|
97
|
+
const i = line.lastIndexOf(",", line.lastIndexOf(",") - 1);
|
|
98
|
+
path = line.slice(0, i);
|
|
99
|
+
rest = line.slice(i + 1);
|
|
100
|
+
}
|
|
101
|
+
const [hash] = rest.split(",");
|
|
102
|
+
out.push({ path, hash: hash || null });
|
|
103
|
+
}
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Files of an installed distribution, as listed by its RECORD, read from disk. */
|
|
108
|
+
export class RecordSource {
|
|
109
|
+
constructor(siteDir, entries, kind = "installed") {
|
|
110
|
+
this.dir = siteDir;
|
|
111
|
+
this.kind = kind;
|
|
112
|
+
const root = resolve(siteDir);
|
|
113
|
+
// Only files inside site-packages: console-script wrappers in bin/ are generated by pip.
|
|
114
|
+
const ok = entries.filter((e) => e.path && !e.path.split(/[\\/]/).includes("..") && resolve(root, e.path).startsWith(root + sep));
|
|
115
|
+
this._files = [...new Set(ok.map((e) => e.path.replace(/\\/g, "/")))].sort().slice(0, MAX_FILES);
|
|
116
|
+
this.recordHashes = new Map(ok.filter((e) => e.hash).map((e) => [e.path.replace(/\\/g, "/"), e.hash]));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
files() { return this._files; }
|
|
120
|
+
|
|
121
|
+
size(rel) {
|
|
122
|
+
try { return lstatSync(join(this.dir, rel)).size; } catch { return 0; }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
read(rel) {
|
|
126
|
+
const p = join(this.dir, rel);
|
|
127
|
+
if (lstatSync(p).isSymbolicLink()) throw new Error("symlink");
|
|
128
|
+
return readFileSync(p);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Installed distributions in one site-packages folder.
|
|
134
|
+
* @returns {{ name, version, distInfo, source: RecordSource }[]} plus `unownedPth` on the array
|
|
135
|
+
*/
|
|
136
|
+
export function listInstalled(siteDir) {
|
|
137
|
+
let entries;
|
|
138
|
+
try { entries = readdirSync(siteDir).sort(); } catch { return Object.assign([], { unownedPth: [] }); }
|
|
139
|
+
const out = [];
|
|
140
|
+
const owned = new Set();
|
|
141
|
+
for (const e of entries) {
|
|
142
|
+
const isDist = e.endsWith(".dist-info");
|
|
143
|
+
const isEgg = e.endsWith(".egg-info");
|
|
144
|
+
if (!isDist && !isEgg) continue;
|
|
145
|
+
const meta = join(siteDir, e, isDist ? "METADATA" : "PKG-INFO");
|
|
146
|
+
let md;
|
|
147
|
+
try { md = parseMetadata(readFileSync(meta, "utf-8")); } catch { continue; }
|
|
148
|
+
if (!md.Name) continue;
|
|
149
|
+
let record = [];
|
|
150
|
+
if (isDist) {
|
|
151
|
+
try { record = parseRecord(readFileSync(join(siteDir, e, "RECORD"), "utf-8")); } catch { /* no RECORD */ }
|
|
152
|
+
} else {
|
|
153
|
+
try {
|
|
154
|
+
record = readFileSync(join(siteDir, e, "installed-files.txt"), "utf-8").split(/\r?\n/).filter(Boolean)
|
|
155
|
+
.map((p) => ({ path: relative(siteDir, resolve(siteDir, e, p)).split(sep).join("/"), hash: null }));
|
|
156
|
+
} catch { /* no file list */ }
|
|
157
|
+
}
|
|
158
|
+
if (!record.length) {
|
|
159
|
+
// No file list: fall back to the folder named after the distribution.
|
|
160
|
+
const guess = md.Name.replace(/-/g, "_");
|
|
161
|
+
if (existsSync(join(siteDir, guess))) record = new DirSource(join(siteDir, guess)).files().map((f) => ({ path: `${guess}/${f}`, hash: null }));
|
|
162
|
+
}
|
|
163
|
+
const src = new RecordSource(siteDir, record);
|
|
164
|
+
for (const f of src.files()) owned.add(f);
|
|
165
|
+
out.push({ name: normalizeName(md.Name), displayName: md.Name, version: md.Version || null, distInfo: e, source: src });
|
|
166
|
+
}
|
|
167
|
+
const unownedPth = entries.filter((e) => e.endsWith(".pth") && !owned.has(e));
|
|
168
|
+
return Object.assign(out, { unownedPth });
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ── Archives ──────────────────────────────────────────────────────────────
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Minimal zip reader for wheels: central directory, stored and deflated
|
|
175
|
+
* entries. Paths that climb out are dropped; total size is capped.
|
|
176
|
+
* @returns {Map<string, Buffer>}
|
|
177
|
+
*/
|
|
178
|
+
export function unzip(buf) {
|
|
179
|
+
const files = new Map();
|
|
180
|
+
let eocd = -1;
|
|
181
|
+
for (let i = buf.length - 22; i >= Math.max(0, buf.length - 65557); i--) {
|
|
182
|
+
if (buf.readUInt32LE(i) === 0x06054b50) { eocd = i; break; }
|
|
183
|
+
}
|
|
184
|
+
if (eocd === -1) throw new Error("not a zip file");
|
|
185
|
+
const count = buf.readUInt16LE(eocd + 10);
|
|
186
|
+
let off = buf.readUInt32LE(eocd + 16);
|
|
187
|
+
let total = 0;
|
|
188
|
+
for (let n = 0; n < count && off + 46 <= buf.length; n++) {
|
|
189
|
+
if (buf.readUInt32LE(off) !== 0x02014b50) break;
|
|
190
|
+
const method = buf.readUInt16LE(off + 10);
|
|
191
|
+
const csize = buf.readUInt32LE(off + 20);
|
|
192
|
+
const usize = buf.readUInt32LE(off + 24);
|
|
193
|
+
const nameLen = buf.readUInt16LE(off + 28);
|
|
194
|
+
const extraLen = buf.readUInt16LE(off + 30);
|
|
195
|
+
const commentLen = buf.readUInt16LE(off + 32);
|
|
196
|
+
const local = buf.readUInt32LE(off + 42);
|
|
197
|
+
const name = buf.subarray(off + 46, off + 46 + nameLen).toString("utf-8");
|
|
198
|
+
off += 46 + nameLen + extraLen + commentLen;
|
|
199
|
+
if (name.endsWith("/") || files.size >= MAX_FILES) continue;
|
|
200
|
+
const parts = name.split("/").filter(Boolean);
|
|
201
|
+
if (parts.includes("..") || name.startsWith("/")) continue;
|
|
202
|
+
if (buf.readUInt32LE(local) !== 0x04034b50) continue;
|
|
203
|
+
const start = local + 30 + buf.readUInt16LE(local + 26) + buf.readUInt16LE(local + 28);
|
|
204
|
+
const raw = buf.subarray(start, start + csize);
|
|
205
|
+
total += usize;
|
|
206
|
+
if (total > MAX_UNPACKED_BYTES) break;
|
|
207
|
+
let data;
|
|
208
|
+
if (method === 0) data = Buffer.from(raw);
|
|
209
|
+
else if (method === 8) data = inflateRawSync(raw, { maxOutputLength: Math.max(usize, 1) });
|
|
210
|
+
else continue;
|
|
211
|
+
files.set(parts.join("/"), data);
|
|
212
|
+
}
|
|
213
|
+
return files;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Unpack a wheel (zip) or sdist (.tar.gz) into a MapSource. */
|
|
217
|
+
export function sourceFromArtifact(bytes, kind) {
|
|
218
|
+
if (bytes.readUInt32LE(0) === 0x04034b50) return new MapSource(unzip(bytes), kind);
|
|
219
|
+
if (bytes[0] === 0x1f && bytes[1] === 0x8b) return new MapSource(untar(gunzipSync(bytes, { maxOutputLength: MAX_UNPACKED_BYTES })), kind);
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ── pip's HTTP cache ──────────────────────────────────────────────────────
|
|
224
|
+
|
|
225
|
+
/** pip's cache folder: PIP_CACHE_DIR, else the platform default. */
|
|
226
|
+
export function pipCacheDir(env = process.env, platform = process.platform) {
|
|
227
|
+
if (env.PIP_CACHE_DIR) return env.PIP_CACHE_DIR;
|
|
228
|
+
if (platform === "darwin") return join(homedir(), "Library", "Caches", "pip");
|
|
229
|
+
if (platform === "win32") return join(env.LOCALAPPDATA || join(homedir(), "AppData", "Local"), "pip", "Cache");
|
|
230
|
+
return join(env.XDG_CACHE_HOME || join(homedir(), ".cache"), "pip");
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** uv's cache folder: UV_CACHE_DIR, else the platform default. */
|
|
234
|
+
export function uvCacheDir(env = process.env, platform = process.platform) {
|
|
235
|
+
if (env.UV_CACHE_DIR) return env.UV_CACHE_DIR;
|
|
236
|
+
if (platform === "win32") return join(env.LOCALAPPDATA || join(homedir(), "AppData", "Local"), "uv", "cache");
|
|
237
|
+
return join(env.XDG_CACHE_HOME || join(homedir(), ".cache"), "uv");
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const pipIndexes = new Map();
|
|
241
|
+
|
|
242
|
+
/** sha256 -> body path for every artifact in pip's http-v2 cache (built once per folder). */
|
|
243
|
+
function pipIndex(dir) {
|
|
244
|
+
if (pipIndexes.has(dir)) return pipIndexes.get(dir);
|
|
245
|
+
const idx = new Map();
|
|
246
|
+
const walk = (d, depth) => {
|
|
247
|
+
let entries;
|
|
248
|
+
try { entries = readdirSync(d, { withFileTypes: true }); } catch { return; }
|
|
249
|
+
for (const e of entries) {
|
|
250
|
+
const p = join(d, e.name);
|
|
251
|
+
if (e.isDirectory() && depth < 8) walk(p, depth + 1);
|
|
252
|
+
else if (e.isFile() && e.name.endsWith(".body")) {
|
|
253
|
+
try {
|
|
254
|
+
if (statSync(p).size > MAX_ARCHIVE_BYTES) continue;
|
|
255
|
+
const bytes = readFileSync(p);
|
|
256
|
+
// Wheels (zip) and sdists (gzip) only; PEP 658 metadata files are text.
|
|
257
|
+
if (!(bytes.readUInt32LE(0) === 0x04034b50 || (bytes[0] === 0x1f && bytes[1] === 0x8b))) continue;
|
|
258
|
+
idx.set(createHash("sha256").update(bytes).digest("hex"), p);
|
|
259
|
+
} catch { /* unreadable entry */ }
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
walk(join(dir, "http-v2"), 0);
|
|
264
|
+
pipIndexes.set(dir, idx);
|
|
265
|
+
return idx;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const sha256Of = (hashes) => hashes.filter((h) => h.startsWith("sha256:")).map((h) => h.slice(7));
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* An artifact from pip's cache whose sha256 is one the lockfile lists.
|
|
272
|
+
* Prefers a wheel (what pip installs) over an sdist.
|
|
273
|
+
*/
|
|
274
|
+
export function sourceFromPipCache(pkg, dir = pipCacheDir()) {
|
|
275
|
+
const wanted = sha256Of(pkg.hashes || []);
|
|
276
|
+
if (!wanted.length || !existsSync(join(dir, "http-v2"))) return null;
|
|
277
|
+
const idx = pipIndex(dir);
|
|
278
|
+
const hits = wanted.map((h) => idx.get(h)).filter(Boolean).map((p) => readFileSync(p));
|
|
279
|
+
// Wheels ("PK") first.
|
|
280
|
+
hits.sort((a, b) => (b[0] === 0x50) - (a[0] === 0x50));
|
|
281
|
+
for (const bytes of hits) {
|
|
282
|
+
// Re-verify: the cache may have changed since it was indexed.
|
|
283
|
+
if (!wanted.includes(createHash("sha256").update(bytes).digest("hex"))) continue;
|
|
284
|
+
try {
|
|
285
|
+
const src = sourceFromArtifact(bytes, "pip-cache");
|
|
286
|
+
if (src) return src;
|
|
287
|
+
} catch { /* corrupt artifact */ }
|
|
288
|
+
}
|
|
289
|
+
return null;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// ── uv's cache ────────────────────────────────────────────────────────────
|
|
293
|
+
|
|
294
|
+
/** Decode the first msgpack value of a buffer (enough for uv's archive pointers). */
|
|
295
|
+
export function msgpackFirst(buf) {
|
|
296
|
+
let i = 0;
|
|
297
|
+
const str = (n) => { const s = buf.subarray(i, i + n).toString("utf-8"); i += n; return s; };
|
|
298
|
+
const bin = (n) => { const s = buf.subarray(i, i + n); i += n; return s; };
|
|
299
|
+
const read = (depth = 0) => {
|
|
300
|
+
if (depth > 16 || i >= buf.length) throw new Error("msgpack");
|
|
301
|
+
const b = buf[i++];
|
|
302
|
+
if (b <= 0x7f) return b;
|
|
303
|
+
if (b >= 0xe0) return b - 256;
|
|
304
|
+
if ((b & 0xe0) === 0xa0) return str(b & 0x1f);
|
|
305
|
+
if ((b & 0xf0) === 0x90) return Array.from({ length: b & 0x0f }, () => read(depth + 1));
|
|
306
|
+
if ((b & 0xf0) === 0x80) return Object.fromEntries(Array.from({ length: b & 0x0f }, () => [read(depth + 1), read(depth + 1)]));
|
|
307
|
+
switch (b) {
|
|
308
|
+
case 0xc0: return null;
|
|
309
|
+
case 0xc2: return false;
|
|
310
|
+
case 0xc3: return true;
|
|
311
|
+
case 0xcc: return buf[i++];
|
|
312
|
+
case 0xcd: i += 2; return buf.readUInt16BE(i - 2);
|
|
313
|
+
case 0xce: i += 4; return buf.readUInt32BE(i - 4);
|
|
314
|
+
case 0xd9: return str(buf[i++]);
|
|
315
|
+
case 0xda: i += 2; return str(buf.readUInt16BE(i - 2));
|
|
316
|
+
case 0xc4: return bin(buf[i++]);
|
|
317
|
+
case 0xc5: i += 2; return bin(buf.readUInt16BE(i - 2));
|
|
318
|
+
case 0xdc: { i += 2; const n = buf.readUInt16BE(i - 2); return Array.from({ length: n }, () => read(depth + 1)); }
|
|
319
|
+
case 0xde: { i += 2; const n = buf.readUInt16BE(i - 2); return Object.fromEntries(Array.from({ length: n }, () => [read(depth + 1), read(depth + 1)])); }
|
|
320
|
+
default: throw new Error("msgpack type");
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
return read();
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
const hexDigests = (v, out = []) => {
|
|
327
|
+
if (typeof v === "string") for (const m of v.matchAll(/[0-9a-f]{64}/g)) out.push(m[0]);
|
|
328
|
+
else if (Buffer.isBuffer(v) && v.length === 32) out.push(v.toString("hex"));
|
|
329
|
+
else if (Array.isArray(v)) v.forEach((x) => hexDigests(x, out));
|
|
330
|
+
else if (v && typeof v === "object") Object.values(v).forEach((x) => hexDigests(x, out));
|
|
331
|
+
return out;
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
/** True when every file of an unpacked wheel still matches its own RECORD. */
|
|
335
|
+
function recordIntact(dir) {
|
|
336
|
+
let distInfo;
|
|
337
|
+
try { distInfo = readdirSync(dir).find((e) => e.endsWith(".dist-info")); } catch { return false; }
|
|
338
|
+
if (!distInfo) return false;
|
|
339
|
+
let entries;
|
|
340
|
+
try { entries = parseRecord(readFileSync(join(dir, distInfo, "RECORD"), "utf-8")); } catch { return false; }
|
|
341
|
+
for (const e of entries) {
|
|
342
|
+
if (!e.hash) continue;
|
|
343
|
+
try {
|
|
344
|
+
const [algo] = e.hash.split("=");
|
|
345
|
+
if (recordHash(readFileSync(join(dir, e.path)), algo) !== e.hash) return false;
|
|
346
|
+
} catch { return false; }
|
|
347
|
+
}
|
|
348
|
+
return entries.length > 0;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* An unpacked wheel from uv's cache for pkg (name, version, hashes, files[].url).
|
|
353
|
+
*/
|
|
354
|
+
export function sourceFromUvCache(pkg, dir = uvCacheDir()) {
|
|
355
|
+
let wheelsRoots;
|
|
356
|
+
try { wheelsRoots = readdirSync(dir).filter((e) => /^wheels-v\d+$/.test(e)).sort().reverse(); } catch { return null; }
|
|
357
|
+
const wanted = sha256Of(pkg.hashes || []);
|
|
358
|
+
const urls = (pkg.files || []).map((f) => f.url).filter((u) => u && /^https:\/\/files\.pythonhosted\.org\/packages\//.test(u));
|
|
359
|
+
if (!wanted.length && !urls.length) return null;
|
|
360
|
+
for (const w of wheelsRoots) {
|
|
361
|
+
const pdir = join(dir, w, "pypi", pkg.name);
|
|
362
|
+
let entries;
|
|
363
|
+
try { entries = readdirSync(pdir); } catch { continue; }
|
|
364
|
+
for (const e of entries.filter((x) => x.startsWith(`${pkg.version}-`) && x.endsWith(".http")).sort()) {
|
|
365
|
+
let buf;
|
|
366
|
+
try { buf = readFileSync(join(pdir, e)); } catch { continue; }
|
|
367
|
+
let pointer;
|
|
368
|
+
try { pointer = msgpackFirst(buf); } catch { continue; }
|
|
369
|
+
if (!Array.isArray(pointer) || typeof pointer[0] !== "string") continue;
|
|
370
|
+
const recorded = hexDigests(pointer[1]);
|
|
371
|
+
const text = buf.toString("latin1");
|
|
372
|
+
const byHash = recorded.some((h) => wanted.includes(h));
|
|
373
|
+
const byUrl = !recorded.length && urls.some((u) => text.includes(u));
|
|
374
|
+
if (!byHash && !byUrl) continue;
|
|
375
|
+
const archive = join(dir, "archive-v0", pointer[0]);
|
|
376
|
+
if (!/^[\w-]+$/.test(pointer[0]) || !existsSync(archive)) continue;
|
|
377
|
+
if (!recordIntact(archive)) continue;
|
|
378
|
+
const src = new DirSource(archive);
|
|
379
|
+
src.kind = "uv-cache";
|
|
380
|
+
return src;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
return null;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/** A local path dependency (uv/poetry path source, `-e ./pkg` in requirements). */
|
|
387
|
+
export function sourceFromLocalPath(p) {
|
|
388
|
+
try {
|
|
389
|
+
if (!statSync(p).isDirectory()) {
|
|
390
|
+
return sourceFromArtifact(readFileSync(p), "local");
|
|
391
|
+
}
|
|
392
|
+
} catch { return null; }
|
|
393
|
+
const src = new DirSource(p, { skip: LOCAL_SKIP });
|
|
394
|
+
src.kind = "local";
|
|
395
|
+
return src;
|
|
396
|
+
}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A small TOML reader for Python lockfiles and pyproject.toml.
|
|
3
|
+
*
|
|
4
|
+
* Covers what poetry.lock, uv.lock, pdm.lock, Pipfile and pyproject.toml use:
|
|
5
|
+
* tables, arrays of tables, dotted keys, basic/literal/multi-line strings,
|
|
6
|
+
* integers, floats, booleans, dates (kept as strings), multi-line arrays and
|
|
7
|
+
* inline tables. Not a validating parser: malformed input yields a best-effort
|
|
8
|
+
* object or throws, never runs anything. Linear in the input size.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
class Reader {
|
|
12
|
+
constructor(src) {
|
|
13
|
+
this.s = src.replace(/\r\n/g, "\n");
|
|
14
|
+
this.i = 0;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
peek(n = 0) { return this.s[this.i + n]; }
|
|
18
|
+
eof() { return this.i >= this.s.length; }
|
|
19
|
+
|
|
20
|
+
error(msg) {
|
|
21
|
+
const line = this.s.slice(0, this.i).split("\n").length;
|
|
22
|
+
throw new Error(`TOML: ${msg} at line ${line}`);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Skip spaces and tabs (and, with nl, newlines and comments). */
|
|
26
|
+
ws(nl = false) {
|
|
27
|
+
while (!this.eof()) {
|
|
28
|
+
const c = this.peek();
|
|
29
|
+
if (c === " " || c === "\t") this.i++;
|
|
30
|
+
else if (nl && c === "\n") this.i++;
|
|
31
|
+
else if (c === "#") { while (!this.eof() && this.peek() !== "\n") this.i++; }
|
|
32
|
+
else break;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
key() {
|
|
37
|
+
this.ws();
|
|
38
|
+
const c = this.peek();
|
|
39
|
+
if (c === '"' || c === "'") return this.string();
|
|
40
|
+
const start = this.i;
|
|
41
|
+
while (!this.eof() && /[A-Za-z0-9_-]/.test(this.peek())) this.i++;
|
|
42
|
+
if (start === this.i) this.error("expected a key");
|
|
43
|
+
return this.s.slice(start, this.i);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
dottedKey() {
|
|
47
|
+
const parts = [this.key()];
|
|
48
|
+
this.ws();
|
|
49
|
+
while (this.peek() === ".") {
|
|
50
|
+
this.i++;
|
|
51
|
+
parts.push(this.key());
|
|
52
|
+
this.ws();
|
|
53
|
+
}
|
|
54
|
+
return parts;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
string() {
|
|
58
|
+
const q = this.peek();
|
|
59
|
+
const triple = this.s.startsWith(q.repeat(3), this.i);
|
|
60
|
+
if (triple) {
|
|
61
|
+
this.i += 3;
|
|
62
|
+
if (this.peek() === "\n") this.i++; // a newline right after the opener is trimmed
|
|
63
|
+
let out = "";
|
|
64
|
+
while (!this.eof()) {
|
|
65
|
+
if (this.s.startsWith(q.repeat(3), this.i) && this.peek(3) !== q) { this.i += 3; return out; }
|
|
66
|
+
const c = this.peek();
|
|
67
|
+
if (q === '"' && c === "\\") {
|
|
68
|
+
if (this.peek(1) === "\n" || /[ \t]/.test(this.peek(1) || "")) { // line-ending backslash
|
|
69
|
+
this.i++;
|
|
70
|
+
while (!this.eof() && /[ \t\n]/.test(this.peek())) this.i++;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
out += this.escape();
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
out += c;
|
|
77
|
+
this.i++;
|
|
78
|
+
}
|
|
79
|
+
this.error("unterminated string");
|
|
80
|
+
}
|
|
81
|
+
this.i++;
|
|
82
|
+
let out = "";
|
|
83
|
+
while (!this.eof() && this.peek() !== q) {
|
|
84
|
+
if (this.peek() === "\n") this.error("newline in string");
|
|
85
|
+
if (q === '"' && this.peek() === "\\") { out += this.escape(); continue; }
|
|
86
|
+
out += this.peek();
|
|
87
|
+
this.i++;
|
|
88
|
+
}
|
|
89
|
+
this.i++;
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
escape() {
|
|
94
|
+
const c = this.peek(1);
|
|
95
|
+
this.i += 2;
|
|
96
|
+
const map = { b: "\b", t: "\t", n: "\n", f: "\f", r: "\r", '"': '"', "\\": "\\" };
|
|
97
|
+
if (c in map) return map[c];
|
|
98
|
+
if (c === "u" || c === "U") {
|
|
99
|
+
const len = c === "u" ? 4 : 8;
|
|
100
|
+
const hex = this.s.slice(this.i, this.i + len);
|
|
101
|
+
this.i += len;
|
|
102
|
+
const cp = parseInt(hex, 16);
|
|
103
|
+
return Number.isFinite(cp) && cp <= 0x10ffff ? String.fromCodePoint(cp) : "";
|
|
104
|
+
}
|
|
105
|
+
return c || "";
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
value() {
|
|
109
|
+
this.ws();
|
|
110
|
+
const c = this.peek();
|
|
111
|
+
if (c === '"' || c === "'") return this.string();
|
|
112
|
+
if (c === "[") return this.array();
|
|
113
|
+
if (c === "{") return this.inlineTable();
|
|
114
|
+
const start = this.i;
|
|
115
|
+
while (!this.eof() && !/[,\]}\n#]/.test(this.peek())) this.i++;
|
|
116
|
+
const raw = this.s.slice(start, this.i).trim();
|
|
117
|
+
if (raw === "true") return true;
|
|
118
|
+
if (raw === "false") return false;
|
|
119
|
+
if (/^[+-]?\d[\d_]*$/.test(raw)) return parseInt(raw.replace(/_/g, ""), 10);
|
|
120
|
+
if (/^[+-]?(?:\d[\d_]*)?(?:\.\d[\d_]*)?(?:[eE][+-]?\d+)?$/.test(raw) && raw !== "") return parseFloat(raw.replace(/_/g, ""));
|
|
121
|
+
if (raw === "") this.error("expected a value");
|
|
122
|
+
return raw; // dates, inf, nan: kept as text
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
array() {
|
|
126
|
+
this.i++;
|
|
127
|
+
const out = [];
|
|
128
|
+
for (;;) {
|
|
129
|
+
this.ws(true);
|
|
130
|
+
if (this.peek() === "]") { this.i++; return out; }
|
|
131
|
+
if (this.eof()) this.error("unterminated array");
|
|
132
|
+
out.push(this.value());
|
|
133
|
+
this.ws(true);
|
|
134
|
+
if (this.peek() === ",") this.i++;
|
|
135
|
+
else if (this.peek() !== "]") this.error("expected , or ]");
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
inlineTable() {
|
|
140
|
+
this.i++;
|
|
141
|
+
const out = {};
|
|
142
|
+
for (;;) {
|
|
143
|
+
this.ws(true);
|
|
144
|
+
if (this.peek() === "}") { this.i++; return out; }
|
|
145
|
+
if (this.eof()) this.error("unterminated inline table");
|
|
146
|
+
const k = this.dottedKey();
|
|
147
|
+
this.ws();
|
|
148
|
+
if (this.peek() !== "=") this.error("expected =");
|
|
149
|
+
this.i++;
|
|
150
|
+
setPath(out, k, this.value());
|
|
151
|
+
this.ws(true);
|
|
152
|
+
if (this.peek() === ",") this.i++;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const isObj = (v) => v && typeof v === "object" && !Array.isArray(v);
|
|
158
|
+
|
|
159
|
+
function setPath(obj, keys, value) {
|
|
160
|
+
let t = obj;
|
|
161
|
+
for (const k of keys.slice(0, -1)) {
|
|
162
|
+
if (!isObj(t[k])) t[k] = {};
|
|
163
|
+
t = t[k];
|
|
164
|
+
}
|
|
165
|
+
t[keys[keys.length - 1]] = value;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Walk to a table header path; the last element of an array of tables counts as the table. */
|
|
169
|
+
function tableAt(root, keys, create) {
|
|
170
|
+
let t = root;
|
|
171
|
+
for (const k of keys) {
|
|
172
|
+
let next = t[k];
|
|
173
|
+
if (Array.isArray(next)) next = next[next.length - 1];
|
|
174
|
+
if (!isObj(next)) {
|
|
175
|
+
if (!create) return null;
|
|
176
|
+
next = t[k] = {};
|
|
177
|
+
}
|
|
178
|
+
t = next;
|
|
179
|
+
}
|
|
180
|
+
return t;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Parse TOML text into a plain object. */
|
|
184
|
+
export function parseToml(src) {
|
|
185
|
+
const r = new Reader(String(src));
|
|
186
|
+
const root = {};
|
|
187
|
+
let current = root;
|
|
188
|
+
for (;;) {
|
|
189
|
+
r.ws(true);
|
|
190
|
+
if (r.eof()) break;
|
|
191
|
+
if (r.peek() === "[") {
|
|
192
|
+
const isArray = r.peek(1) === "[";
|
|
193
|
+
r.i += isArray ? 2 : 1;
|
|
194
|
+
const keys = r.dottedKey();
|
|
195
|
+
r.ws();
|
|
196
|
+
if (!r.s.startsWith(isArray ? "]]" : "]", r.i)) r.error("expected ]");
|
|
197
|
+
r.i += isArray ? 2 : 1;
|
|
198
|
+
if (isArray) {
|
|
199
|
+
const parent = tableAt(root, keys.slice(0, -1), true);
|
|
200
|
+
const last = keys[keys.length - 1];
|
|
201
|
+
if (!Array.isArray(parent[last])) parent[last] = [];
|
|
202
|
+
const t = {};
|
|
203
|
+
parent[last].push(t);
|
|
204
|
+
current = t;
|
|
205
|
+
} else {
|
|
206
|
+
current = tableAt(root, keys, true);
|
|
207
|
+
}
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
const k = r.dottedKey();
|
|
211
|
+
r.ws();
|
|
212
|
+
if (r.peek() !== "=") r.error("expected =");
|
|
213
|
+
r.i++;
|
|
214
|
+
setPath(current, k, r.value());
|
|
215
|
+
r.ws();
|
|
216
|
+
if (!r.eof() && r.peek() !== "\n") r.error("expected end of line");
|
|
217
|
+
}
|
|
218
|
+
return root;
|
|
219
|
+
}
|