pi-lens 3.8.63 → 3.8.65

package/scripts/download-grammars.js

@@ -1,16 +1,33 @@
 #!/usr/bin/env node
 /**
- * Downloads tree-sitter WASM grammar files into node_modules/web-tree-sitter/grammars/.
- * Run automatically via postinstall. Skips gracefully if grammars already exist.
+ * Downloads tree-sitter WASM grammars, each verified against the committed
+ * provenance manifest (`scripts/grammars.lock.json`: package, version, per-grammar
+ * sha256) and stamped with a `<grammar>.wasm.json` sidecar. On a version bump or a
+ * hash mismatch the stale file is re-downloaded instead of skipped — closing the
+ * old skip-if-exists hole where an ABI-mismatched grammar could silently persist
+ * against the pinned `web-tree-sitter` (#177).
  *
- * Source: tree-sitter-wasms package on unpkg (mirrors npm registry artifacts).
+ * Modes:
+ *   node download-grammars.js                          # all grammars → web-tree-sitter/grammars
+ *   node download-grammars.js --core --dest grammars   # core set → ./grammars (used by `prepare`)
+ *   node download-grammars.js --write-manifest         # regenerate grammars.lock.json from the CDN
+ *
+ * Source: tree-sitter-wasms on unpkg (mirrors the npm registry artifacts).
+ *
+ * Progress goes to stderr — stdout must stay clean so `npm pack --silent` (which
+ * runs this via `prepare`) captures only the tarball name (#376/#380).
  */
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
+// The pinned tree-sitter-wasms release. Bump this, run `--write-manifest`, and
+// commit the regenerated grammars.lock.json to move to a new grammar set.
 const TREE_SITTER_WASMS_VERSION = "0.1.13";
-const BASE_URL = `https://unpkg.com/tree-sitter-wasms@${TREE_SITTER_WASMS_VERSION}/out`;
-const GRAMMARS = [
+const PACKAGE = "tree-sitter-wasms";
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const MANIFEST_PATH = join(SCRIPT_DIR, "grammars.lock.json");
+export const GRAMMARS = [
     // Core typed languages
     "tree-sitter-typescript.wasm",
     "tree-sitter-tsx.wasm",
@@ -40,33 +57,153 @@ const GRAMMARS = [
     "tree-sitter-yaml.wasm",
     "tree-sitter-zig.wasm",
 ];
+// The core set bundled into the tarball (via `prepare` → `grammars/`, shipped in
+// `files[]`) so the common languages parse offline on every package manager. The
+// long tail stays lazy-fetched at runtime. ~8.6MB uncompressed; ts/tsx dominate.
+export const CORE = [
+    "tree-sitter-typescript.wasm",
+    "tree-sitter-tsx.wasm",
+    "tree-sitter-javascript.wasm",
+    "tree-sitter-python.wasm",
+    "tree-sitter-go.wasm",
+    "tree-sitter-rust.wasm",
+    "tree-sitter-json.wasm",
+    "tree-sitter-yaml.wasm",
+    "tree-sitter-bash.wasm",
+    "tree-sitter-html.wasm",
+    "tree-sitter-css.wasm",
+    "tree-sitter-java.wasm",
+];
+/** `sha256:<hex>` digest of a buffer, matching the manifest/sidecar format. */
+export function sha256(buf) {
+    return `sha256:${createHash("sha256").update(buf).digest("hex")}`;
+}
+/** Read the committed provenance manifest (`scripts/grammars.lock.json`). */
+export function loadManifest() {
+    return JSON.parse(readFileSync(MANIFEST_PATH, "utf-8"));
+}
+/** Sidecar path for a grammar wasm file: `<grammar>.wasm.json`. */
+export function sidecarPathFor(wasmPath) {
+    return `${wasmPath}.json`;
+}
+/**
+ * Whether `filename` must be (re)downloaded into `destDir`: true if the grammar
+ * or its sidecar is missing, the sidecar's version differs from the manifest, or
+ * the sidecar's recorded hash differs from the expected one. Pure and cheap (it
+ * trusts the sidecar's recorded hash — the CI provenance guard re-hashes the
+ * bytes for the full integrity check).
+ */
+export function needsDownload(destDir, filename, manifest) {
+    const wasm = join(destDir, filename);
+    const sidecar = sidecarPathFor(wasm);
+    if (!existsSync(wasm) || !existsSync(sidecar))
+        return true;
+    let meta;
+    try {
+        meta = JSON.parse(readFileSync(sidecar, "utf-8"));
+    }
+    catch {
+        return true;
+    }
+    if (meta.version !== manifest.version)
+        return true;
+    const expected = manifest.grammars[filename];
+    if (expected && meta.sha256 !== expected)
+        return true;
+    return false;
+}
 function findGrammarsDir() {
-    const scriptDir = dirname(fileURLToPath(import.meta.url));
-    const pkgRoot = dirname(scriptDir);
-    // Prefer local node_modules next to this package
+    const pkgRoot = dirname(SCRIPT_DIR);
+    // Prefer local node_modules next to this package.
     return join(pkgRoot, "node_modules", "web-tree-sitter", "grammars");
 }
-async function downloadGrammar(destDir, filename) {
-    const dest = join(destDir, filename);
-    if (existsSync(dest)) {
-        console.log(`  skip  ${filename} (already exists)`);
+function baseUrl(version) {
+    return `https://unpkg.com/${PACKAGE}@${version}/out`;
+}
+async function fetchGrammar(version, filename) {
+    const res = await fetch(`${baseUrl(version)}/${filename}`);
+    if (!res.ok)
+        throw new Error(`HTTP ${res.status} fetching ${filename}`);
+    return Buffer.from(await res.arrayBuffer());
+}
+/**
+ * Download `filename` into `destDir` (if it isn't already verified), checking its
+ * bytes against the manifest and writing a provenance sidecar. Throws on an
+ * integrity mismatch — a grammar whose bytes don't match the pinned manifest is
+ * never written (guards CDN corruption / tampering / ABI drift).
+ */
+async function downloadGrammar(destDir, filename, manifest) {
+    if (!needsDownload(destDir, filename, manifest)) {
+        console.error(`  skip  ${filename} (verified)`);
         return;
     }
-    const url = `${BASE_URL}/${filename}`;
-    const res = await fetch(url);
-    if (!res.ok)
-        throw new Error(`HTTP ${res.status} fetching ${url}`);
-    const buf = await res.arrayBuffer();
-    writeFileSync(dest, Buffer.from(buf));
-    console.log(`  ok    ${filename}`);
+    const buf = await fetchGrammar(manifest.version, filename);
+    const actual = sha256(buf);
+    const expected = manifest.grammars[filename];
+    if (expected && actual !== expected) {
+        throw new Error(`integrity mismatch for ${filename}: expected ${expected}, got ${actual}`);
+    }
+    const wasm = join(destDir, filename);
+    writeFileSync(wasm, buf);
+    const sidecar = {
+        npmPackage: manifest.package,
+        version: manifest.version,
+        sha256: actual,
+    };
+    writeFileSync(sidecarPathFor(wasm), `${JSON.stringify(sidecar, null, 2)}\n`);
+    console.error(`  ok    ${filename}`);
+}
+function parseArgs(argv) {
+    const out = {
+        core: false,
+        dest: undefined,
+        writeManifest: false,
+    };
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === "--core")
+            out.core = true;
+        else if (argv[i] === "--write-manifest")
+            out.writeManifest = true;
+        else if (argv[i] === "--dest")
+            out.dest = argv[++i];
+    }
+    return out;
+}
+/** Regenerate grammars.lock.json by fetching every grammar and hashing it. */
+async function regenerateManifest() {
+    console.error(`Regenerating manifest from ${PACKAGE}@${TREE_SITTER_WASMS_VERSION} …`);
+    const grammars = {};
+    for (const g of GRAMMARS) {
+        grammars[g] = sha256(await fetchGrammar(TREE_SITTER_WASMS_VERSION, g));
+        console.error(`  hashed ${g}`);
+    }
+    const sorted = Object.fromEntries(Object.keys(grammars)
+        .sort()
+        .map((k) => [k, grammars[k]]));
+    const manifest = {
+        package: PACKAGE,
+        version: TREE_SITTER_WASMS_VERSION,
+        grammars: sorted,
+    };
+    writeFileSync(MANIFEST_PATH, `${JSON.stringify(manifest, null, 2)}\n`);
+    console.error(`Wrote ${MANIFEST_PATH} (${GRAMMARS.length} grammars).`);
 }
 async function main() {
-    const grammarsDir = findGrammarsDir();
+    const args = parseArgs(process.argv.slice(2));
+    if (args.writeManifest)
+        return regenerateManifest();
+    const manifest = loadManifest();
+    // `--dest` is relative to cwd (used by `prepare` to bundle into `./grammars/`);
+    // default is the installed web-tree-sitter/grammars dir.
+    const grammarsDir = args.dest
+        ? join(process.cwd(), args.dest)
+        : findGrammarsDir();
+    const list = args.core ? CORE : GRAMMARS;
     if (!existsSync(grammarsDir)) {
         mkdirSync(grammarsDir, { recursive: true });
     }
-    console.log(`Downloading tree-sitter grammars → ${grammarsDir}`);
-    const results = await Promise.allSettled(GRAMMARS.map((g) => downloadGrammar(grammarsDir, g)));
+    console.error(`Downloading ${args.core ? "core" : "all"} tree-sitter grammars (${list.length}) → ${grammarsDir}`);
+    const results = await Promise.allSettled(list.map((g) => downloadGrammar(grammarsDir, g, manifest)));
     const failed = results.filter((r) => r.status === "rejected");
     if (failed.length > 0) {
         for (const f of failed) {
@@ -75,11 +212,15 @@ async function main() {
         console.warn(`${failed.length} grammar(s) failed — tree-sitter analysis may be unavailable.`);
     }
     else {
-        console.log("All grammars downloaded successfully.");
+        console.error("All grammars downloaded successfully.");
     }
 }
-main().catch((err) => {
-    // Never fail the install — tree-sitter is optional
-    console.warn("Warning: grammar download failed:", err.message);
-    process.exit(0);
-});
+// Only run when invoked directly (not when imported by the provenance check or
+// tests) — the import guard keeps the exported helpers side-effect-free.
+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
+    main().catch((err) => {
+        // Never fail the install — tree-sitter is optional.
+        console.warn("Warning: grammar download failed:", err.message);
+        process.exit(0);
+    });
+}

package/scripts/grammars.lock.json

@@ -0,0 +1,32 @@
+{
+  "package": "tree-sitter-wasms",
+  "version": "0.1.13",
+  "grammars": {
+    "tree-sitter-bash.wasm": "sha256:807dcdb1380a59befb112ed8fbd3d3872c7fadaf5903a769282b50973b30696d",
+    "tree-sitter-c.wasm": "sha256:056b25072382f72deee2c64ec238ffc4bb8cf42844ef21502c0e70f03a8a0d66",
+    "tree-sitter-c_sharp.wasm": "sha256:6266a7e32d68a3459104d994dc848df15d5672b0ea8e86d327274b694f8e6991",
+    "tree-sitter-cpp.wasm": "sha256:f6afdf53bfd6de76557bb7edb624a3a3869e14d9a83b78433f93617ecee42527",
+    "tree-sitter-css.wasm": "sha256:5fc615467b1b98420ed7517e5bf9e1f88468132dd903d842dfb13714f6a1cb0c",
+    "tree-sitter-dart.wasm": "sha256:7f5364e4256cf7e55efd01dd52421ef2663caa8061b82659b7e4bf61064545ec",
+    "tree-sitter-elixir.wasm": "sha256:82e91b9759ddca30d8978ebbfa8e347b4451b64c931f9ae62112e6db9b8fac20",
+    "tree-sitter-go.wasm": "sha256:9963ca89b616eaf04b08a43bc1fb0f07b85395bec313330851f1f1ead2f755b6",
+    "tree-sitter-html.wasm": "sha256:11b3405c1543fb012f5ed7f8ee73125076dce8b168301e1e787e4c717da6b456",
+    "tree-sitter-java.wasm": "sha256:637aac4415fb39a211a4f4292d63c66b5ce9c32fa2cd35464af4f681d91b9a1f",
+    "tree-sitter-javascript.wasm": "sha256:63812b9e275d26851264734868d27a1656bd44a2ef6eb3e85e6b03728c595ab5",
+    "tree-sitter-json.wasm": "sha256:fdb5219abe058369e16897aaa11eecf47ef4f546752c3ddbac339cdd89e1e667",
+    "tree-sitter-kotlin.wasm": "sha256:b5cb00c8d06ed0f10f1dbe497205b437809d7e87db1f638721a8cfb30e044449",
+    "tree-sitter-lua.wasm": "sha256:75ef809136d610068c5b2135741d89f5df62690a3d55169203351cb7cc85727d",
+    "tree-sitter-ocaml.wasm": "sha256:60849b6320ee956233d77b017c65c45660e507d03ae70aa1bd5783458e2e9e18",
+    "tree-sitter-php.wasm": "sha256:55bb617b6f01e14bab997861f0b20a2420cf6ba3199ffeb295b9ec398966d8a3",
+    "tree-sitter-python.wasm": "sha256:9056d0fb0c337810d019fae350e8167786119da98f0f282aceae7ab89ee8253b",
+    "tree-sitter-ruby.wasm": "sha256:93a5022855314cdb45458c7bb026a24a0ebc3a5ff6439e542e881f14dfa13a39",
+    "tree-sitter-rust.wasm": "sha256:4409921a70d0aa5bec7d1d7ce809a557a8ee1cf6ace901e3ac6a76e62cfea903",
+    "tree-sitter-swift.wasm": "sha256:41c4fdb2249a3aa6d87eed0d383081ff09725c2248b4977043a43825980ffcc7",
+    "tree-sitter-toml.wasm": "sha256:7849ac8ce9d10a4684ca189ea8ad3654c20c38acb2d674a014a164398cbd37a2",
+    "tree-sitter-tsx.wasm": "sha256:6aa3b2c70e76f5d48eafef1093e9c4de383e13f2fdde2f4e9b98a378f6a8f1b6",
+    "tree-sitter-typescript.wasm": "sha256:8515404dceed38e1ed86aa34b09fcf3379fff1b4ff9dd3967bcd6d1eb5ac3d8f",
+    "tree-sitter-vue.wasm": "sha256:6244521bb3fb60f34ce5f677f2af81facb2c38691193985ca5fa85e1b6f29250",
+    "tree-sitter-yaml.wasm": "sha256:5dea7cfff83d41d8f87fb8e434e1a5b292c0d670bfcdc42cb2af420ef490dde5",
+    "tree-sitter-zig.wasm": "sha256:59cc4531aa661e2de4c5bc04e4045b6bdd5d2bfa75045cbda5f673102d140eef"
+  }
+}