sproutboat 0.10.2 → 0.11.0

package/src/sqlite.ts

@@ -13,17 +13,67 @@
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
 import { existsSync } from "node:fs";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
 import { resolve } from "node:path";
 import { homedir } from "node:os";
+import { resolveHostCompiler } from "./toolchain";
 
 export const SQLITE_VERSION = "3.50.4";
 
 /** Same cache root Zig uses, so one `rm -rf ~/.cache/sproutboat` clears everything. */
-const cacheDir = (): string => resolve(homedir(), ".cache/sproutboat", `sqlite-${SQLITE_VERSION}`);
+const cacheDir = (): string =>
+  resolve(
+    process.env.SPROUTBOAT_TOOLCHAIN_CACHE ?? resolve(homedir(), ".cache/sproutboat"),
+    `sqlite-${SQLITE_VERSION}`,
+  );
 const SQLITE_ZIP = `https://sqlite.org/2025/sqlite-amalgamation-3500400.zip`;
 const SQLITE_SHA256 = "1d3049dd0f830a025a53105fc79fd2ab9431aea99e137809d064d8ee8356b032";
 
+const digest = async (path: string): Promise<string> =>
+  createHash("sha256")
+    .update(await readFile(path))
+    .digest("hex");
+
+async function withSqliteLock<T>(dir: string, name: string, action: () => Promise<T>): Promise<T> {
+  await mkdir(dir, { recursive: true });
+  const lock = resolve(dir, `.${name}.lock`);
+  const deadline = Date.now() + 30_000;
+  for (;;) {
+    try {
+      await mkdir(lock);
+      break;
+    } catch {
+      if (Date.now() > deadline) throw new Error(`timed out waiting for SQLite cache lock ${lock}`);
+      try {
+        if (Date.now() - (await stat(lock)).mtimeMs > 5 * 60_000) await rm(lock, { recursive: true, force: true });
+      } catch {
+        /* publisher released the lock while it was being inspected */
+      }
+      await Bun.sleep(25);
+    }
+  }
+  try {
+    return await action();
+  } finally {
+    await rm(lock, { recursive: true, force: true });
+  }
+}
+
+async function validAmalgamation(dir: string): Promise<string | null> {
+  const source = resolve(dir, "sqlite3.c");
+  try {
+    // SAFETY: this manifest is private data written below. The immutable
+    // archive identity and the consumed source digest are both verified.
+    const manifest = JSON.parse(await readFile(resolve(dir, ".sproutboat-source"), "utf8")) as {
+      archiveSha256?: string;
+      sourceSha256?: string;
+    };
+    return manifest.archiveSha256 === SQLITE_SHA256 && manifest.sourceSha256 === (await digest(source)) ? source : null;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Compile flags. THREADSAFE=0 because a sprout serves one turn at a time;
  * DQS=0 rejects double-quoted string literals, which is the setting that turns
@@ -48,23 +98,39 @@ const run = (cmd: string, args: string[]): Promise<{ code: number; stderr: strin
 
 async function amalgamation(): Promise<string> {
   const dir = cacheDir();
-  const source = resolve(dir, "sqlite3.c");
-  if (existsSync(source)) return source;
-  await mkdir(dir, { recursive: true });
+  const ready = await validAmalgamation(dir);
+  if (ready) return ready;
+  return withSqliteLock(dir, "source", async () => (await validAmalgamation(dir)) ?? acquireAmalgamation(dir));
+}
 
-  const response = await fetch(SQLITE_ZIP);
-  if (!response.ok) throw new Error(`could not download SQLite ${SQLITE_VERSION}: HTTP ${response.status}`);
-  const zip = Buffer.from(await response.arrayBuffer());
-  const digest = createHash("sha256").update(zip).digest("hex");
-  if (digest !== SQLITE_SHA256) {
-    throw new Error(`SQLite amalgamation checksum mismatch: expected ${SQLITE_SHA256}, got ${digest}`);
+async function acquireAmalgamation(dir: string): Promise<string> {
+  const source = resolve(dir, "sqlite3.c");
+  const manifestPath = resolve(dir, ".sproutboat-source");
+  const stage = resolve(dir, `.source-${process.pid}-${crypto.randomUUID()}`);
+  try {
+    await mkdir(stage);
+    const response = await fetch(SQLITE_ZIP);
+    if (!response.ok) throw new Error(`could not download SQLite ${SQLITE_VERSION}: HTTP ${response.status}`);
+    const zip = Buffer.from(await response.arrayBuffer());
+    const archiveDigest = createHash("sha256").update(zip).digest("hex");
+    if (archiveDigest !== SQLITE_SHA256) {
+      throw new Error(`SQLite amalgamation checksum mismatch: expected ${SQLITE_SHA256}, got ${archiveDigest}`);
+    }
+    const zipPath = resolve(stage, "amalgamation.zip");
+    await writeFile(zipPath, zip);
+    const unzip = await run("unzip", ["-oqj", zipPath, "-d", stage]);
+    if (unzip.code !== 0) throw new Error(`could not unpack the SQLite amalgamation: ${unzip.stderr}`);
+    const stagedSource = resolve(stage, "sqlite3.c");
+    if (!existsSync(stagedSource)) throw new Error("the SQLite amalgamation did not contain sqlite3.c");
+    const sourceSha256 = await digest(stagedSource);
+    const stagedManifest = resolve(stage, ".sproutboat-source");
+    await writeFile(stagedManifest, JSON.stringify({ archiveSha256: archiveDigest, sourceSha256 }), { mode: 0o444 });
+    await rename(stagedSource, source);
+    await rename(stagedManifest, manifestPath);
+    return source;
+  } finally {
+    await rm(stage, { recursive: true, force: true });
   }
-  const zipPath = resolve(dir, "amalgamation.zip");
-  await writeFile(zipPath, zip);
-  const unzip = await run("unzip", ["-oqj", zipPath, "-d", dir]);
-  if (unzip.code !== 0) throw new Error(`could not unpack the SQLite amalgamation: ${unzip.stderr}`);
-  if (!existsSync(source)) throw new Error("the SQLite amalgamation did not contain sqlite3.c");
-  return source;
 }
 
 export type SqliteObjectInput = {
@@ -78,16 +144,58 @@ export type SqliteObjectInput = {
 export async function ensureSqliteObject(input: SqliteObjectInput): Promise<string> {
   const dir = cacheDir();
   const objectPath = resolve(dir, `sqlite3-${input.target}.o`);
-  if (existsSync(objectPath)) return objectPath;
-
   const source = await amalgamation();
-  const [cmd, prefix] =
+  const sourceSha256 = await digest(source);
+  const manifestPath = `${objectPath}.sproutboat-complete`;
+  const flags = ["-O2", ...SQLITE_DEFINES];
+  const [cmd, ...prefix] =
     input.target === "host"
-      ? (["cc", []] as const)
-      : ([input.zigBin ?? "zig", ["cc", "-target", "x86_64-linux-musl"]] as const);
-  const result = await run(cmd, [...prefix, "-c", source, "-o", objectPath, "-O2", ...SQLITE_DEFINES]);
-  if (result.code !== 0) throw new Error(`could not compile SQLite for ${input.target}:\n${result.stderr}`);
-  return objectPath;
+      ? (await resolveHostCompiler()).cc
+      : [input.zigBin ?? "zig", "cc", "-target", "x86_64-linux-musl"];
+  const command = [cmd, ...prefix];
+  const validCache = async (): Promise<boolean> => {
+    try {
+      // SAFETY: the manifest is private data written below. Its source, target,
+      // flags, and object digest must all agree before the cached object is used.
+      const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as {
+        sourceSha256?: string;
+        target?: string;
+        command?: string[];
+        flags?: string[];
+        objectSha256?: string;
+      };
+      return (
+        manifest.sourceSha256 === sourceSha256 &&
+        manifest.target === input.target &&
+        JSON.stringify(manifest.command) === JSON.stringify(command) &&
+        JSON.stringify(manifest.flags) === JSON.stringify(flags) &&
+        manifest.objectSha256 === (await digest(objectPath))
+      );
+    } catch {
+      return false;
+    }
+  };
+  if (await validCache()) return objectPath;
+  return withSqliteLock(dir, `object-${input.target}`, async () => {
+    if (await validCache()) return objectPath;
+    const stage = `${objectPath}.${process.pid}.${crypto.randomUUID()}`;
+    try {
+      const result = await run(cmd, [...prefix, "-c", source, "-o", stage, ...flags]);
+      if (result.code !== 0) throw new Error(`could not compile SQLite for ${input.target}:\n${result.stderr}`);
+      const objectSha256 = await digest(stage);
+      const stagedManifest = `${manifestPath}.${process.pid}.${crypto.randomUUID()}`;
+      await writeFile(
+        stagedManifest,
+        JSON.stringify({ sourceSha256, target: input.target, command, flags, objectSha256 }),
+        { mode: 0o444 },
+      );
+      await rename(stage, objectPath);
+      await rename(stagedManifest, manifestPath);
+      return objectPath;
+    } finally {
+      await rm(stage, { force: true });
+    }
+  });
 }
 
 /** The SQLite build stamp, for the artifact manifest's provenance string. */

package/src/surface.ts

@@ -102,6 +102,15 @@ export const COMMANDS: readonly Command[] = [
     summary:
       "Cross-compile the native-fetch sprout (Porffor + Zig). `--target host` builds for this machine instead, to run locally — not deployable. `--standalone` emits one executable carrying its own bindings, with SQLite compiled in (~2 MB) and no broker process.",
   },
+  {
+    name: "toolchain",
+    group: "Develop",
+    emoji: "🩺",
+    args: "doctor [--json]",
+    brief: "doctor",
+    summary:
+      "Report the selected compiler inputs, cache locations, overrides, and host prerequisites without downloading or compiling anything.",
+  },
 
   {
     name: "deploy",
@@ -207,9 +216,17 @@ export const ENV_VARS: readonly EnvVar[] = [
   {
     name: "CC",
     purpose:
-      "C compiler used to build uSockets for a host build (default `cc`). A host build needs one regardless: it is what Porffor compiles its own generated C with.",
+      "C compiler for a host build (default `cc` if present, else the vendored Zig cross-compiler targeting this host).",
+  },
+  {
+    name: "CXX",
+    purpose:
+      "C++ compiler for a host build's native-fetch link step (default `c++` if present, else the vendored Zig cross-compiler targeting this host).",
+  },
+  {
+    name: "AR",
+    purpose: "Archiver used to assemble uSockets.a for a host build (default `ar` if present, else Zig's).",
   },
-  { name: "AR", purpose: "Archiver used to assemble uSockets.a for a host build (default `ar`)." },
   { name: "SPROUTBOAT_COMPILE_TIMEOUT_MS", purpose: "Porffor compile timeout in ms (default 600000)." },
   {
     name: "SPROUTBOAT_TOOLCHAIN_CACHE",