spawnfile 0.1.17 → 0.1.18

package/dist/compiler/workspaceBundleArtifacts.js

@@ -1,5 +1,6 @@
 import { createHash } from "node:crypto";
-import { copyFile, mkdir, readFile, stat } from "node:fs/promises";
+import { constants } from "node:fs";
+import { copyFile, link, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { SpawnfileError } from "../shared/index.js";
 // Sanity bound for an operator-declared local tar; integrity comes from its digest, not its size.
@@ -75,32 +76,80 @@ export const validateWorkspaceBundleTar = (bytes) => {
     if (entries === 0)
         fail("Workspace bundle is empty");
 };
-export const stageWorkspaceBundles = async (outputDirectory, plan) => {
-    const bundles = new Map();
+export const workspaceBundleContextPath = (outputDirectory, identity) => path.join(outputDirectory, "container/workspace-bundles", `${identity.slice(7)}.tar`);
+/**
+ * Hard-links a read-only cached archive into the Docker context (cloning or
+ * copying across filesystems). Never writes through an existing name: with
+ * `clean: false` it may already be a link into the cache. Returns the staged
+ * path, or undefined when the cached archive no longer exists.
+ */
+export const linkBuiltBundle = async (archive, outputDirectory, identity) => {
+    const target = workspaceBundleContextPath(outputDirectory, identity);
+    await mkdir(path.dirname(target), { recursive: true });
+    await rm(target, { force: true });
+    try {
+        await link(archive, target);
+        return target;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return undefined;
+    }
+    try {
+        await copyFile(archive, target, constants.COPYFILE_FICLONE);
+        return target;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return undefined;
+        throw error;
+    }
+};
+/**
+ * Completes the bundle archives in the Docker context. Built bundles were
+ * already linked by `resolveWorkspaceBundles` (`built` maps their digests to
+ * the staged path). Every `source` tar is hashed, validated and staged from
+ * the exact bytes that were verified, even when a built bundle shares its
+ * digest; distinct source paths with identical bytes stage once.
+ */
+export const stageWorkspaceBundles = async (outputDirectory, plan, built = new Map()) => {
+    const sources = new Map();
+    let any = false;
     for (const node of plan.nodes)
         if (node.kind === "agent")
             for (const resource of node.value.workspaceResources ?? []) {
                 if (resource.kind !== "bundle")
                     continue;
-                const source = path.resolve(path.dirname(resource.scope.key), resource.source), prior = bundles.get(resource.sha256);
-                if (prior && prior !== source)
-                    throw new SpawnfileError("validation_error", "Workspace bundle digest maps to multiple sources");
-                bundles.set(resource.sha256, source);
+                any = true;
+                if (resource.sha256 === undefined || (resource.source === undefined && !built.has(resource.sha256))) {
+                    throw new SpawnfileError("compile_error", `Workspace bundle ${resource.id} has not been built`);
+                }
+                if (resource.source === undefined)
+                    continue;
+                const paths = sources.get(resource.sha256) ?? new Set();
+                paths.add(path.resolve(path.dirname(resource.scope.key), resource.source));
+                sources.set(resource.sha256, paths);
             }
-    if (bundles.size === 0)
+    if (!any)
         return false;
-    const destination = path.join(outputDirectory, "container/workspace-bundles");
-    await mkdir(destination, { recursive: true });
-    for (const [identity, source] of bundles) {
-        const info = await stat(source);
-        if (!info.isFile() || info.size < 1 || info.size > CAP)
-            throw new SpawnfileError("validation_error", "Workspace bundle must be a bounded regular tar file");
-        const bytes = await readFile(source);
-        const actual = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
-        if (actual !== identity)
-            throw new SpawnfileError("validation_error", "Workspace bundle checksum mismatch");
-        validateWorkspaceBundleTar(bytes);
-        await copyFile(source, path.join(destination, `${identity.slice(7)}.tar`));
+    await mkdir(path.join(outputDirectory, "container/workspace-bundles"), { recursive: true });
+    for (const [identity, paths] of sources) {
+        let verified;
+        for (const source of paths) {
+            const info = await stat(source);
+            if (!info.isFile() || info.size < 1 || info.size > CAP)
+                throw new SpawnfileError("validation_error", "Workspace bundle must be a bounded regular tar file");
+            const bytes = await readFile(source);
+            const actual = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+            if (actual !== identity)
+                throw new SpawnfileError("validation_error", "Workspace bundle checksum mismatch");
+            validateWorkspaceBundleTar(bytes);
+            verified ??= bytes;
+        }
+        // Replace by unlinking first: the name may be a hard link into the bundle cache.
+        const target = workspaceBundleContextPath(outputDirectory, identity);
+        await rm(target, { force: true });
+        await writeFile(target, verified);
     }
     return true;
 };

package/dist/compiler/workspaceBundleCache.d.ts

@@ -0,0 +1,23 @@
+import type { BundleTarSummary } from "./workspaceBundleTar.js";
+/** Built archives kept per cache directory; the least recently used beyond this are pruned. */
+export declare const WORKSPACE_BUNDLE_CACHE_LIMIT = 24;
+export interface CachedBundle extends BundleTarSummary {
+    key: string;
+    tarPath: string;
+}
+export declare const resolveWorkspaceBundleCacheDirectory: (override?: string) => string;
+/**
+ * A hit is a record whose archive still has the size, inode and mtime it had
+ * when it was written. Anything else — a missing, replaced or edited archive,
+ * an unreadable record — is a miss and the archive is rebuilt.
+ */
+export declare const lookupCachedBundle: (directory: string, key: string) => Promise<CachedBundle | undefined>;
+/** Builds into a private temporary file, validates it, then publishes archive and record by atomic rename. */
+export declare const storeBuiltBundle: (directory: string, key: string, build: (temporaryPath: string) => Promise<BundleTarSummary>) => Promise<CachedBundle>;
+/** An archive used this recently may belong to a concurrent compile that has not staged it yet. */
+export declare const WORKSPACE_BUNDLE_CACHE_GRACE_MS = 3600000;
+/**
+ * Removes the least recently used archives beyond the limit. Never removes
+ * one this compile uses, nor one any compile used within the grace period.
+ */
+export declare const pruneBundleCache: (directory: string, keep: ReadonlySet<string>, limit?: number, now?: number) => Promise<void>;

package/dist/compiler/workspaceBundleCache.js

@@ -0,0 +1,79 @@
+import { randomBytes } from "node:crypto";
+import { chmod, mkdir, readdir, readFile, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveSpawnfileHome } from "../auth/paths.js";
+import { validateWorkspaceBundleTar } from "./workspaceBundleArtifacts.js";
+const RECORD_VERSION = "spawnfile.workspace-bundle-cache.v1";
+/** Built archives kept per cache directory; the least recently used beyond this are pruned. */
+export const WORKSPACE_BUNDLE_CACHE_LIMIT = 24;
+export const resolveWorkspaceBundleCacheDirectory = (override) => override ?? path.join(resolveSpawnfileHome(), "cache", "workspace-bundles");
+const paths = (directory, key) => ({ record: path.join(directory, `${key}.json`), tar: path.join(directory, `${key}.tar`) });
+/**
+ * Archives are named by key AND digest: two concurrent builds of one key that
+ * produce different bytes publish two files, and each record names the file
+ * its digest describes, so a record can never pair one digest with another's bytes.
+ */
+const archiveName = (key, sha256) => `${key}.${sha256.slice(7)}.tar`;
+/**
+ * A hit is a record whose archive still has the size, inode and mtime it had
+ * when it was written. Anything else — a missing, replaced or edited archive,
+ * an unreadable record — is a miss and the archive is rebuilt.
+ */
+export const lookupCachedBundle = async (directory, key) => {
+    const location = paths(directory, key);
+    try {
+        const record = JSON.parse(await readFile(location.record, "utf8"));
+        if (typeof record.sha256 !== "string" || !/^sha256:[a-f0-9]{64}$/u.test(record.sha256))
+            return undefined;
+        const tarPath = path.join(directory, archiveName(key, record.sha256));
+        const info = await stat(tarPath);
+        if (record.version !== RECORD_VERSION || record.key !== key || !info.isFile() || (info.mode & 0o222) !== 0 || info.size !== record.size || info.ino !== record.ino || info.mtimeMs !== record.mtimeMs)
+            return undefined;
+        const now = new Date();
+        await utimes(location.record, now, now).catch(() => undefined);
+        return { contentBytes: record.contentBytes, fileCount: record.fileCount, key, sha256: record.sha256, size: record.size, tarPath };
+    }
+    catch {
+        return undefined;
+    }
+};
+/** Builds into a private temporary file, validates it, then publishes archive and record by atomic rename. */
+export const storeBuiltBundle = async (directory, key, build) => {
+    await mkdir(directory, { mode: 0o700, recursive: true });
+    const location = paths(directory, key), suffix = `${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
+    const temporaryTar = `${location.tar}.${suffix}`, temporaryRecord = `${location.record}.${suffix}`;
+    try {
+        const summary = await build(temporaryTar);
+        validateWorkspaceBundleTar(await readFile(temporaryTar));
+        // Read-only: staging hard-links this inode into build contexts, which must never write through to it.
+        await chmod(temporaryTar, 0o444);
+        const tarPath = path.join(directory, archiveName(key, summary.sha256));
+        await rename(temporaryTar, tarPath);
+        const info = await stat(tarPath);
+        const record = { ...summary, ino: info.ino, key, mtimeMs: info.mtimeMs, version: RECORD_VERSION };
+        await writeFile(temporaryRecord, `${JSON.stringify(record)}\n`, { mode: 0o600 });
+        await rename(temporaryRecord, location.record);
+        return { ...summary, key, tarPath };
+    }
+    finally {
+        await rm(temporaryTar, { force: true });
+        await rm(temporaryRecord, { force: true });
+    }
+};
+/** An archive used this recently may belong to a concurrent compile that has not staged it yet. */
+export const WORKSPACE_BUNDLE_CACHE_GRACE_MS = 3_600_000;
+/**
+ * Removes the least recently used archives beyond the limit. Never removes
+ * one this compile uses, nor one any compile used within the grace period.
+ */
+export const pruneBundleCache = async (directory, keep, limit = WORKSPACE_BUNDLE_CACHE_LIMIT, now = Date.now()) => {
+    const names = await readdir(directory).catch(() => []);
+    const records = await Promise.all(names.filter((name) => /^[a-f0-9]{64}\.json$/u.test(name)).map(async (name) => ({
+        key: name.slice(0, 64), used: (await stat(path.join(directory, name)).catch(() => undefined))?.mtimeMs ?? 0
+    })));
+    const stale = records.sort((left, right) => right.used - left.used).slice(limit).filter((record) => !keep.has(record.key) && now - record.used > WORKSPACE_BUNDLE_CACHE_GRACE_MS);
+    await Promise.all(stale.flatMap((record) => [
+        rm(path.join(directory, `${record.key}.json`), { force: true }),
+        ...names.filter((name) => name.startsWith(`${record.key}.`) && name.endsWith(".tar")).map((name) => rm(path.join(directory, name), { force: true }))
+    ]));
+};

package/dist/compiler/workspaceBundleDependencies.d.ts

@@ -0,0 +1,21 @@
+import type { WorkspaceBundleBuild } from "../manifest/index.js";
+import { type BundleBuildContext, type BundleBuildPlan } from "./workspaceBundleKey.js";
+type DependenciesSpec = NonNullable<WorkspaceBundleBuild["dependencies"]>;
+/**
+ * The lockfile is the whole dependency identity, so it must describe exactly
+ * what package.json asks for: a modern npm lock (v2+) whose root entry
+ * declares the same dependency sets, with no linked (workspace or file:)
+ * packages that would reach outside the install directory.
+ */
+export declare const assertNpmLockMatchesManifest: (manifestBytes: Buffer, lockBytes: Buffer, directory: string) => void;
+/**
+ * `dependencies`: package.json + package-lock.json installed with `npm ci` in
+ * a digest-pinned image on the TARGET platform, so native packages and
+ * platform-specific optional dependencies match the container, never the host.
+ * The key is the lockfile, the manifest, the install recipe (manager, flags,
+ * check command, pinned image = tool versions) and the platform; installs run
+ * only on a cache miss. The archive holds the installed `node_modules` tree;
+ * `.bin` symlinks and npm's hidden lockfile are left out.
+ */
+export declare const planDependenciesBundle: (spec: DependenciesSpec, context: BundleBuildContext) => Promise<BundleBuildPlan>;
+export {};

package/dist/compiler/workspaceBundleDependencies.js

@@ -0,0 +1,122 @@
+import { createHash } from "node:crypto";
+import { copyFile, lstat, mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+import { SpawnfileError } from "../shared/index.js";
+import { assertCommittedInputs, resolveBundleRoot, writeBundleFiles } from "./workspaceBundleFiles.js";
+import { computeRecipeBundleKey } from "./workspaceBundleKey.js";
+import { bundleContainerName, runContainerStep } from "./workspaceBundleRun.js";
+import { walkBuiltTree } from "./workspaceBundleTree.js";
+const MANIFEST_LIMIT = 67_108_864;
+const INSTALL_TIMEOUT_MS = 1_800_000;
+const DEPENDENCY_FIELDS = ["dependencies", "devDependencies", "optionalDependencies"];
+const fail = (message) => {
+    throw new SpawnfileError("validation_error", message);
+};
+const sha256 = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+const stable = (value) => JSON.stringify(value, (_key, item) => item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([left], [right]) => left.localeCompare(right))) : item);
+const readManifest = async (directory, name) => {
+    const file = path.join(directory, name), info = await lstat(file).catch(() => undefined);
+    if (!info?.isFile() || info.size > MANIFEST_LIMIT)
+        fail(`Dependency bundle needs a regular ${name} in ${directory}`);
+    return readFile(file);
+};
+/**
+ * The lockfile is the whole dependency identity, so it must describe exactly
+ * what package.json asks for: a modern npm lock (v2+) whose root entry
+ * declares the same dependency sets, with no linked (workspace or file:)
+ * packages that would reach outside the install directory.
+ */
+export const assertNpmLockMatchesManifest = (manifestBytes, lockBytes, directory) => {
+    const parse = (bytes, name) => {
+        try {
+            return JSON.parse(bytes.toString("utf8"));
+        }
+        catch {
+            return fail(`Dependency bundle ${name} is not valid JSON in ${directory}`);
+        }
+    };
+    const manifest = parse(manifestBytes, "package.json"), lock = parse(lockBytes, "package-lock.json");
+    const packages = lock.packages;
+    if (typeof lock.lockfileVersion !== "number" || lock.lockfileVersion < 2 || !packages?.[""])
+        fail(`Dependency bundle needs an npm lockfile v2 or later in ${directory}`);
+    for (const field of DEPENDENCY_FIELDS) {
+        if (stable(manifest[field] ?? {}) !== stable(packages[""][field] ?? {}))
+            fail(`Dependency bundle package.json ${field} differs from package-lock.json in ${directory}; run npm install`);
+    }
+    for (const [location, entry] of Object.entries(packages)) {
+        if (!location)
+            continue;
+        if (entry.link === true || !location.startsWith("node_modules/") || location.split("/").includes("..")) {
+            fail(`Dependency bundle lock entry ${location} is linked or outside node_modules; only registry installs are archived`);
+        }
+        // A cold install must fetch exactly what the key names: every fetched package needs content
+        // integrity, or a git source pinned to a full commit. Bundled packages ship inside their parent.
+        if (entry.inBundle === true)
+            continue;
+        const resolved = typeof entry.resolved === "string" ? entry.resolved : "";
+        const pinnedGit = /^git\+.+#[a-f0-9]{40}$/u.test(resolved);
+        if (!pinnedGit && !validIntegrity(entry.integrity)) {
+            fail(`Dependency bundle lock entry ${location} has no content integrity; its download is not pinned by the lockfile`);
+        }
+    }
+};
+// Base64 lengths of each SRI algorithm npm verifies; a token npm would parse as empty is not integrity.
+const SRI_LENGTHS = { sha1: 28, sha256: 44, sha384: 64, sha512: 88 };
+const validIntegrity = (value) => typeof value === "string" && value.trim() !== "" &&
+    value.trim().split(/\s+/u).every((token) => {
+        const match = /^(sha1|sha256|sha384|sha512)-([A-Za-z0-9+/]+={0,2})$/u.exec(token);
+        return match !== null && match[2].length === SRI_LENGTHS[match[1]];
+    });
+const inNodeModules = (relativePath) => relativePath === "node_modules" || relativePath.startsWith("node_modules/");
+/**
+ * `dependencies`: package.json + package-lock.json installed with `npm ci` in
+ * a digest-pinned image on the TARGET platform, so native packages and
+ * platform-specific optional dependencies match the container, never the host.
+ * The key is the lockfile, the manifest, the install recipe (manager, flags,
+ * check command, pinned image = tool versions) and the platform; installs run
+ * only on a cache miss. The archive holds the installed `node_modules` tree;
+ * `.bin` symlinks and npm's hidden lockfile are left out.
+ */
+export const planDependenciesBundle = async (spec, context) => {
+    const directory = await resolveBundleRoot(spec.directory);
+    // A release build installs exactly the committed manifests.
+    if (context.identity === "release")
+        await assertCommittedInputs(directory, ["package.json", "package-lock.json"]);
+    const [manifestBytes, lockBytes] = await Promise.all([readManifest(directory, "package.json"), readManifest(directory, "package-lock.json")]);
+    assertNpmLockMatchesManifest(manifestBytes, lockBytes, directory);
+    const dev = spec.dev ?? false, scripts = spec.scripts ?? true;
+    const recipe = {
+        check: spec.check ?? null, dev, image: spec.image, lock: sha256(lockBytes), manager: "npm", manifest: sha256(manifestBytes), scripts
+    };
+    return {
+        input: "dependencies",
+        key: computeRecipeBundleKey("dependencies", recipe, context.platform),
+        write: async (temporaryTar) => {
+            await mkdir(context.workRoot, { mode: 0o700, recursive: true });
+            const work = await mkdtemp(path.join(context.workRoot, "dependencies-"));
+            try {
+                await copyFile(path.join(directory, "package.json"), path.join(work, "package.json"));
+                await copyFile(path.join(directory, "package-lock.json"), path.join(work, "package-lock.json"));
+                if (sha256(await readFile(path.join(work, "package-lock.json"))) !== recipe.lock || sha256(await readFile(path.join(work, "package.json"))) !== recipe.manifest) {
+                    fail(`Dependency bundle manifests changed while the bundle was built: ${directory}`);
+                }
+                const step = (argv, label) => runContainerStep({
+                    argv, dockerCommand: context.dockerCommand, env: { npm_config_cache: "/tmp/spawnfile-home/.npm", npm_config_update_notifier: "false" },
+                    image: spec.image, mounts: [[work, "/spawnfile/install"]], name: bundleContainerName(), platform: context.platform, workdir: "/spawnfile/install"
+                }, INSTALL_TIMEOUT_MS, label, work);
+                const install = ["npm", "ci", "--no-audit", "--no-fund", dev ? "--include=dev" : "--omit=dev", ...(scripts ? [] : ["--ignore-scripts"])];
+                await step(install, `Dependency install for ${directory}`);
+                if (spec.check)
+                    await step(spec.check, `Dependency check for ${directory}`);
+                const input = await walkBuiltTree(work, {
+                    dropSymlink: (relativePath) => /(^|\/)\.bin\/[^/]+$/u.test(relativePath),
+                    skip: (relativePath) => !inNodeModules(relativePath) || relativePath === "node_modules/.package-lock.json"
+                });
+                return await writeBundleFiles(input, temporaryTar);
+            }
+            finally {
+                await rm(work, { force: true, recursive: true });
+            }
+        }
+    };
+};

package/dist/compiler/workspaceBundleFiles.d.ts

@@ -0,0 +1,51 @@
+import { type BundleFileMode, type BundleTarSummary } from "./workspaceBundleTar.js";
+export type BundleIdentityMode = "dev" | "release";
+/** One archived file. `identity` is a git object id (`git:`, content read from the object store) or a content digest (`sha256:`, read from disk). */
+export interface BundleFileEntry {
+    identity: string;
+    mode: BundleFileMode;
+    objectId?: string;
+    path: string;
+}
+export interface BundleFilesInput {
+    directory: string;
+    entries: BundleFileEntry[];
+    mode: BundleIdentityMode;
+}
+/**
+ * Exclude patterns match the path relative to the bundle root: `*` and `?`
+ * stay inside one segment, `**` crosses segments, and a pattern that matches
+ * a directory excludes everything beneath it.
+ */
+export declare const compileExcludePatterns: (patterns?: readonly string[]) => ((relativePath: string) => boolean);
+export declare const resolveBundleRoot: (directory: string) => Promise<string>;
+/**
+ * Release identity: the root must be clean against HEAD (ignoring excluded
+ * paths), and the identity is the committed tree's blob ids and modes. No
+ * input file is read.
+ */
+export declare const resolveReleaseFiles: (directory: string, exclude?: readonly string[]) => Promise<BundleFilesInput>;
+/**
+ * Release rule for recipe inputs that are named files rather than a tree:
+ * each must be tracked at HEAD with no staged, unstaged or untracked change.
+ */
+export declare const assertCommittedInputs: (directory: string, names: readonly string[]) => Promise<void>;
+/**
+ * A pinned revision: identity and bytes come from the tree of `ref` (resolved
+ * to a commit once), so the work tree's state is irrelevant and no clean
+ * checkout is needed. The same in dev and release compiles.
+ */
+export declare const resolveCommittedFiles: (directory: string, ref: string, exclude?: readonly string[]) => Promise<BundleFilesInput>;
+/**
+ * Dev identity: the working tree as it is now. Files git reports as unchanged
+ * keep their index blob id; modified and untracked (not ignored) files are
+ * hashed from disk, so an uncommitted edit or a new file always moves the key.
+ */
+export declare const resolveDevFiles: (directory: string, exclude?: readonly string[]) => Promise<BundleFilesInput>;
+/**
+ * Writes the archive for a resolved input set. Every `git:` entry is read from
+ * the object store, so its bytes are exactly the blob its identity names;
+ * every `sha256:` entry is read from disk and re-checked against its digest.
+ * The archive is therefore a pure function of the cache key.
+ */
+export declare const writeBundleFiles: (input: BundleFilesInput, outputPath: string) => Promise<BundleTarSummary>;