spawnfile 0.1.14 → 0.1.18

package/dist/compiler/workspaceBundleResolve.d.ts

@@ -0,0 +1,56 @@
+import type { CompileReportWorkspaceBundle } from "../report/index.js";
+import type { BundleIdentityMode } from "./workspaceBundleFiles.js";
+import type { BundleBuildContext, BundleBuildPlan } from "./workspaceBundleKey.js";
+import type { MoltnetTargetArchitecture } from "./moltnetReleaseAuthority.js";
+import type { CompilePlan } from "./types.js";
+import { type ResolvedWorkspaceResource } from "./workspaceResources.js";
+import type { WorkspaceBundleBuild } from "../manifest/index.js";
+/** One build plan per declaration; `build` paths are already absolute. */
+export declare const planBundleBuild: (build: WorkspaceBundleBuild, context: BundleBuildContext, options?: {
+    captureTools?: boolean;
+}) => Promise<BundleBuildPlan>;
+/** Explicit, else the same target-arch override the Moltnet binaries honour, else the host. */
+export declare const resolveBundleArchitecture: (architecture?: MoltnetTargetArchitecture) => MoltnetTargetArchitecture;
+export interface ResolveWorkspaceBundlesOptions {
+    /** Compile output directory; built archives are linked into its Docker context as soon as they resolve. */
+    outputDirectory: string;
+    /** Target container architecture; resolved like the Moltnet binaries when omitted. */
+    architecture?: MoltnetTargetArchitecture;
+    /** Defaults to `$SPAWNFILE_HOME/cache/workspace-bundles`. */
+    cacheDirectory?: string;
+    /** `release` requires clean inputs and takes identity from the committed tree; `dev` (default) hashes the work tree. */
+    identity?: BundleIdentityMode;
+    /** Docker CLI for dependency installs and image-run generated steps (cache misses only). */
+    dockerCommand?: string;
+}
+export interface ResolvedWorkspaceBundles {
+    /** Digest → built archive already linked into the Docker context this compile. */
+    built: Map<string, string>;
+    report: CompileReportWorkspaceBundle[];
+    builtCount: number;
+    reusedCount: number;
+}
+type BundleResource = Extract<ResolvedWorkspaceResource, {
+    kind: "bundle";
+}>;
+export { computeWorkspaceBundleKey } from "./workspaceBundleKey.js";
+/**
+ * A stable pin for a bundle without building it: the declared or computed
+ * archive digest for a prebuilt tar, else `bundle-key:<key>` — the cache key a
+ * dev compile for the target architecture would build under, which moves with
+ * every input, recipe and platform change — or, for a generated bundle,
+ * `bundle-recipe:<key>`, the same key with tools identified by argv only.
+ * Nothing is installed, generated or executed.
+ */
+export declare const pinWorkspaceBundle: (resource: BundleResource, options?: {
+    architecture?: MoltnetTargetArchitecture;
+    cacheDirectory?: string;
+    dockerCommand?: string;
+}) => Promise<string>;
+/**
+ * Gives every bundle resource in the plan a concrete digest. Declared-input
+ * bundles are built (or reused from the cache by key); prebuilt tars without a
+ * declared digest are hashed. Staging verifies every prebuilt tar again from
+ * the bytes it stages. Every bundle resource is reported with its digest.
+ */
+export declare const resolveWorkspaceBundles: (plan: CompilePlan, options: ResolveWorkspaceBundlesOptions) => Promise<ResolvedWorkspaceBundles>;

package/dist/compiler/workspaceBundleResolve.js

@@ -0,0 +1,144 @@
+import { createHash } from "node:crypto";
+import { readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { SpawnfileError } from "../shared/index.js";
+import { linkBuiltBundle, validateWorkspaceBundleTar } from "./workspaceBundleArtifacts.js";
+import { lookupCachedBundle, pruneBundleCache, resolveWorkspaceBundleCacheDirectory, storeBuiltBundle } from "./workspaceBundleCache.js";
+import { planDependenciesBundle } from "./workspaceBundleDependencies.js";
+import { planFilesBundle, planGeneratedBundle } from "./workspaceBundleGenerated.js";
+import { WORKSPACE_BUNDLE_MAX_BYTES } from "./workspaceBundleTar.js";
+import { resolveBundleBuildPaths } from "./workspaceResources.js";
+/** One build plan per declaration; `build` paths are already absolute. */
+export const planBundleBuild = (build, context, options = {}) => build.files ? planFilesBundle(build.files, context)
+    : build.dependencies ? planDependenciesBundle(build.dependencies, context)
+        : planGeneratedBundle(build.generated, context, options);
+/** Explicit, else the same target-arch override the Moltnet binaries honour, else the host. */
+export const resolveBundleArchitecture = (architecture) => {
+    const value = architecture ?? (process.env.SPAWNFILE_MOLTNET_TARGET_ARCH?.trim() || process.arch);
+    if (value === "amd64" || value === "x64" || value === "x86_64")
+        return "amd64";
+    if (value === "arm64" || value === "aarch64")
+        return "arm64";
+    throw new SpawnfileError("compile_error", `Workspace bundles do not support target architecture ${value}`);
+};
+export { computeWorkspaceBundleKey } from "./workspaceBundleKey.js";
+const declarationKey = (resource) => JSON.stringify({
+    build: resource.build ?? null, scope: path.dirname(resource.scope.key), source: resource.source ?? null
+});
+const hashPrebuilt = async (source) => {
+    const info = await stat(source).catch(() => undefined);
+    if (!info?.isFile() || info.size < 1 || info.size > WORKSPACE_BUNDLE_MAX_BYTES)
+        throw new SpawnfileError("validation_error", "Workspace bundle must be a bounded regular tar file");
+    const bytes = await readFile(source);
+    validateWorkspaceBundleTar(bytes);
+    return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+};
+/**
+ * A stable pin for a bundle without building it: the declared or computed
+ * archive digest for a prebuilt tar, else `bundle-key:<key>` — the cache key a
+ * dev compile for the target architecture would build under, which moves with
+ * every input, recipe and platform change — or, for a generated bundle,
+ * `bundle-recipe:<key>`, the same key with tools identified by argv only.
+ * Nothing is installed, generated or executed.
+ */
+export const pinWorkspaceBundle = async (resource, options = {}) => {
+    const base = path.dirname(resource.scope.key);
+    if (resource.source !== undefined)
+        return resource.sha256 ?? hashPrebuilt(path.resolve(base, resource.source));
+    const plan = await planBundleBuild(resolveBundleBuildPaths(resource.build, base), {
+        dockerCommand: options.dockerCommand ?? "docker", identity: "dev", outputReal: path.join(base, ".spawnfile-no-output"),
+        platform: `linux/${resolveBundleArchitecture(options.architecture)}`, workRoot: path.join(resolveWorkspaceBundleCacheDirectory(options.cacheDirectory), "work")
+    }, { captureTools: false });
+    // A generated bundle's build key includes captured tool output, which a pin never executes to obtain.
+    return `${plan.input === "generated" ? "bundle-recipe" : "bundle-key"}:${plan.key}`;
+};
+/**
+ * Gives every bundle resource in the plan a concrete digest. Declared-input
+ * bundles are built (or reused from the cache by key); prebuilt tars without a
+ * declared digest are hashed. Staging verifies every prebuilt tar again from
+ * the bytes it stages. Every bundle resource is reported with its digest.
+ */
+export const resolveWorkspaceBundles = async (plan, options) => {
+    const result = { built: new Map(), builtCount: 0, report: [], reusedCount: 0 };
+    const recording = plan.nodes.some((node) => node.kind === "agent" && node.value.training?.record !== undefined);
+    const cacheDirectory = resolveWorkspaceBundleCacheDirectory(options.cacheDirectory), { outputDirectory } = options;
+    const identity = options.identity ?? "dev", usedKeys = new Set();
+    const bundles = plan.nodes.flatMap((node) => node.kind === "agent" ? (node.value.workspaceResources ?? []).filter((resource) => resource.kind === "bundle") : []);
+    if (bundles.length === 0)
+        return result;
+    const outputReal = await realpath(outputDirectory).catch(() => path.resolve(outputDirectory));
+    let platform = "";
+    // Phase 1: every input snapshot is taken before this compile writes anything, so no
+    // staged archive (or other output under a bundle root) can leak into a later bundle.
+    const planned = new Map();
+    for (const resource of bundles) {
+        const memoKey = declarationKey(resource);
+        if (planned.has(memoKey))
+            continue;
+        const base = path.dirname(resource.scope.key);
+        planned.set(memoKey, (async () => {
+            if (resource.source !== undefined) {
+                return { facts: { origin: "prebuilt", sha256: resource.sha256 ?? await hashPrebuilt(path.resolve(base, resource.source)) } };
+            }
+            platform ||= `linux/${resolveBundleArchitecture(options.architecture)}`;
+            return planBundleBuild(resolveBundleBuildPaths(resource.build, base), {
+                dockerCommand: options.dockerCommand ?? "docker", identity, outputReal, platform, workRoot: path.join(cacheDirectory, "work")
+            });
+        })());
+        await planned.get(memoKey);
+    }
+    // Phase 2: build or reuse each archive and link it into this compile's context at once,
+    // so a concurrent compile's pruning cannot take it away. A hit pruned before linking is rebuilt.
+    const facts = new Map();
+    for (const [memoKey, pending] of planned) {
+        const step = await pending;
+        if ("facts" in step) {
+            facts.set(memoKey, step.facts);
+            continue;
+        }
+        const { input, key, write } = step;
+        usedKeys.add(key);
+        let cached = await lookupCachedBundle(cacheDirectory, key), staged = cached && await linkBuiltBundle(cached.tarPath, outputDirectory, cached.sha256);
+        if (cached && staged)
+            result.reusedCount += 1;
+        else {
+            cached = await storeBuiltBundle(cacheDirectory, key, write);
+            staged = await linkBuiltBundle(cached.tarPath, outputDirectory, cached.sha256);
+            if (!staged)
+                throw new SpawnfileError("compile_error", `Workspace bundle archive vanished from the cache while it was staged: ${key}`);
+            result.builtCount += 1;
+        }
+        result.built.set(cached.sha256, staged);
+        facts.set(memoKey, { cache_key: key, content_bytes: cached.contentBytes, file_count: cached.fileCount, identity, input, origin: "built", platform, sha256: cached.sha256 });
+    }
+    for (const node of plan.nodes) {
+        if (node.kind !== "agent" || !node.value.workspaceResources?.some((resource) => resource.kind === "bundle"))
+            continue;
+        node.value.workspaceResources = node.value.workspaceResources.map((resource) => {
+            if (resource.kind !== "bundle")
+                return resource;
+            const resolved = facts.get(declarationKey(resource));
+            if (resource.sha256 !== undefined && resource.sha256 !== resolved.sha256) {
+                throw new SpawnfileError("validation_error", `Workspace bundle ${resource.id} built to ${resolved.sha256}, but it declares ${resource.sha256}`);
+            }
+            if (recording) {
+                const declared_by = plan.nodes.find((entry) => entry.value.source === resource.scope.key)?.id;
+                let pin = result.report.find((entry) => entry.id === resource.id && entry.sha256 === resolved.sha256 && entry.declared_by === declared_by);
+                if (!pin) {
+                    pin = { id: resource.id, ...resolved, agent_node_ids: [], ...(declared_by ? { declared_by } : {}) };
+                    result.report.push(pin);
+                }
+                if (!pin.agent_node_ids.includes(node.id))
+                    pin.agent_node_ids.push(node.id);
+                pin.agent_node_ids.sort();
+            }
+            else if (!result.report.some((existing) => existing.id === resource.id && existing.sha256 === resolved.sha256))
+                result.report.push({ id: resource.id, ...resolved });
+            return { ...resource, sha256: resolved.sha256 };
+        });
+    }
+    if (usedKeys.size > 0)
+        await pruneBundleCache(cacheDirectory, usedKeys);
+    result.report.sort((left, right) => left.id.localeCompare(right.id) || left.sha256.localeCompare(right.sha256));
+    return result;
+};

package/dist/compiler/workspaceBundleRun.d.ts

@@ -0,0 +1,41 @@
+export interface BundleCommand {
+    argv: readonly string[];
+    cwd: string;
+    env?: NodeJS.ProcessEnv;
+    timeoutMs: number;
+    /** A container this step starts by name; removed (forcefully) if the step times out or fails. */
+    container?: {
+        dockerCommand: string;
+        name: string;
+    };
+}
+/**
+ * Runs one build step in its own process group; resolves with stdout, fails
+ * with the stderr tail on a non-zero exit, a signal or the timeout. A timeout
+ * kills the whole group, and a named container is removed, so neither stray
+ * descendants nor a detached container outlive the step.
+ */
+export declare const runBundleCommand: (command: BundleCommand, label: string) => Promise<Buffer>;
+export interface ContainerStep {
+    dockerCommand: string;
+    /** Container name, so a timed-out step can be removed. */
+    name: string;
+    image: string;
+    platform: string;
+    /** Host directory → container directory. */
+    mounts: ReadonlyArray<readonly [string, string]>;
+    workdir: string;
+    env?: Record<string, string>;
+    network?: boolean;
+    argv: readonly string[];
+}
+/**
+ * `docker run` for one build step on the target platform: removed on exit,
+ * running as the invoking user so outputs stay owned by them, no inherited
+ * host environment, and a throwaway HOME.
+ */
+export declare const containerArgv: (step: ContainerStep) => string[];
+/** A unique container name for one build step. */
+export declare const bundleContainerName: () => string;
+/** Runs a container step; the container is removed if the step fails or times out. */
+export declare const runContainerStep: (step: ContainerStep, timeoutMs: number, label: string, cwd: string) => Promise<Buffer>;

package/dist/compiler/workspaceBundleRun.js

@@ -0,0 +1,79 @@
+import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { SpawnfileError } from "../shared/index.js";
+const OUTPUT_TAIL = 4_096;
+const CLEANUP_TIMEOUT_MS = 30_000;
+/** Best-effort, bounded: a stalled Docker client is killed so cleanup never outlives its deadline. */
+const removeContainer = (container) => new Promise((resolve) => {
+    const child = spawn(container.dockerCommand, ["rm", "--force", container.name], { stdio: "ignore" });
+    const timer = setTimeout(() => { child.kill("SIGKILL"); resolve(); }, CLEANUP_TIMEOUT_MS);
+    const done = () => { clearTimeout(timer); resolve(); };
+    child.once("error", done);
+    child.once("close", done);
+});
+/**
+ * Runs one build step in its own process group; resolves with stdout, fails
+ * with the stderr tail on a non-zero exit, a signal or the timeout. A timeout
+ * kills the whole group, and a named container is removed, so neither stray
+ * descendants nor a detached container outlive the step.
+ */
+export const runBundleCommand = (command, label) => new Promise((resolve, reject) => {
+    const [file, ...args] = command.argv;
+    const child = spawn(file, args, { cwd: command.cwd, detached: true, env: command.env ?? process.env, stdio: ["ignore", "pipe", "pipe"] });
+    const stdout = [];
+    let stderr = "", timedOut = false, settled = false;
+    const killGroup = () => { try {
+        process.kill(-child.pid, "SIGKILL");
+    }
+    catch {
+        child.kill("SIGKILL");
+    } };
+    const finish = async (error) => {
+        if (settled)
+            return;
+        settled = true;
+        clearTimeout(timer);
+        if (error && command.container)
+            await removeContainer(command.container);
+        if (error)
+            reject(error);
+        else
+            resolve(Buffer.concat(stdout));
+    };
+    const timer = setTimeout(() => { timedOut = true; killGroup(); void finish(new SpawnfileError("compile_error", `${label} timed out after ${command.timeoutMs} ms${stderr.trim() ? `: ${stderr.trim()}` : ""}`)); }, command.timeoutMs);
+    child.stdout.on("data", (chunk) => { stdout.push(chunk); });
+    child.stderr.on("data", (chunk) => { stderr = (stderr + chunk.toString("utf8")).slice(-OUTPUT_TAIL); });
+    child.once("error", (error) => { void finish(new SpawnfileError("compile_error", `${label} could not start ${file}: ${error.message}`)); });
+    child.once("close", (code, signal) => {
+        if (timedOut)
+            return;
+        if (code === 0) {
+            void finish();
+            return;
+        }
+        killGroup();
+        const why = signal ? `was killed by ${signal}` : `exited ${code}`;
+        void finish(new SpawnfileError("compile_error", `${label} ${why}${stderr.trim() ? `: ${stderr.trim()}` : ""}`));
+    });
+});
+/**
+ * `docker run` for one build step on the target platform: removed on exit,
+ * running as the invoking user so outputs stay owned by them, no inherited
+ * host environment, and a throwaway HOME.
+ */
+export const containerArgv = (step) => {
+    const user = typeof process.getuid === "function" && typeof process.getgid === "function" ? [`--user`, `${process.getuid()}:${process.getgid()}`] : [];
+    const env = { HOME: "/tmp/spawnfile-home", ...step.env };
+    return [
+        step.dockerCommand, "run", "--rm", "--name", step.name, "--platform", step.platform, ...user,
+        ...(step.network === false ? ["--network", "none"] : []),
+        ...step.mounts.flatMap(([host, container]) => ["--volume", `${host}:${container}`]),
+        "--workdir", step.workdir,
+        ...Object.entries(env).sort(([left], [right]) => left.localeCompare(right)).flatMap(([key, value]) => ["--env", `${key}=${value}`]),
+        step.image, ...step.argv
+    ];
+};
+/** A unique container name for one build step. */
+export const bundleContainerName = () => `spawnfile-bundle-${randomBytes(8).toString("hex")}`;
+/** Runs a container step; the container is removed if the step fails or times out. */
+export const runContainerStep = (step, timeoutMs, label, cwd) => runBundleCommand({ argv: containerArgv(step), container: { dockerCommand: step.dockerCommand, name: step.name }, cwd, timeoutMs }, label);

package/dist/compiler/workspaceBundleTar.d.ts

@@ -0,0 +1,47 @@
+/** Bumped whenever the bytes this writer emits for the same entries change. */
+export declare const WORKSPACE_BUNDLE_TAR_WRITER = "spawnfile.workspace-bundle-tar.v1";
+/** Same bounds the prebuilt-tar staging path enforces. */
+export declare const WORKSPACE_BUNDLE_MAX_BYTES = 536870912;
+export declare const WORKSPACE_BUNDLE_MAX_ENTRIES = 65536;
+export type BundleFileMode = 0o644 | 0o755;
+/** Git records one bit of mode: executable or not. Every emitted mode collapses to one of two values. */
+export declare const normalizeBundleMode: (rawMode: number) => BundleFileMode;
+/** Splits a path across the ustar prefix (155 bytes) and name (100 bytes) fields, or refuses it. */
+export declare const splitUstarPath: (name: string) => {
+    base: Buffer;
+    prefix: Buffer;
+};
+/** One deterministic ustar header: uid/gid 0, mtime 0, owner root, regular file. */
+export declare const ustarHeader: (name: string, size: number, mode: BundleFileMode) => Buffer;
+export interface BundleTarSummary {
+    contentBytes: number;
+    fileCount: number;
+    sha256: `sha256:${string}`;
+    size: number;
+}
+/**
+ * Streams entries into a tar file while hashing it. Callers add entries in
+ * sorted path order; the writer refuses an unsorted or duplicate path so two
+ * builds of the same entry set cannot disagree on order.
+ */
+export declare class BundleTarWriter {
+    private readonly hash;
+    private pending;
+    private pendingBytes;
+    private size;
+    private contentBytes;
+    private fileCount;
+    private remaining;
+    private currentSize;
+    private lastPath;
+    private readonly handle;
+    private constructor();
+    static create(filePath: string): Promise<BundleTarWriter>;
+    begin(name: string, size: number, mode: BundleFileMode): Promise<void>;
+    data(chunk: Buffer): Promise<void>;
+    end(): Promise<void>;
+    finish(): Promise<BundleTarSummary>;
+    abort(): Promise<void>;
+    private push;
+    private flush;
+}

package/dist/compiler/workspaceBundleTar.js

@@ -0,0 +1,140 @@
+import { createHash } from "node:crypto";
+import { open } from "node:fs/promises";
+import { SpawnfileError } from "../shared/index.js";
+/** Bumped whenever the bytes this writer emits for the same entries change. */
+export const WORKSPACE_BUNDLE_TAR_WRITER = "spawnfile.workspace-bundle-tar.v1";
+/** Same bounds the prebuilt-tar staging path enforces. */
+export const WORKSPACE_BUNDLE_MAX_BYTES = 536_870_912;
+export const WORKSPACE_BUNDLE_MAX_ENTRIES = 65_536;
+const BLOCK = 512;
+const FLUSH_BYTES = 1_048_576;
+/** Git records one bit of mode: executable or not. Every emitted mode collapses to one of two values. */
+export const normalizeBundleMode = (rawMode) => (rawMode & 0o111) ? 0o755 : 0o644;
+const fail = (message) => {
+    throw new SpawnfileError("validation_error", message);
+};
+const octal = (value, width) => `${value.toString(8).padStart(width - 1, "0")}\0`;
+/** Splits a path across the ustar prefix (155 bytes) and name (100 bytes) fields, or refuses it. */
+export const splitUstarPath = (name) => {
+    const raw = Buffer.from(name, "utf8");
+    if (raw.length <= 100)
+        return { base: raw, prefix: Buffer.alloc(0) };
+    for (let index = name.indexOf("/"); index > 0; index = name.indexOf("/", index + 1)) {
+        const prefix = Buffer.from(name.slice(0, index), "utf8"), base = Buffer.from(name.slice(index + 1), "utf8");
+        if (prefix.length <= 155 && base.length <= 100 && base.length > 0)
+            return { base, prefix };
+    }
+    return fail(`Workspace bundle path exceeds ustar bounds: ${name}`);
+};
+/** One deterministic ustar header: uid/gid 0, mtime 0, owner root, regular file. */
+export const ustarHeader = (name, size, mode) => {
+    const bytes = Buffer.alloc(BLOCK), { base, prefix } = splitUstarPath(name);
+    base.copy(bytes, 0);
+    bytes.write(octal(mode, 8), 100, "ascii");
+    bytes.write(octal(0, 8), 108, "ascii");
+    bytes.write(octal(0, 8), 116, "ascii");
+    bytes.write(octal(size, 12), 124, "ascii");
+    bytes.write(octal(0, 12), 136, "ascii");
+    bytes.fill(0x20, 148, 156);
+    bytes[156] = 0x30;
+    bytes.write("ustar\0", 257, "ascii");
+    bytes.write("00", 263, "ascii");
+    bytes.write("root", 265, "ascii");
+    bytes.write("root", 297, "ascii");
+    prefix.copy(bytes, 345);
+    let sum = 0;
+    for (const byte of bytes)
+        sum += byte;
+    bytes.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148, "ascii");
+    return bytes;
+};
+/**
+ * Streams entries into a tar file while hashing it. Callers add entries in
+ * sorted path order; the writer refuses an unsorted or duplicate path so two
+ * builds of the same entry set cannot disagree on order.
+ */
+export class BundleTarWriter {
+    hash = createHash("sha256");
+    pending = [];
+    pendingBytes = 0;
+    size = 0;
+    contentBytes = 0;
+    fileCount = 0;
+    remaining = 0;
+    currentSize = 0;
+    lastPath;
+    handle;
+    constructor(handle) {
+        this.handle = handle;
+    }
+    static async create(filePath) {
+        return new BundleTarWriter(await open(filePath, "wx", 0o600));
+    }
+    async begin(name, size, mode) {
+        if (this.remaining !== 0)
+            fail("Workspace bundle entry was not fully written");
+        if (this.lastPath !== undefined && name <= this.lastPath)
+            fail(`Workspace bundle entries are not in strict path order: ${name}`);
+        this.fileCount += 1;
+        if (this.fileCount > WORKSPACE_BUNDLE_MAX_ENTRIES)
+            fail("Workspace bundle exceeds the maximum entry count");
+        this.contentBytes += size;
+        // Header, padded content and the two-block terminator must all fit.
+        if (this.size + BLOCK + Math.ceil(size / BLOCK) * BLOCK + BLOCK * 2 > WORKSPACE_BUNDLE_MAX_BYTES)
+            fail("Workspace bundle exceeds the maximum archive size");
+        this.lastPath = name;
+        this.remaining = size;
+        this.currentSize = size;
+        await this.push(ustarHeader(name, size, mode));
+    }
+    async data(chunk) {
+        if (chunk.length > this.remaining)
+            fail("Workspace bundle input grew while it was archived");
+        this.remaining -= chunk.length;
+        await this.push(chunk);
+    }
+    async end() {
+        if (this.remaining !== 0)
+            fail("Workspace bundle input shrank while it was archived");
+        const padding = (BLOCK - (this.currentSize % BLOCK)) % BLOCK;
+        if (padding)
+            await this.push(Buffer.alloc(padding));
+    }
+    async finish() {
+        try {
+            if (this.remaining !== 0)
+                fail("Workspace bundle entry was not fully written");
+            if (this.fileCount === 0)
+                fail("Workspace bundle has no input files");
+            await this.push(Buffer.alloc(BLOCK * 2));
+            await this.flush();
+            await this.handle.sync();
+        }
+        finally {
+            await this.handle.close();
+        }
+        return { contentBytes: this.contentBytes, fileCount: this.fileCount, sha256: `sha256:${this.hash.digest("hex")}`, size: this.size };
+    }
+    async abort() {
+        await this.handle.close().catch(() => undefined);
+    }
+    async push(chunk) {
+        this.hash.update(chunk);
+        this.size += chunk.length;
+        this.pending.push(chunk);
+        this.pendingBytes += chunk.length;
+        if (this.pendingBytes >= FLUSH_BYTES)
+            await this.flush();
+    }
+    async flush() {
+        if (this.pendingBytes === 0)
+            return;
+        const buffer = Buffer.concat(this.pending, this.pendingBytes);
+        this.pending = [];
+        this.pendingBytes = 0;
+        for (let offset = 0; offset < buffer.length;) {
+            const { bytesWritten } = await this.handle.write(buffer, offset, buffer.length - offset);
+            offset += bytesWritten;
+        }
+    }
+}

package/dist/compiler/workspaceBundleTree.d.ts

@@ -0,0 +1,15 @@
+import type { BundleFilesInput } from "./workspaceBundleFiles.js";
+export interface WalkOptions {
+    /** Root-relative paths left out of the archive entirely. */
+    skip?: (relativePath: string) => boolean;
+    /** Symlinks for which this returns true are dropped; any other symlink fails the build. */
+    dropSymlink?: (relativePath: string) => boolean;
+}
+/**
+ * Every regular file under `directory`, hashed, as an archive input set.
+ * Built output (installed dependencies, generated files) has no git
+ * identity, so each file is content-addressed. Symlinks, devices and sockets
+ * cannot be archived safely and fail unless explicitly dropped. Entry and byte
+ * budgets are enforced while walking, before the archive is written.
+ */
+export declare const walkBuiltTree: (directory: string, options?: WalkOptions) => Promise<BundleFilesInput>;

package/dist/compiler/workspaceBundleTree.js

@@ -0,0 +1,58 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { lstat, readdir } from "node:fs/promises";
+import path from "node:path";
+import { SpawnfileError } from "../shared/index.js";
+import { normalizeBundleMode, WORKSPACE_BUNDLE_MAX_BYTES, WORKSPACE_BUNDLE_MAX_ENTRIES } from "./workspaceBundleTar.js";
+const fail = (message) => {
+    throw new SpawnfileError("validation_error", message);
+};
+/**
+ * Every regular file under `directory`, hashed, as an archive input set.
+ * Built output (installed dependencies, generated files) has no git
+ * identity, so each file is content-addressed. Symlinks, devices and sockets
+ * cannot be archived safely and fail unless explicitly dropped. Entry and byte
+ * budgets are enforced while walking, before the archive is written.
+ */
+export const walkBuiltTree = async (directory, options = {}) => {
+    const root = await lstat(directory);
+    if (!root.isDirectory())
+        fail(`Built workspace bundle output root was replaced by something other than a directory: ${directory}`);
+    const entries = [];
+    let bytes = 0;
+    const visit = async (relative) => {
+        for (const name of (await readdir(path.join(directory, relative))).sort()) {
+            const child = relative ? `${relative}/${name}` : name;
+            if (options.skip?.(child))
+                continue;
+            const info = await lstat(path.join(directory, child));
+            if (info.isDirectory()) {
+                await visit(child);
+                continue;
+            }
+            if (info.isSymbolicLink()) {
+                if (options.dropSymlink?.(child))
+                    continue;
+                fail(`Built workspace bundle output contains a symlink: ${child}`);
+            }
+            if (!info.isFile())
+                fail(`Built workspace bundle output contains an unsupported file type: ${child}`);
+            if (child.includes("\\"))
+                fail(`Built workspace bundle output path cannot be archived safely: ${child}`);
+            bytes += info.size;
+            if (bytes > WORKSPACE_BUNDLE_MAX_BYTES)
+                fail("Workspace bundle exceeds the maximum archive size");
+            if (entries.length >= WORKSPACE_BUNDLE_MAX_ENTRIES)
+                fail("Workspace bundle exceeds the maximum entry count");
+            const hash = createHash("sha256");
+            for await (const chunk of createReadStream(path.join(directory, child)))
+                hash.update(chunk);
+            entries.push({ identity: `sha256:${hash.digest("hex")}`, mode: normalizeBundleMode(info.mode), path: child });
+        }
+    };
+    await visit("");
+    if (entries.length === 0)
+        fail(`Built workspace bundle output is empty: ${directory}`);
+    entries.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));
+    return { directory, entries, mode: "dev" };
+};

package/dist/compiler/workspaceGitFetch.d.ts

@@ -0,0 +1,41 @@
+/** Which remote ref a `fetch: build` git resource follows. `none` follows the remote's default branch. */
+export interface GitPinSelector {
+    kind: "branch" | "none" | "ref" | "tag";
+    value: string;
+}
+/** Host-side SSH credential. Exactly one source; neither is ever written into the compile output. */
+export interface GitPinAuth {
+    sshKey?: string;
+    sshKeyEnv?: string;
+}
+export interface GitPinFetch {
+    /** Bare cache repository the commit's objects now live in. */
+    repository: string;
+    commit: string;
+}
+/** The remote ref to fetch, and the commit it must equal when the selector already names one. */
+export declare const gitPinRefspec: (selector: GitPinSelector) => {
+    expected?: string;
+    source: string;
+};
+/**
+ * Runs `use` with the git environment for one fetch. A key held in an env var
+ * is copied to a private 0600 file in a fresh temporary directory that is
+ * removed afterwards; the key only ever reaches git through `GIT_SSH_COMMAND`.
+ */
+export declare const withGitPinEnvironment: <T>(auth: GitPinAuth | undefined, use: (env: NodeJS.ProcessEnv) => Promise<T>, environment?: NodeJS.ProcessEnv) => Promise<T>;
+/**
+ * Serializes work on one cache repository across processes on this host: git
+ * keeps one repository-wide `shallow` file, so two shallow fetches into the
+ * same repository can fail each other. A lock whose owner process is gone is
+ * taken over.
+ */
+export declare const withGitPinLock: <T>(repository: string, use: () => Promise<T>) => Promise<T>;
+/**
+ * Resolves `selector` on `url` to one commit and fetches its objects into the
+ * cache, shallowly, under the repository lock. A full commit id already in the
+ * cache is not fetched again, so a recorded pin rebuilds offline. Any failure
+ * to resolve refuses the compile: a pinned resource never falls back to stale
+ * or empty content.
+ */
+export declare const fetchGitPin: (cacheDirectory: string, url: string, selector: GitPinSelector, env: NodeJS.ProcessEnv) => Promise<GitPinFetch>;