spawnfile 0.1.14 → 0.1.18

package/dist/compiler/training/preparation/prepare.js

@@ -0,0 +1,223 @@
+import { lstat, mkdir, readFile, realpath, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { mapTrainingExecution, trainingExecutionIdentity } from "../execution.js";
+import { trainingContainerConfigSchema } from "../container/contract.js";
+import { assertTrainingSubjectIsolation } from "../container/subjectIsolation.js";
+import { trainingPreparationSchema, parseTrainingMappedPreparation } from "./contract.js";
+import { assertInputRoot, exactPath, fileIdentity, hashJson, sealTree, within } from "./files.js";
+import { planInputs, readBoundedJson, stageInput, verifyCanonicalPins } from "./inputs.js";
+import { planTrainingImage, buildTrainingImage } from "./image.js";
+import { planMeasurementRepair, stageMeasurementRepair, validateMeasurementRepairOutput, writeTrainingWitness } from "../repair/index.js";
+import { prepareTrainingBroker } from "./broker.js";
+import { claimTrainingPreparationScratch, removeTrainingPreparationScratch } from "./scratch.js";
+function mappedReceipt(digest, image, config) {
+    return parseTrainingMappedPreparation({ version: "spawnfile.training-preparation.v1", preparationDigest: digest, imageId: image,
+        bindings: config.inputs.map(input => ({ inputId: input.id, destination: input.destination })), outputRoot: "/run/training/output",
+        packagePaths: { spawnfile: "build" in config.image && config.image.build.compiler
+                ? "/opt/training/compiler/dist/cli/index.js" : "/opt/training/spawnfile/dist/cli/index.js", paideia: "/opt/training/paideia", bridge: "/opt/training/paideia/bridges/dspy",
+            nativeWorker: "/opt/training/paideia/dist/src/adapters/daimon-native", integration: "/opt/training/integration", bootstrap: "/opt/training/bootstrap" }, integration: config.integration });
+}
+/** Reads only until the explicit dry-run boundary; preparation never executes project code. */
+export async function prepareTraining(options) {
+    const raw = await readBoundedJson(options.configPath);
+    const config = trainingPreparationSchema.parse({ ...raw,
+        ...(options.outputOverride ? { output: { source: options.outputOverride, destination: "/run/training/output" } } : {}),
+        ...(options.imageOverride ? { image: { ref: options.imageOverride } } : {}) });
+    await assertTrainingSubjectIsolation(config, options.context);
+    // A v2 declaration still lowers to the unchanged v1 launch config; v4 carries the broker slots into Docker.
+    const launchVersion = config.version === "spawnfile.training-container.v4" ? config.version : "spawnfile.training-container.v1";
+    const root = path.dirname(path.resolve(options.configPath));
+    const auth = config.auth.map(entry => ({ ...entry, source: path.resolve(root, entry.source) }));
+    const evaluatorSecrets = config.evaluatorSecrets?.map(entry => ({ ...entry, source: path.resolve(root, entry.source) }));
+    const protectedLeaves = [...auth, ...evaluatorSecrets ?? []];
+    const evaluatorConfig = { ...(config.evaluatorRoutes ? { evaluatorRoutes: config.evaluatorRoutes } : {}),
+        ...(config.subjectIsolation ? { subjectIsolation: config.subjectIsolation } : {}),
+        ...(evaluatorSecrets ? { evaluatorSecrets } : {}) };
+    const output = path.resolve(root, config.output.source), parent = path.dirname(output);
+    for (let index = 0; index < options.args.length; index++) {
+        if (options.args[index] !== "--out")
+            continue;
+        const declared = options.args[++index];
+        if (declared === undefined || path.resolve(declared) !== output)
+            throw Error("Training --out must match configured output");
+    }
+    // Refuse invalid repair destinations before auth validation or source/witness reads.
+    if (options.repairMeasurements)
+        await validateMeasurementRepairOutput({ parent: options.repairMeasurements,
+            output, resume: options.args.includes("--resume") });
+    // Validate references before traversing image/input sources; aliases must never bypass secret exclusions.
+    for (const entry of options.dryRun ? evaluatorSecrets ?? [] : protectedLeaves) {
+        const source = await exactPath(entry.source), info = await lstat(source);
+        if (source !== entry.source || !info.isFile())
+            throw Error("Training auth must be a canonical regular leaf");
+        if (evaluatorSecrets?.includes(entry) && info.nlink !== 1)
+            throw Error("Evaluator secrets must not have hardlink aliases");
+    }
+    assertInputRoot(output, protectedLeaves.map(entry => entry.source));
+    if (await realpath(parent) !== parent)
+        throw Error("Training output parent must be canonical and already exist");
+    const inputs = await planInputs(config, root, protectedLeaves.map(entry => entry.source));
+    const canonicalPins = [...options.context.sources, ...options.execution?.datasetPins ?? []];
+    const roots = inputs.map(input => input.source);
+    for (let index = 0; index < inputs.length; index++) {
+        const input = inputs[index];
+        // A selected/Git snapshot is staged before launch; never bind the source project over its own output.
+        const snapshotPaths = input.files?.map(file => path.join(input.source, file.destination))
+            ?? (input.git ? [...input.git.paths, ...input.git.overlays.map(file => file.destination)].map(file => path.join(input.source, file)) : undefined);
+        if (within(output, input.source) || (within(input.source, output) &&
+            (!snapshotPaths || snapshotPaths.some(file => within(output, file) || within(file, output))))) {
+            throw Error("Training output overlaps readonly input");
+        }
+        for (const other of inputs.slice(index + 1))
+            if (within(input.source, other.source) || within(other.source, input.source) ||
+                within(input.destination, other.destination) || within(other.destination, input.destination))
+                throw Error("Training inputs overlap");
+    }
+    const imagePlan = "build" in config.image
+        ? await planTrainingImage(config.image.build, root, protectedLeaves.map(entry => entry.source), options.packageRoot, path.resolve(options.configPath))
+        : undefined;
+    if (options.repairWitness && !options.repairMeasurements)
+        throw Error("A repair witness requires --repair-measurements");
+    if (options.repairMeasurements && !imagePlan)
+        throw Error("Measurement repair requires a verifiable image build recipe");
+    const repair = options.repairMeasurements ? await planMeasurementRepair({ parent: options.repairMeasurements,
+        witness: options.repairWitness, output, auth: protectedLeaves.map(entry => entry.source), image: imagePlan, inputs,
+        context: options.context, resume: options.args.includes("--resume") }) : undefined;
+    if (repair && hashJson(config.integration) !== hashJson(repair.witness.manifest.config.integration))
+        throw Error("Repair integration settings binding changed");
+    const digest = hashJson({ config, sources: inputs.map(input => ({ id: input.id, digest: input.digest })), image: imagePlan?.digest ?? config.image,
+        canonical: options.context.project.sourceDigest,
+        ...(options.execution ? { execution: trainingExecutionIdentity(options.execution) } : {}), ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) });
+    if (config.broker)
+        await prepareTrainingBroker(config.broker, root, path.dirname(path.resolve(options.configPath)), false, config.slots);
+    if (options.dryRun)
+        return { digest, dryRun: true };
+    options.signal?.throwIfAborted();
+    const execute = (args) => options.process(args, { timeoutMs: options.timeoutMs, signal: options.signal });
+    const endpoint = await execute(["context", "inspect", config.dockerContext, "--format", "{{json .Endpoints.docker.Host}}"]);
+    if (endpoint.code !== 0 || !/^unix:\/\//u.test(JSON.parse(endpoint.stdout)))
+        throw Error("Training preparation requires a local Unix Docker context");
+    if (repair) {
+        const label = await execute(["--context", config.dockerContext, "image", "inspect", repair.witness.manifest.parentImage,
+            "--format", '{{json .Id}}\n{{json (index .Config.Labels "com.spawnfile.training.recipe")}}']);
+        const values = label.stdout.trim().split("\n").map(value => JSON.parse(value));
+        if (label.code !== 0 || values[0] !== repair.witness.manifest.parentImage || values[1] !== repair.witness.manifest.imagePlanDigest)
+            throw Error("Parent image recipe witness is unverified");
+    }
+    const staging = path.join(parent, `.spawnfile-training-preparation-${hashJson(output).slice(7, 23)}`);
+    const configPath = path.join(staging, "launch.json"), preparationPath = path.join(staging, "mapped.json"), statePath = path.join(staging, "state.json");
+    const resume = options.args.includes("--resume");
+    let image, staged;
+    if (resume) {
+        await claimTrainingPreparationScratch(staging, digest, options.streams.stderr, "resume");
+        const previous = JSON.parse(await readFile(statePath, "utf8"));
+        if (previous.digest !== digest || !/^sha256:[a-f0-9]{64}$/u.test(previous.image) ||
+            JSON.stringify(previous.staged) !== JSON.stringify(inputs.map(input => input.git || input.files ? path.join(staging, input.id) : input.source)))
+            throw Error("Training preparation changed; exact resume rejected");
+        image = previous.image;
+        staged = previous.staged;
+        const snapshots = await Promise.all(inputs.map(async (input, index) => input.git || input.files ? hashJson(fileIdentity(await sealTree(staged[index], "input", { ignoreGit: true, internalSymlinks: true }))) : null));
+        if (JSON.stringify(previous.snapshots) !== JSON.stringify(snapshots))
+            throw Error("Persisted training input snapshot changed");
+        const mapped = parseTrainingMappedPreparation(await readBoundedJson(preparationPath));
+        if (mapped.preparationDigest !== digest || mapped.imageId !== image)
+            throw Error("Persisted training preparation identity mismatch");
+        await verifyCanonicalPins(inputs, canonicalPins, staged);
+    }
+    else {
+        let owned = false;
+        try {
+            await claimTrainingPreparationScratch(staging, digest, options.streams.stderr);
+            owned = true;
+            try {
+                await mkdir(output, { mode: 0o700 });
+            }
+            catch (error) {
+                if (error.code !== "EEXIST" || !(await lstat(output)).isDirectory() || await realpath(output) !== output)
+                    throw error;
+            }
+            // Do not remove owned staging while sibling input jobs can still write into it.
+            const results = await Promise.allSettled(inputs.map(input => stageInput(input, path.join(staging, input.id))));
+            const failure = results.find(result => result.status === "rejected");
+            if (failure?.status === "rejected")
+                throw failure.reason;
+            staged = results.map(result => result.value);
+            await verifyCanonicalPins(inputs, canonicalPins, staged);
+            image = imagePlan ? (await buildTrainingImage(imagePlan, { parent, dockerContext: config.dockerContext, process: options.process,
+                timeoutMs: options.timeoutMs, signal: options.signal, streams: options.streams })).imageId : "ref" in config.image ? config.image.ref : "";
+            if (!image.startsWith("sha256:")) {
+                const inspected = await execute(["--context", config.dockerContext, "image", "inspect", image, "--format", "{{.Id}}"]);
+                if (inspected.code !== 0)
+                    throw Error("Training image is unavailable");
+                image = inspected.stdout.trim();
+            }
+            const mapped = mappedReceipt(digest, image, config);
+            const preparedBroker = config.broker ? await prepareTrainingBroker(config.broker, root, staging, true, config.slots) : undefined;
+            const launch = trainingContainerConfigSchema.parse({ version: launchVersion, slots: config.slots, dockerContext: config.dockerContext,
+                inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth, ...evaluatorConfig,
+                ...(preparedBroker ? { broker: preparedBroker.launch } : {}) });
+            await writeFile(configPath, JSON.stringify(launch), { flag: "wx", mode: 0o600 });
+            await writeFile(preparationPath, JSON.stringify(parseTrainingMappedPreparation(mapped)), { flag: "wx", mode: 0o400 });
+            const snapshots = await Promise.all(inputs.map(async (input, index) => input.git || input.files ? hashJson(fileIdentity(await sealTree(staged[index], "input", { ignoreGit: true, internalSymlinks: true }))) : null));
+            await writeFile(statePath, JSON.stringify({ digest, image, staged, snapshots }), { flag: "wx", mode: 0o600 });
+            if (imagePlan)
+                await writeTrainingWitness({ directory: staging, digest, image, config, context: options.context,
+                    args: options.args, execution: options.execution, inputs, staged, snapshots, plan: imagePlan,
+                    ...(repair ? { repair: { witness: repair.witness.digest, parent: repair.manifestDigest } } : {}) });
+        }
+        catch (error) {
+            if (owned)
+                await removeTrainingPreparationScratch(staging);
+            throw error;
+        }
+    }
+    const expectedBroker = config.broker ? await prepareTrainingBroker(config.broker, root, staging, false, config.slots) : undefined;
+    const expectedLaunch = trainingContainerConfigSchema.parse({ version: launchVersion, slots: config.slots, dockerContext: config.dockerContext,
+        inputs: inputs.map((input, index) => ({ source: staged[index], destination: input.destination })), output: { source: output, destination: "/run/training/output" }, auth, ...evaluatorConfig,
+        ...(expectedBroker ? { broker: expectedBroker.launch } : {}) });
+    if (hashJson(await readBoundedJson(configPath)) !== hashJson(expectedLaunch) ||
+        hashJson(await readBoundedJson(preparationPath)) !== hashJson(mappedReceipt(digest, image, config)))
+        throw Error("Saved training launch or mapped receipt changed");
+    const inspected = await execute(["--context", config.dockerContext, "image", "inspect", image, "--format", "{{.Id}}"]);
+    if (inspected.code !== 0 || inspected.stdout.trim() !== image)
+        throw Error("Saved training image is missing or changed");
+    const map = (file) => {
+        const absolute = path.resolve(file);
+        if (within(output, absolute))
+            return absolute;
+        const index = roots.findIndex(source => within(source, absolute));
+        return index < 0 ? absolute : path.join(staged[index], path.relative(roots[index], absolute));
+    };
+    const context = { ...options.context,
+        project: { ...options.context.project, root: map(options.context.project.root), manifest: map(options.context.project.manifest) },
+        agent: { ...options.context.agent, source: map(options.context.agent.source) },
+        sources: options.context.sources.map(source => ({ ...source, sourcePath: map(source.sourcePath) })),
+        documents: options.context.documents.map(source => ({ ...source, sourcePath: map(source.sourcePath) })),
+        skills: options.context.skills.map(source => ({ ...source, sourcePath: map(source.sourcePath) })) };
+    const args = [...options.args];
+    for (let index = 0; index < args.length; index++) {
+        if (["--train", "--validation", "--test", "--cost-config"].includes(args[index]))
+            args[++index] = map(args[index]);
+        else if (args[index] === "--resource") {
+            const value = args[++index], split = value.indexOf("=");
+            args[index] = `${value.slice(0, split)}=${map(value.slice(split + 1))}`;
+        }
+    }
+    if (repair) {
+        const projected = await stageMeasurementRepair(repair, staging, image, options.context.project.sourceDigest, resume);
+        const repairLaunch = { ...expectedLaunch, inputs: [...expectedLaunch.inputs,
+                { source: projected.root, destination: "/run/training/inputs/repair-parent" }] };
+        const repairConfig = path.join(staging, "repair-launch.json");
+        if (resume) {
+            if (hashJson(await readBoundedJson(repairConfig)) !== hashJson(repairLaunch))
+                throw Error("Saved repair launch changed");
+        }
+        else
+            await writeFile(repairConfig, JSON.stringify(repairLaunch), { flag: "wx", mode: 0o600 });
+        return { digest, image, configPath: repairConfig, preparationPath, repairPath: projected.repairPath, context,
+            ...(options.execution ? { execution: mapTrainingExecution(options.execution, map) } : {}),
+            args: [...args, "--repair-measurements", "/run/training/inputs/repair-parent", "--repair-context", "/run/paideia/repair.json"] };
+    }
+    return { digest, image, configPath, preparationPath, context, args,
+        ...(options.execution ? { execution: mapTrainingExecution(options.execution, map) } : {}) };
+}

package/dist/compiler/training/preparation/scratch.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The receipt binds identity AND the owning process before staging starts.
+ * A matching digest alone never permits deletion. Unknown/remote/reused PIDs
+ * fail closed; only ESRCH on this host proves that a writer has stopped.
+ */
+export declare const claimTrainingPreparationScratch: (staging: string, digest: string, notify: (line: string) => void, mode?: "fresh" | "resume") => Promise<void>;
+/** Make sealed directories removable without following input links out of owned scratch. */
+export declare const removeTrainingPreparationScratch: (staging: string) => Promise<void>;

package/dist/compiler/training/preparation/scratch.js

@@ -0,0 +1,114 @@
+import path from "node:path";
+import { hostname } from "node:os";
+import { chmod, lstat, mkdir, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
+/**
+ * The receipt binds identity AND the owning process before staging starts.
+ * A matching digest alone never permits deletion. Unknown/remote/reused PIDs
+ * fail closed; only ESRCH on this host proves that a writer has stopped.
+ */
+export const claimTrainingPreparationScratch = async (staging, digest, notify, mode = "fresh") => {
+    const recordOwner = (flag = "wx") => writeFile(path.join(staging, "owner.json"), JSON.stringify({ digest, host: hostname(), pid: process.pid }), { flag, mode: 0o600 });
+    // This guard lives outside scratch so removal/recreation cannot drop the
+    // exclusion barrier. Never break a stranded guard automatically: its absence
+    // of a receipt cannot establish that its holder stopped.
+    const guard = `${staging}.reclaim`;
+    try {
+        await mkdir(guard, { mode: 0o700 });
+    }
+    catch (error) {
+        if (error.code !== "EEXIST")
+            throw error;
+        throw Error(`Training preparation scratch recovery is already claimed: ${guard}; if no preparation is running, remove that guard and retry`);
+    }
+    try {
+        if (mode === "fresh") {
+            let created = false;
+            try {
+                await mkdir(staging, { mode: 0o700 });
+                created = true;
+            }
+            catch (error) {
+                if (error.code !== "EEXIST")
+                    throw error;
+            }
+            if (created) {
+                await recordOwner();
+                return;
+            }
+        }
+        // Re-read under exclusion, never delete based on an earlier observation.
+        await assertAbandonedScratch(staging, digest, mode === "resume");
+        if (mode === "resume") {
+            // Claim before validating or reading snapshots: fresh recovery must see
+            // this live process rather than the stopped original preparer.
+            await recordOwner("w");
+            return;
+        }
+        notify(`Reusing the preparation scratch left by an earlier launch of this exact preparation: ${staging}`);
+        await removeTrainingPreparationScratch(staging);
+        await mkdir(staging, { mode: 0o700 });
+        await recordOwner();
+    }
+    finally {
+        await rm(guard, { recursive: true });
+    }
+};
+const assertAbandonedScratch = async (staging, digest, resume) => {
+    const info = await lstat(staging);
+    if (!info.isDirectory() || info.isSymbolicLink() || await realpath(staging) !== staging) {
+        throw Error(`Training preparation scratch ${staging} is not a canonical directory; remove it and retry`);
+    }
+    let previous;
+    try {
+        previous = JSON.parse(await readFile(`${staging}/state.json`, "utf8"));
+    }
+    catch {
+        try {
+            previous = JSON.parse(await readFile(path.join(staging, "owner.json"), "utf8"));
+        }
+        catch {
+            previous = undefined;
+        }
+    }
+    if (previous?.digest !== digest) {
+        if (resume)
+            throw Error("Training preparation changed; exact resume rejected");
+        throw Error(previous === undefined
+            ? `Training preparation scratch ${staging} is left over from an interrupted launch that never recorded its identity; remove it to retry: rm -rf ${JSON.stringify(staging)}`
+            : `Training preparation scratch ${staging} belongs to a different preparation (${String(previous.digest)}); remove it to retry: rm -rf ${JSON.stringify(staging)}`);
+    }
+    let owner;
+    try {
+        owner = JSON.parse(await readFile(path.join(staging, "owner.json"), "utf8"));
+    }
+    catch { /* Unknown ownership fails closed. */ }
+    if (owner?.digest !== digest || owner.host !== hostname() || typeof owner.pid !== "number" ||
+        !Number.isSafeInteger(owner.pid) || owner.pid <= 0) {
+        throw Error(`Training preparation scratch ${staging} has no verifiable local owner; cannot prove its previous launch stopped`);
+    }
+    // Revalidation by the owning process is idempotent; other live owners refuse.
+    if (resume && owner.pid === process.pid)
+        return;
+    let stopped = false;
+    try {
+        process.kill(owner.pid, 0);
+    }
+    catch (error) {
+        stopped = error.code === "ESRCH";
+    }
+    if (!stopped)
+        throw Error(`Training preparation scratch ${staging} still has an active or unverifiable owner (${owner.pid}); retry after it stops`);
+};
+/** Make sealed directories removable without following input links out of owned scratch. */
+export const removeTrainingPreparationScratch = async (staging) => {
+    const writable = async (entry) => {
+        const info = await lstat(entry);
+        if (!info.isDirectory() || info.isSymbolicLink())
+            return;
+        await chmod(entry, (info.mode & 0o777) | 0o700);
+        for (const child of await readdir(entry))
+            await writable(path.join(entry, child));
+    };
+    await writable(staging);
+    await rm(staging, { recursive: true, force: true });
+};

package/dist/compiler/training/repair/compatibility.d.ts

@@ -0,0 +1,11 @@
+import type { TrainingContext } from "../contract.js";
+import type { TrainingImagePlan } from "../preparation/image.js";
+import type { PlannedInput } from "../preparation/inputs.js";
+import type { TrainingWitness } from "./contract.js";
+/** Subject and optimizer semantics remain pinned; only evaluation/fork plumbing may change. */
+export declare function verifyTrainingCompatibility(witness: TrainingWitness, current: TrainingImagePlan, inputs: PlannedInput[], context: TrainingContext): {
+    components: Record<string, string>;
+    inputs: {
+        [k: string]: string;
+    };
+};

package/dist/compiler/training/repair/compatibility.js

@@ -0,0 +1,34 @@
+import { fileIdentity, hashJson } from "../preparation/files.js";
+/** Subject and optimizer semantics remain pinned; only evaluation/fork plumbing may change. */
+export function verifyTrainingCompatibility(witness, current, inputs, context) {
+    if (context.agent.runtime !== "daimon" || context.project.sourceDigest !== witness.context.project.sourceDigest ||
+        JSON.stringify(context.agent) !== JSON.stringify(witness.context.agent))
+        throw Error("Repair canonical agent changed");
+    const old = witness.image.files, next = fileIdentity(current.files);
+    const selection = (files, prefix, excludes = []) => {
+        const values = files.filter(file => file.destination.startsWith(prefix) && !excludes.some(value => file.destination === value || (value.endsWith("/") && file.destination.startsWith(value))))
+            .map(file => ({ ...file, destination: file.destination.slice(prefix.length) })).sort((a, b) => a.destination.localeCompare(b.destination));
+        if (!values.length)
+            throw Error(`Repair lacks compatibility evidence for ${prefix}`);
+        return hashJson(values);
+    };
+    const components = {};
+    const compare = (name, previous, updated) => {
+        if (previous !== updated)
+            throw Error(`Repair subject compatibility changed: ${name}`);
+        components[name] = updated;
+    };
+    for (const field of ["nativeImage", "pythonImage", "platform"])
+        compare(field, hashJson(witness.image.build[field]), hashJson(current.build[field]));
+    compare("compiler", selection(old, witness.image.build.compiler ? "compiler/" : "spawnfile/"), selection(next, "compiler/"));
+    for (const [name, prefix] of Object.entries({ integration: "integration/", native: "paideia/dist/src/adapters/daimon-native/",
+        trials: "paideia/dist/src/experiments/trials/" }))
+        compare(name, selection(old, prefix), selection(next, prefix));
+    const excluded = ["bridge/paideia_dspy/checkpoint.py", "bridge/paideia_dspy/protocol.py", "bridge/README.md", "bridge/protocol.md", "bridge/.coverage", "bridge/.pytest_cache/", "bridge/coverage.json"];
+    compare("optimizer", selection(old, "bridge/", excluded), selection(next, "bridge/", excluded));
+    const previousInputs = witness.inputs.map(input => ({ id: input.id, destination: input.destination, digest: input.digest }));
+    const currentInputs = inputs.map(input => ({ id: input.id, destination: input.destination, digest: input.digest }));
+    if (hashJson(previousInputs) !== hashJson(currentInputs))
+        throw Error("Repair evidence or dataset inputs changed");
+    return { components, inputs: Object.fromEntries(currentInputs.map(input => [input.id, input.digest])) };
+}