spawnfile 0.1.17 → 0.1.18

package/dist/compiler/training/container/launch.js

@@ -1,31 +1,59 @@
 import { randomUUID } from "node:crypto";
-import { chmod, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises";
+import { chmod, mkdtemp, readFile, realpath, stat, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS } from "../../../shared/index.js";
+import { parseTrainingBrokerDeclaration } from "../broker/declaration.js";
+import { trainingSlotsDocument, TRAINING_SLOTS_FILE } from "../broker/paths.js";
 import { assertNotDesktopGrokAuth } from "../broker/entrypoint.js";
-import { TRAINING_BROKER_ENTRYPOINT, trainingBrokerMounts, trainingBrokerSecurityArgs, trainingBrokerTmpfsTargets } from "./security.js";
+import { TRAINING_BROKER_ENTRYPOINT, trainingBrokerMounts, trainingBrokerSecurityArgs, trainingBrokerTmpfsTargets, trainingRunRequestMount } from "./security.js";
 import { parseDetachedContainerInspect } from "../../runProjectDocker.js";
+import { mapTrainingExecution, trainingExecutionSchema, TRAINING_EXECUTION_FILE, TRAINING_EXECUTION_CAPABILITY_LABEL, TRAINING_EXECUTION_VERSION } from "../execution.js";
+import { verifyCanonicalPins } from "../preparation/inputs.js";
+import { removeTrainingPreparationScratch } from "../preparation/scratch.js";
 import { trainingImageSchema } from "./contract.js";
+import { prepareTrainingEvaluators } from "./evaluators.js";
 import { prepareTrainingContainer } from "./prepare.js";
 import { runTrainingDocker } from "./process.js";
 import { repairEnvelopeSchema } from "../repair/contract.js";
 import { hashJson } from "../preparation/files.js";
 import { parseTrainingMappedPreparation } from "../preparation/contract.js";
+import { persistTrainingGrokCredential } from "./grokWriteback.js";
+import { nativeTrainingSlotsDocument } from "./nativeSlots.js";
+import { createTrainingOutputVolume, trainingOutputVolumeEnabled } from "./outputVolume.js";
+// The image bakes /run/training/inputs as 0:2000 0750 for brokered worker isolation; the
+// non-brokered host-uid evaluator joins that group so it can still traverse its own inputs.
+export const TRAINING_INPUTS_GID = 2_000;
 const inspectFormat = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}";
 export const launchTrainingContainer = async (options) => {
     if (options.signal?.aborted)
         return 130;
     const image = trainingImageSchema.parse(options.image);
+    const volumeEnabled = trainingOutputVolumeEnabled();
     const uid = process.getuid?.(), gid = process.getgid?.();
     if (uid === undefined || gid === undefined || uid === 0)
         throw Error("Training requires an explicit non-root local user");
     const configBytes = await readFile(options.configPath, "utf8");
     if (Buffer.byteLength(configBytes) > 1024 * 1024)
         throw Error("Training launch config exceeds 1 MiB");
-    const prepared = await prepareTrainingContainer(JSON.parse(configBytes), options.context, options.args);
+    const raw = JSON.parse(configBytes);
+    const prepared = await prepareTrainingContainer(options.execution
+        ? { ...raw, output: { ...raw.output, source: options.execution.output } } : raw, options.context, options.args, { createOutput: options.execution !== undefined && !options.args.includes("--resume") });
+    const execution = options.execution ? mapTrainingExecution(options.execution, file => {
+        const absolute = path.resolve(file);
+        const binding = [...prepared.config.inputs, prepared.config.output].find(entry => absolute === entry.source || absolute.startsWith(entry.source + path.sep));
+        if (!binding)
+            throw Error(`Training execution path is not covered by a declared mount: ${file}`);
+        return path.posix.join(binding.destination, path.relative(binding.source, absolute).split(path.sep).join("/"));
+    }) : undefined;
+    if (options.execution)
+        await verifyCanonicalPins(prepared.config.inputs.map((input, index) => ({ ...input, id: String(index), digest: "" })), options.execution.datasetPins, prepared.config.inputs.map(input => input.source));
     // D2: the training Grok login is a dedicated one. Refuse the developer's desktop leaf before Docker ever sees it.
-    if (prepared.config.broker)
+    if (prepared.config.broker) {
         assertNotDesktopGrokAuth(prepared.config.broker.bootstrap);
+        const declaration = parseTrainingBrokerDeclaration(JSON.parse(await readFile(prepared.config.broker.declaration, "utf8")));
+        if (declaration.slots !== prepared.config.slots)
+            throw Error("Training launch and broker declaration slots differ");
+    }
     if (options.preparationPath) {
         if (await realpath(options.preparationPath) !== options.preparationPath)
             throw Error("Preparation receipt path must be canonical");
@@ -46,23 +74,24 @@ export const launchTrainingContainer = async (options) => {
     const interrupt = () => { interrupted = true; controller.abort(); };
     const timer = setTimeout(() => { timedOut = true; controller.abort(); }, options.timeoutMs);
     options.signal?.addEventListener("abort", interrupt, { once: true });
-    process.once("SIGINT", interrupt);
-    process.once("SIGTERM", interrupt);
+    process.on("SIGINT", interrupt);
+    process.on("SIGTERM", interrupt);
     if (options.signal?.aborted)
         interrupt();
     const prefix = ["--context", prepared.config.dockerContext];
-    const call = async (args, cleanup = false, stream = false) => {
+    const call = async (args, cleanup = false, stream = false, cleanupTimeoutMs = 10_000) => {
         if (!cleanup && controller.signal.aborted)
             throw Error("Container operation cancelled");
         return execute([...prefix, ...args], {
-            timeoutMs: cleanup ? 10_000 : Math.max(1, deadline - Date.now()),
+            timeoutMs: cleanup ? cleanupTimeoutMs : Math.max(1, deadline - Date.now()),
             ...(cleanup ? {} : { signal: controller.signal }), ...(stream ? options.streams : {})
         });
     };
     const name = `spawnfile-training-${randomUUID()}`;
     const labels = { "com.spawnfile.training.owner": name };
-    let privateRoot, containerId, creationAttempted = false;
+    let privateRoot, containerId, creationAttempted = false, startAttempted = false;
     let expectedImage, lastLine = "", closureVerified = false;
+    let outputVolume;
     try {
         const endpoint = await execute(["context", "inspect", prepared.config.dockerContext, "--format", "{{json .Endpoints.docker.Host}}"], { timeoutMs: 10_000, signal: controller.signal });
         if (endpoint.code !== 0 || !/^unix:\/\//u.test(JSON.parse(endpoint.stdout)))
@@ -71,38 +100,69 @@ export const launchTrainingContainer = async (options) => {
         expectedImage = inspected.stdout.trim();
         if (inspected.code !== 0 || !/^sha256:[a-f0-9]{64}$/u.test(expectedImage) || (image.startsWith("sha256:") && image !== expectedImage))
             throw Error("Training image must be locally available with an immutable identity");
+        if (execution) {
+            const capability = await call(["image", "inspect", expectedImage, "--format", `{{json (index .Config.Labels "${TRAINING_EXECUTION_CAPABILITY_LABEL}")}}`]);
+            if (capability.code !== 0 || capability.stdout.trim() !== JSON.stringify(TRAINING_EXECUTION_VERSION)) {
+                throw Error(`Authored training requires an image implementing --spawnfile-execution ${TRAINING_EXECUTION_FILE} and labelled ${TRAINING_EXECUTION_CAPABILITY_LABEL}=${TRAINING_EXECUTION_VERSION}; legacy Paideia datasets are incompatible`);
+            }
+        }
         privateRoot = await realpath(await mkdtemp(path.join(path.dirname(prepared.config.output.source), ".spawnfile-training-container-")));
         await chmod(privateRoot, 0o700);
         const contextFile = path.join(privateRoot, "context.json");
         await writeFile(contextFile, JSON.stringify(prepared.context), { mode: 0o444, flag: "wx" });
-        const mounts = [...prepared.config.inputs.map((entry) => `type=bind,src=${entry.source},dst=${entry.destination},readonly`),
-            `type=bind,src=${prepared.config.output.source},dst=/run/training/output`,
+        const executionFile = path.join(privateRoot, "execution.json");
+        if (execution)
+            await writeFile(executionFile, JSON.stringify(trainingExecutionSchema.parse(execution)), { mode: 0o444, flag: "wx" });
+        const evaluators = await prepareTrainingEvaluators(privateRoot, prepared.config, prepared.args);
+        if (volumeEnabled) {
+            outputVolume = createTrainingOutputVolume({ name, image: expectedImage, host: prepared.config.output.source, scratch: privateRoot,
+                uid, gid, runtimeUid: prepared.config.broker ? 2000 : uid, runtimeGid: prepared.config.broker ? 2000 : gid,
+                context: prepared.config.dockerContext, call: (args, cleanup) => call(args, cleanup, false, 120_000), stderr: options.streams.stderr });
+            await outputVolume.seed();
+        }
+        const mounts = [...evaluators.mounts, ...prepared.config.inputs.map((entry) => `type=bind,src=${entry.source},dst=${entry.destination},readonly`),
+            outputVolume?.mount ?? `type=bind,src=${prepared.config.output.source},dst=/run/training/output`,
             `type=bind,src=${contextFile},dst=/run/paideia/context.json,readonly`,
+            ...execution ? [`type=bind,src=${executionFile},dst=${TRAINING_EXECUTION_FILE},readonly`] : [],
             ...options.preparationPath ? [`type=bind,src=${options.preparationPath},dst=/run/paideia/preparation.json,readonly`] : [],
             ...options.repairPath ? [`type=bind,src=${options.repairPath},dst=/run/paideia/repair.json,readonly`] : [],
             ...prepared.config.auth.map((entry) => `type=bind,src=${entry.source},dst=/run/paideia-auth/${entry.provider},readonly`)];
         const broker = prepared.config.broker;
+        if (broker) {
+            mounts.push(trainingRunRequestMount(prepared.config.output.source, prepared.args));
+            const slotsFile = path.join(privateRoot, "slots.json");
+            await writeFile(slotsFile, `${JSON.stringify(trainingSlotsDocument(prepared.config.slots))}\n`, { mode: 0o444, flag: "wx" });
+            mounts.push(`type=bind,src=${slotsFile},dst=${TRAINING_SLOTS_FILE},readonly`);
+        }
+        else {
+            const slotsFile = path.join(privateRoot, "slots.json");
+            await writeFile(slotsFile, `${JSON.stringify(nativeTrainingSlotsDocument(prepared.config.slots, uid))}\n`, { mode: 0o444, flag: "wx" });
+            mounts.push(`type=bind,src=${slotsFile},dst=${TRAINING_SLOTS_FILE},readonly`);
+        }
         // A brokered Grok slot runs the launcher, the broker and the model's own worker uid inside this container, so it
         // starts as root with the production Daimon capability set instead of the host user with no capabilities at all.
         const privilege = broker
             ? [...await trainingBrokerSecurityArgs(privateRoot), "--pids-limit", "2048",
-                ...trainingBrokerTmpfsTargets().flatMap((entry) => ["--tmpfs", `${entry.path}:rw,nosuid,nodev,size=${entry.size},mode=${entry.mode}`])]
-            : ["--user", `${uid}:${gid}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
-                ...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS, "--pids-limit", "512",
-                "--tmpfs", `/tmp:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=1777`, "--tmpfs", `/work:rw,nosuid,nodev,size=4g,uid=${uid},gid=${gid},mode=700`,
-                "--tmpfs", `/home/training:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid},mode=700`];
+                ...trainingBrokerTmpfsTargets(prepared.config.slots).flatMap((entry) => ["--tmpfs", `${entry.path}:rw,nosuid,nodev,size=${entry.size},mode=${entry.mode}`])]
+            : ["--user", `${uid}:${gid}`, "--group-add", String(TRAINING_INPUTS_GID), "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
+                ...DAIMON_CODEX_NATIVE_SANDBOX_DOCKER_SECURITY_OPTS, "--pids-limit", String(512 * prepared.config.slots),
+                "--tmpfs", `/tmp:rw,nosuid,nodev,size=${prepared.config.slots}g,uid=${uid},gid=${gid},mode=1777`, "--tmpfs", `/work:rw,nosuid,nodev,size=${4 * prepared.config.slots}g,uid=${uid},gid=${gid},mode=700`,
+                "--tmpfs", `/home/training:rw,nosuid,nodev,size=${prepared.config.slots}g,uid=${uid},gid=${gid},mode=700`];
         const args = ["create", "--name", name, "--label", `com.spawnfile.training.owner=${name}`,
             "--init", "--read-only", ...privilege,
             "--env", "HOME=/home/training", "--workdir", "/work", "--entrypoint", broker ? TRAINING_BROKER_ENTRYPOINT : "/opt/training/bin/train",
+            ...(broker ? ["--env", "PAIDEIA_MODEL_REQUEST_CAP=broker"] : []),
+            ...evaluators.env.flatMap(value => ["--env", value]),
             ...[...mounts, ...(broker ? trainingBrokerMounts(broker) : [])].flatMap((mount) => ["--mount", mount]),
             ...(prepared.viewerPort === undefined ? [] : ["--publish", `127.0.0.1:${prepared.viewerPort}:${prepared.viewerPort}`]), expectedImage,
-            "train", "--spawnfile-context", "/run/paideia/context.json", ...prepared.args];
+            "train", "--spawnfile-context", "/run/paideia/context.json",
+            ...(execution ? ["--spawnfile-execution", TRAINING_EXECUTION_FILE] : []), ...prepared.args];
         creationAttempted = true;
         const created = await call(args);
         containerId = created.stdout.trim();
         if (created.code !== 0 || !/^[a-f0-9]{64}$/u.test(containerId)) {
             let diagnostic = created.stderr;
-            for (const auth of prepared.config.auth)
+            for (const auth of [...prepared.config.auth, ...prepared.config.evaluatorSecrets ?? []])
                 diagnostic = diagnostic.replaceAll(auth.source, "[auth source]");
             diagnostic = diagnostic.replace(/\p{Cc}/gu, " ").trim().slice(0, 2048);
             throw Error(`Docker did not return a valid training container identity${diagnostic ? `: ${diagnostic}` : ""}`);
@@ -112,6 +172,7 @@ export const launchTrainingContainer = async (options) => {
             throw Error("Training container identity mismatch");
         if (controller.signal.aborted)
             throw Error("Container operation cancelled");
+        startAttempted = true;
         const result = await execute([...prefix, "start", "--attach", containerId], {
             timeoutMs: Math.max(1, deadline - Date.now()), signal: controller.signal,
             stdout: (line) => { if (line.trim())
@@ -121,6 +182,7 @@ export const launchTrainingContainer = async (options) => {
         const state = JSON.parse(terminal.stdout);
         if (terminal.code !== 0 || state.Running !== false || !Number.isInteger(state.ExitCode) || result.code !== state.ExitCode)
             throw Error("Training container final state is unverified");
+        await outputVolume?.export();
         if (state.ExitCode !== 0 && state.ExitCode !== 1)
             return state.ExitCode;
         const receipt = JSON.parse(lastLine);
@@ -140,9 +202,6 @@ export const launchTrainingContainer = async (options) => {
     }
     finally {
         clearTimeout(timer);
-        options.signal?.removeEventListener("abort", interrupt);
-        process.removeListener("SIGINT", interrupt);
-        process.removeListener("SIGTERM", interrupt);
         try {
             if (creationAttempted) {
                 // A timed-out create may still have created our uniquely labelled container.
@@ -151,11 +210,28 @@ export const launchTrainingContainer = async (options) => {
                     const id = JSON.parse(found.stdout.split("\n")[0]);
                     if (typeof id !== "string" || parseDetachedContainerInspect(found.stdout, id, labels, name).imageId !== expectedImage || (containerId && /^[a-f0-9]{64}$/u.test(containerId) && id !== containerId))
                         throw Error("Refusing cleanup of unverified training container");
-                    const removed = await call(["rm", "--force", id], true);
-                    const remaining = await call(["container", "ls", "--all", "--no-trunc", "--filter", `id=${id}`, "--format", "{{.ID}}"], true);
-                    if (removed.code !== 0 || remaining.code !== 0 || remaining.stdout.trim())
-                        throw Error("Training container cleanup is unverified");
-                    closureVerified = true;
+                    try {
+                        if (startAttempted) {
+                            const current = await call(["inspect", "--format", "{{json .State}}", id], true);
+                            if (current.code !== 0)
+                                throw Error("Cannot verify training writer state");
+                            if (JSON.parse(current.stdout).Running === true) {
+                                const stopped = await call(["stop", "--time", "5", id], true);
+                                if (stopped.code !== 0)
+                                    throw Error("Cannot stop training output writer");
+                            }
+                        }
+                        if (startAttempted && prepared.config.broker && privateRoot) {
+                            await persistTrainingGrokCredential(id, prepared.config.broker.bootstrap, privateRoot, call);
+                        }
+                    }
+                    finally {
+                        const removed = await call(["rm", "--force", id], true);
+                        const remaining = await call(["container", "ls", "--all", "--no-trunc", "--filter", `id=${id}`, "--format", "{{.ID}}"], true);
+                        if (removed.code !== 0 || remaining.code !== 0 || remaining.stdout.trim())
+                            throw Error("Training container cleanup is unverified");
+                        closureVerified = true;
+                    }
                 }
                 else {
                     const remaining = await call(["container", "ls", "--all", "--filter", `name=^/${name}$`, "--format", "{{.ID}}"], true);
@@ -166,8 +242,23 @@ export const launchTrainingContainer = async (options) => {
             }
         }
         finally {
-            if (privateRoot && (!creationAttempted || closureVerified))
-                await rm(privateRoot, { recursive: true, force: true });
+            try {
+                const closed = !creationAttempted || closureVerified;
+                try {
+                    if (closed)
+                        await outputVolume?.export();
+                }
+                finally {
+                    await outputVolume?.dispose(closed);
+                }
+            }
+            finally {
+                options.signal?.removeEventListener("abort", interrupt);
+                process.removeListener("SIGINT", interrupt);
+                process.removeListener("SIGTERM", interrupt);
+                if (privateRoot && (!creationAttempted || closureVerified) && (!outputVolume || outputVolume.safeToCleanup()))
+                    await removeTrainingPreparationScratch(privateRoot);
+            }
         }
     }
 };

package/dist/compiler/training/container/nativeSlots.d.ts

@@ -0,0 +1,14 @@
+/** Native trials keep fresh per-trial roots; only their port blocks are provisioned. */
+export declare const nativeTrainingSlotsDocument: (slots: number, workerUid: number) => {
+    schema: "spawnfile.training-slots.v2";
+    slots: {
+        index: number;
+        workerUid: number;
+        isolation: "shared-uid";
+        roots: {};
+        ports: {
+            base: number;
+            count: number;
+        };
+    }[];
+};

package/dist/compiler/training/container/nativeSlots.js

@@ -0,0 +1,13 @@
+import { assertTrainingSlotCount, TRAINING_PORT_BASE, TRAINING_PORT_STRIDE } from "../broker/paths.js";
+/** Native trials keep fresh per-trial roots; only their port blocks are provisioned. */
+export const nativeTrainingSlotsDocument = (slots, workerUid) => {
+    if (!Number.isInteger(workerUid) || workerUid <= 0 || workerUid > 4_294_967_294)
+        throw Error("Native slots require a non-root worker uid");
+    return {
+        schema: "spawnfile.training-slots.v2",
+        slots: Array.from({ length: assertTrainingSlotCount(slots) }, (_, index) => ({
+            index, workerUid, isolation: "shared-uid", roots: {},
+            ports: { base: TRAINING_PORT_BASE + index * TRAINING_PORT_STRIDE, count: TRAINING_PORT_STRIDE }
+        }))
+    };
+};

package/dist/compiler/training/container/outputCopy.d.ts

@@ -0,0 +1,2 @@
+/** Runs only in the verified Linux training image. No project code is imported. */
+export declare const trainingOutputCopyProgram: string;

package/dist/compiler/training/container/outputCopy.js

@@ -0,0 +1,72 @@
+/** Runs only in the verified Linux training image. No project code is imported. */
+export const trainingOutputCopyProgram = String.raw `
+const fs = require('node:fs');
+const path = require('node:path');
+const [source, destination, owner, group] = process.argv.slice(1);
+const uid = Number(owner), gid = Number(group);
+const limits = { entries: 1_000_000, bytes: 64 * 1024 ** 3, depth: 64 };
+let entries = 0, bytes = 0;
+const links = new Map();
+const buffer = Buffer.alloc(1024 * 1024);
+// Linux fd-relative traversal keeps renamed directories and symlink swaps from
+// redirecting reads/writes outside the two mounts. The fallback permits local QA.
+const fdPath = (fd, file) => process.platform === 'linux' ? '/proc/self/fd/' + fd : file;
+const directory = fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW;
+function copy(src, dst, depth) {
+  if (++entries > limits.entries || depth > limits.depth) throw Error('Output tree exceeds entry/depth limit');
+  const info = fs.lstatSync(src);
+  if (info.isSymbolicLink()) {
+    const target = fs.readlinkSync(src);
+    bytes += Buffer.byteLength(target);
+    if (bytes > limits.bytes) throw Error('Output exceeds 64 GiB');
+    fs.symlinkSync(target, dst); fs.lchownSync(dst, uid, gid);
+    fs.lutimesSync(dst, info.atime, info.mtime);
+    return;
+  }
+  if (info.isDirectory()) {
+    const input = fs.openSync(src, directory);
+    let output;
+    try {
+      if (depth !== 0) fs.mkdirSync(dst, { mode: 0o700 });
+      output = fs.openSync(dst, directory);
+      fs.fchownSync(output, uid, gid);
+      const from = fdPath(input, src), to = fdPath(output, dst);
+      if (depth === 0 && fs.readdirSync(to).length) throw Error('Output copy destination must be empty');
+      for (const name of fs.readdirSync(from)) copy(path.join(from, name), path.join(to, name), depth + 1);
+      fs.fchownSync(output, uid, gid); fs.fchmodSync(output, info.mode & 0o7777);
+      fs.futimesSync(output, info.atime, info.mtime); fs.fsyncSync(output);
+    } finally { if (output !== undefined) fs.closeSync(output); fs.closeSync(input); }
+    return;
+  }
+  if (!info.isFile()) throw Error('Output contains a special file');
+  const input = fs.openSync(src, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK);
+  let output;
+  try {
+    const opened = fs.fstatSync(input);
+    if (!opened.isFile() || opened.dev !== info.dev || opened.ino !== info.ino) throw Error('Output changed during copy');
+    const key = info.dev + ':' + info.ino;
+    // Store a path rooted at the destination mount, not an fd that closes after recursion.
+    const previous = links.get(key);
+    if (previous) { fs.linkSync(previous, dst); return; }
+    bytes += info.size;
+    if (bytes > limits.bytes) throw Error('Output exceeds 64 GiB');
+    output = fs.openSync(dst, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o600);
+    fs.fchownSync(output, uid, gid);
+    let position = 0;
+    while (position < info.size) {
+      const count = fs.readSync(input, buffer, 0, Math.min(buffer.length, info.size - position), position);
+      if (!count) throw Error('Output truncated during copy');
+      let written = 0;
+      while (written < count) written += fs.writeSync(output, buffer, written, count - written);
+      position += count;
+    }
+    const after = fs.fstatSync(input);
+    if (after.size !== info.size || after.mtimeMs !== info.mtimeMs) throw Error('Output changed during copy');
+    fs.fchownSync(output, uid, gid); fs.fchmodSync(output, info.mode & 0o7777);
+    fs.futimesSync(output, info.atime, info.mtime); fs.fsyncSync(output);
+    links.set(key, fs.realpathSync(dst));
+  } finally { if (output !== undefined) fs.closeSync(output); fs.closeSync(input); }
+}
+copy(source, destination, 0);
+console.log('spawnfile-output-copy-complete');
+`;

package/dist/compiler/training/container/outputVolume.d.ts

@@ -0,0 +1,27 @@
+type DockerCall = (args: string[], cleanup?: boolean) => Promise<{
+    code: number;
+    stdout: string;
+    stderr: string;
+}>;
+export declare const trainingOutputVolumeEnabled: () => boolean;
+/** One launch owns this volume; budgets and auth volumes are never touched. */
+export declare const createTrainingOutputVolume: (options: {
+    name: string;
+    image: string;
+    host: string;
+    scratch: string;
+    uid: number;
+    gid: number;
+    runtimeUid: number;
+    runtimeGid: number;
+    context: string;
+    call: DockerCall;
+    stderr(line: string): void;
+}) => {
+    safeToCleanup: () => boolean;
+    mount: string;
+    seed(): Promise<void>;
+    export(): Promise<void>;
+    dispose(closureVerified: boolean): Promise<void>;
+};
+export {};

package/dist/compiler/training/container/outputVolume.js

@@ -0,0 +1,161 @@
+import { constants } from "node:fs";
+import { lstat, mkdir, open, realpath, rename } from "node:fs/promises";
+import path from "node:path";
+import { parseDetachedContainerInspect } from "../../runProjectDocker.js";
+import { trainingOutputCopyProgram } from "./outputCopy.js";
+const ownerLabel = "com.spawnfile.training.owner";
+const inspectFormat = "{{json .Id}}\n{{json .Name}}\n{{json .Image}}\n{{json .Config.Labels}}";
+const target = "/run/training/output";
+export const trainingOutputVolumeEnabled = () => {
+    const setting = process.env.SPAWNFILE_TRAINING_OUTPUT_STORAGE;
+    if (setting !== undefined && setting !== "volume" && setting !== "bind")
+        throw Error("SPAWNFILE_TRAINING_OUTPUT_STORAGE must be volume or bind");
+    return setting !== "bind";
+};
+/** One launch owns this volume; budgets and auth volumes are never touched. */
+export const createTrainingOutputVolume = (options) => {
+    const { call } = options;
+    const volume = `${options.name}-output`;
+    let attempted = false, seeded = false, exported = false, exportAttempted = false, helperClosed = true;
+    const recovery = () => `Training output retained in Docker volume ${volume}. Recover with: docker --context ${options.context} run --rm --network none --read-only --mount type=volume,src=${volume},dst=/output,readonly --entrypoint /bin/tar ${options.image} -C /output -cpf - . > training-output-recovery.tar`;
+    const owned = async () => {
+        const result = await call(["volume", "inspect", volume, "--format", "{{json .}}"], true);
+        if (result.code !== 0) {
+            const listed = await call(["volume", "ls", "--filter", `name=^${volume}$`, "--format", "{{.Name}}"], true);
+            if (listed.code === 0 && !listed.stdout.trim())
+                return false;
+            throw Error("Training output volume closure is unknown");
+        }
+        const info = JSON.parse(result.stdout);
+        if (info.Name !== volume || info.Driver !== "local" || (info.Options && Object.keys(info.Options).length) || info.Labels?.[ownerLabel] !== options.name)
+            throw Error("Refusing unverified training output volume");
+        return true;
+    };
+    const transfer = async (direction) => {
+        if (!await owned())
+            throw Error("Training output volume is missing");
+        const name = `${volume}-${direction}`;
+        const labels = { [ownerLabel]: name };
+        const hostStage = path.join(options.scratch, "output-export");
+        if (direction === "export")
+            await mkdir(hostStage, { mode: 0o700 });
+        const mounts = direction === "seed"
+            ? [`type=bind,src=${options.host},dst=/source,readonly`, `type=volume,src=${volume},dst=/destination,volume-nocopy`]
+            : [`type=volume,src=${volume},dst=/source,readonly,volume-nocopy`, `type=bind,src=${hostStage},dst=/destination`];
+        let creationAttempted = false, containerId;
+        try {
+            creationAttempted = true;
+            helperClosed = false;
+            const created = await call(["container", "create", "--name", name, "--label", `${ownerLabel}=${name}`,
+                "--network", "none", "--read-only", "--user", "0:0", "--security-opt", "no-new-privileges",
+                // Retain only Docker's default CHOWN, DAC_OVERRIDE, FOWNER and FSETID for copying
+                // private runtime files and assigning the native/broker uid. No cap-add.
+                ...["AUDIT_WRITE", "KILL", "MKNOD", "NET_BIND_SERVICE", "NET_RAW", "SETFCAP", "SETGID", "SETPCAP", "SETUID", "SYS_CHROOT"].flatMap(cap => ["--cap-drop", cap]),
+                "--pids-limit", "32", "--memory", "256m", "--cpus", "1", "--workdir", "/",
+                ...mounts.flatMap(mount => ["--mount", mount]), "--entrypoint", "node", options.image,
+                "-e", trainingOutputCopyProgram, "/source", "/destination",
+                String(direction === "seed" ? options.runtimeUid : options.uid), String(direction === "seed" ? options.runtimeGid : options.gid)], direction === "export");
+            const id = created.stdout.trim();
+            containerId = id;
+            if (created.code !== 0 || !/^[a-f0-9]{64}$/u.test(id))
+                throw Error(`Cannot create training output ${direction} helper`);
+            const checked = await call(["container", "inspect", "--format", inspectFormat, name], true);
+            if (checked.code !== 0 || parseDetachedContainerInspect(checked.stdout, id, labels, name).imageId !== options.image)
+                throw Error("Training output helper identity mismatch");
+            const result = await call(["container", "start", "--attach", id], direction === "export");
+            const terminal = await call(["container", "inspect", "--format", "{{json .State}}", id], true);
+            const state = JSON.parse(terminal.stdout);
+            if (result.code !== 0 || result.stdout.trim() !== "spawnfile-output-copy-complete" || terminal.code !== 0 || state.Running !== false || state.ExitCode !== 0)
+                throw Error(`Training output ${direction} failed (copy incomplete or exceeds limits): ${result.stderr.replace(/\p{Cc}/gu, " ").slice(0, 2048)}`);
+        }
+        finally {
+            if (creationAttempted) {
+                const found = await call(["container", "inspect", "--format", inspectFormat, name], true);
+                if (found.code === 0) {
+                    const id = JSON.parse(found.stdout.split("\n")[0]);
+                    if (typeof id !== "string" || parseDetachedContainerInspect(found.stdout, id, labels, name).imageId !== options.image || (containerId && /^[a-f0-9]{64}$/u.test(containerId) && id !== containerId))
+                        throw Error("Refusing cleanup of unverified training output helper");
+                    const removed = await call(["container", "rm", "--force", id], true);
+                    if (removed.code !== 0)
+                        throw Error("Cannot remove training output helper");
+                }
+                const remaining = await call(["container", "ls", "--all", "--filter", `name=^/${name}$`, "--format", "{{.ID}}"], true);
+                if (remaining.code !== 0 || remaining.stdout.trim())
+                    throw Error("Training output helper closure is unknown");
+                helperClosed = true;
+            }
+        }
+        if (direction === "export") {
+            if (await realpath(options.host) !== options.host || !(await lstat(options.host)).isDirectory())
+                throw Error("Training host output changed before export");
+            const backup = path.join(options.scratch, "output-before-export");
+            await rename(options.host, backup);
+            try {
+                await rename(hostStage, options.host);
+            }
+            catch (error) {
+                await rename(backup, options.host);
+                throw error;
+            }
+            for (const directory of [options.scratch, path.dirname(options.host)]) {
+                const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
+                try {
+                    await handle.sync();
+                }
+                finally {
+                    await handle.close();
+                }
+            }
+        }
+    };
+    return {
+        safeToCleanup: () => helperClosed && (!exportAttempted || exported),
+        mount: `type=volume,src=${volume},dst=${target},volume-nocopy`,
+        async seed() {
+            attempted = true;
+            const created = await call(["volume", "create", "--driver", "local", "--label", `${ownerLabel}=${options.name}`, volume]);
+            if (created.code !== 0 || created.stdout.trim() !== volume || !await owned())
+                throw Error("Cannot create owned training output volume");
+            await transfer("seed");
+            seeded = true;
+            options.stderr(`Training output uses VM-local volume ${volume}; host checkpoints update at exit.`);
+        },
+        async export() {
+            if (!seeded || exportAttempted)
+                return;
+            exportAttempted = true;
+            try {
+                await transfer("export");
+                exported = true;
+            }
+            catch (error) {
+                options.stderr(`Training output export failed: ${error instanceof Error ? error.message : "unknown failure"}. Scratch retained at ${options.scratch}. ${recovery()}`);
+                throw Error(`Training output export failed. ${recovery()}`, { cause: error });
+            }
+        },
+        async dispose(closureVerified) {
+            if (!attempted)
+                return;
+            try {
+                if (!await owned())
+                    return;
+                if (!helperClosed)
+                    throw Error("Training output helper closure is unverified");
+                if (!closureVerified) {
+                    options.stderr(recovery());
+                    return;
+                }
+                if (seeded && !exported)
+                    throw Error("Training output has not been exported");
+                const removed = await call(["volume", "rm", volume], true);
+                const remaining = await call(["volume", "ls", "--filter", `name=^${volume}$`, "--format", "{{.Name}}"], true);
+                if (removed.code !== 0 || remaining.code !== 0 || remaining.stdout.trim())
+                    throw Error("Training output volume removal is unverified");
+            }
+            catch (error) {
+                options.stderr(recovery());
+                throw Error(recovery(), { cause: error });
+            }
+        }
+    };
+};

package/dist/compiler/training/container/prepare.d.ts

@@ -1,6 +1,8 @@
 import { type TrainingContext } from "../contract.js";
 import { type TrainingContainerConfig } from "./contract.js";
-export declare const prepareTrainingContainer: (raw: unknown, context: TrainingContext, args: readonly string[]) => Promise<{
+export declare const prepareTrainingContainer: (raw: unknown, context: TrainingContext, args: readonly string[], options?: {
+    createOutput?: boolean;
+}) => Promise<{
     config: TrainingContainerConfig;
     context: TrainingContext;
     args: string[];