spawnfile 0.1.14 → 0.1.18

package/dist/compiler/training/broker/paths.js

@@ -0,0 +1,249 @@
+import path from "node:path";
+import { DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_SUBSCRIPTION_REALM, DAIMON_GROK_TURN_USAGE_LEDGER } from "../../../runtime/daimon/contractManifest.js";
+import { DAIMON_BROKER_UID, DAIMON_FIRST_WORKER_UID, DAIMON_ORGANIZATION_UID } from "../../../runtime/daimon/runtimeIdentity.js";
+import { DAIMON_WAKE_FUSE_DIRECTORY } from "../../../runtime/daimon/config.js";
+import { DAIMON_WORKER_ROOT } from "../../containerDaimonGrokWorkerRender.js";
+/**
+ * Every container path the broker-capable training container fixes.
+ *
+ * They are deterministic because three parties must agree on them
+ * without talking to each other: the root entrypoint that provisions them, the
+ * root slot supervisor that wipes and re-provisions them on recycle, and
+ * Paideia's native adapter, which writes them into `paideia.daimon-native.launch.v2`
+ * and has no way to ask the container what it chose. Daimon's projection never
+ * resolves a path, so each one must also be canonical and never a symlink; the
+ * provisioning program asserts that before a slot is used.
+ *
+ * Everything under `/run/training/slots/<i>` is per-trial state on tmpfs — never the
+ * Grok realm volume (R2). The realm volume holds only `auth.json` and the
+ * broker credential journal, so a recycle can wipe the slot without touching
+ * the one durable rotating credential.
+ */
+export const TRAINING_RUN_ROOT = "/run/training/output";
+export const TRAINING_SEALED_INPUTS_ROOT = "/run/training/inputs";
+export const TRAINING_PAIDEIA_ROOT = "/run/paideia";
+export const TRAINING_CONTEXT_FILE = `${TRAINING_PAIDEIA_ROOT}/context.json`;
+/** Host-written, read-only `spawnfile.training-broker.v2` declaration the root entrypoint reads. */
+export const TRAINING_BROKER_DECLARATION_FILE = `${TRAINING_PAIDEIA_ROOT}/training-broker.json`;
+export const TRAINING_INFERENCE_DIRECTORY = "/run/training/inference";
+export const TRAINING_INFERENCE_LEDGER = `${TRAINING_INFERENCE_DIRECTORY}/inference.jsonl`;
+/** Judge grant homes: `2000:2000 0700`, denied to every worker uid. Paideia reads it as `PAIDEIA_GROK_GRANT_HOME_ROOT`. */
+export const TRAINING_GRANT_HOME_ROOT = "/run/training/grants";
+/**
+ * Private temp for the broker and its relay (uid 2100, outside the organization
+ * group), deliberately **outside** `/run/daimon-engine-broker`.
+ *
+ * Production puts it at `<control root>/tmp`, which is fine there: that root is
+ * removed and recreated exactly once, at container start. Training
+ * re-provisions the same root on every recycle, and the launch mounts this
+ * directory as its own tmpfs — so a `tmp/` inside the cleared root is a *mount
+ * point*, and clearing a mount point fails `EBUSY`. A live P8 launch died
+ * exactly there. Its own tmpfs under `/run/training` is never a wipe target and
+ * is denied to every worker uid.
+ */
+export const TRAINING_BROKER_TMPDIR = "/run/training/broker-tmp";
+export const TRAINING_SUPERVISOR_DIRECTORY = "/run/training/supervisor";
+export const TRAINING_SUPERVISOR_SOCKET = `${TRAINING_SUPERVISOR_DIRECTORY}/control.sock`;
+export const TRAINING_SUPERVISOR_LOG = `${TRAINING_SUPERVISOR_DIRECTORY}/supervisor.log`;
+export const TRAINING_WORKER_ROOT = DAIMON_WORKER_ROOT;
+/**
+ * Paideia's own fixed container paths (`containerPaths` in its native launch
+ * schema). They are *not* individual deny entries: Grok 1.0.34 materializes
+ * every `deny` target inside bubblewrap as the worker uid, and it cannot
+ * create a file inside `/run/paideia`, which belongs to uid 2000 alone. The
+ * single `/run/paideia` mask below covers all of them, which is also stronger
+ * — a file Paideia adds later is covered without re-provisioning the slot.
+ */
+export const TRAINING_CALLER_PROTECTED_PATHS = [
+    `${TRAINING_PAIDEIA_ROOT}/config.json`,
+    `${TRAINING_PAIDEIA_ROOT}/control`,
+    `${TRAINING_PAIDEIA_ROOT}/launch.json`,
+    `${TRAINING_PAIDEIA_ROOT}/token`,
+    `${TRAINING_PAIDEIA_ROOT}/env`,
+    `${TRAINING_PAIDEIA_ROOT}/preparation.json`,
+    `${TRAINING_PAIDEIA_ROOT}/repair.json`
+];
+/**
+ * `paideia.daimon-native.launch.v2`'s `broker.evaluatorPaths`, in full.
+ *
+ * This is not a summary of the five roles — it is the **whole** set of deny
+ * entries this container adds beyond Daimon's own protected paths, because
+ * Paideia's worker resolves the projection from exactly
+ * `[...new Set(broker.evaluatorPaths.map(row => row.path))].sort()`. A launch
+ * that carried only the five roles made Daimon hash a different deny list than
+ * the one provisioning wrote, and a live run died on
+ * `Grok broker projection digest differs from the launch receipt`.
+ *
+ * `role` is Paideia's closed five-value enum and may repeat; only `path` is
+ * unique. The four roles with a dedicated path keep it, and everything else is
+ * tagged `context` — the caller-state role — since there is no other way to
+ * carry a path through that contract.
+ */
+export const TRAINING_SLOTS_FILE = `${TRAINING_PAIDEIA_ROOT}/slots.json`;
+export const TRAINING_SLOTS_VERSION = "spawnfile.training-slots.v1";
+export const TRAINING_DEFAULT_SLOTS = 4;
+export const TRAINING_MAX_SLOTS = 8;
+/** 100 ports per slot: offsets 0=Moltnet, 1=worker control, 2=bridge; 3..99 reserved. */
+export const TRAINING_PORT_BASE = 20_000;
+export const TRAINING_PORT_STRIDE = 100;
+export const TRAINING_BROKER_SOCKET_PATHS = [
+    DAIMON_GROK_ENGINE_BROKER.controlSocketPath, DAIMON_GROK_ENGINE_BROKER.backendSocketPath,
+    DAIMON_GROK_ENGINE_BROKER.launcherSocketPath
+];
+export const TRAINING_TURN_STORE_ROOT = "/run/training/turns";
+/** Persistent, broker-only run counter; never part of a slot recycle. */
+export const TRAINING_RUN_REQUEST_STORE = "/run/training/request-budget";
+export const assertTrainingSlotCount = (slots) => {
+    if (!Number.isInteger(slots) || slots < 1 || slots > TRAINING_MAX_SLOTS)
+        throw Error("Training slots must be an integer from 1 to 8");
+    return slots;
+};
+export const trainingSlotPaths = (slot) => {
+    if (!Number.isInteger(slot) || slot < 0 || slot >= TRAINING_MAX_SLOTS)
+        throw Error("Training slot index must be an integer from 0 to 7");
+    const root = `/run/training/slots/${slot}`;
+    const workerUid = DAIMON_FIRST_WORKER_UID + slot;
+    const workerHome = `${TRAINING_WORKER_ROOT}/${workerUid}`;
+    const portBase = TRAINING_PORT_BASE + slot * TRAINING_PORT_STRIDE;
+    const clockDirectory = `/run/daimon-engine-broker/task-clock/slot-${slot}`;
+    return {
+        slot, workerUid, root, workerHome, workspace: `${root}/workspace`, runtimeHome: `${root}/runtime-home`,
+        state: `${root}/state`, acceptanceStore: `${root}/state/wake-acceptance`,
+        turns: `${TRAINING_TURN_STORE_ROOT}/slot-${slot}`, usageDirectory: `${root}/usage`, usageLedger: `${root}/usage/usage.jsonl`,
+        preflightReceipt: `${root}/preflight.json`, generationFile: `${root}/generation.json`,
+        sealedInputsAttestation: `${root}/sealed-inputs.json`, privateTmp: `${workerHome}/tmp`,
+        clockDirectory, clockFile: `${clockDirectory}/faketime`,
+        ports: { base: portBase, stride: TRAINING_PORT_STRIDE, moltnet: portBase, workerControl: portBase + 1, bridge: portBase + 2 }
+    };
+};
+/** The complete additional projection deny set. Sibling roots and homes never nest with own masks. */
+export const trainingEvaluatorPaths = (slot, slots) => {
+    assertTrainingSlotCount(slots);
+    if (slot >= slots)
+        throw Error("Training slot is outside the declared pool");
+    const own = trainingSlotPaths(slot);
+    return [
+        { role: "run-root", path: TRAINING_RUN_ROOT },
+        { role: "sealed-inputs", path: TRAINING_SEALED_INPUTS_ROOT },
+        { role: "judge-home", path: TRAINING_GRANT_HOME_ROOT },
+        { role: "slot-ledger", path: own.usageDirectory },
+        ...[TRAINING_PAIDEIA_ROOT, TRAINING_BROKER_TMPDIR, TRAINING_INFERENCE_DIRECTORY,
+            TRAINING_TURN_STORE_ROOT, TRAINING_RUN_REQUEST_STORE, TRAINING_SUPERVISOR_DIRECTORY,
+            ...TRAINING_BROKER_SOCKET_PATHS,
+            path.posix.dirname(DAIMON_GROK_ENGINE_BROKER.registrationPath),
+            DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, DAIMON_WAKE_FUSE_DIRECTORY,
+            ...Array.from({ length: slots }, (_, i) => trainingSlotPaths(i)).filter(p => p.slot !== slot)
+                .flatMap(p => [p.root, p.workerHome, p.clockDirectory])
+        ].map(path => ({ role: "context", path }))
+    ];
+};
+export const trainingSlotDescriptors = (slots = TRAINING_DEFAULT_SLOTS) => Array.from({ length: assertTrainingSlotCount(slots) }, (_, slot) => ({
+    ...trainingSlotPaths(slot), evaluatorPaths: trainingEvaluatorPaths(slot, slots),
+    containerPaths: TRAINING_CALLER_PROTECTED_PATHS.map(path => ({ path })),
+    controlRoot: trainingSlotPaths(slot).state
+}));
+export const trainingSlotsDocument = (slots = TRAINING_DEFAULT_SLOTS) => ({
+    schema: TRAINING_SLOTS_VERSION,
+    slots: trainingSlotDescriptors(slots).map(p => ({
+        index: p.slot, workerUid: p.workerUid,
+        // Paideia owns a runtime-neutral lease pool: its strict descriptor carries opaque named paths.
+        roots: {
+            ...Object.fromEntries(Object.entries(p).filter((entry) => typeof entry[1] === "string")),
+            home: p.runtimeHome, control: p.state, context: TRAINING_PAIDEIA_ROOT,
+            ...Object.fromEntries(p.evaluatorPaths.map((entry, index) => [`evaluator:${entry.role}:${index}`, entry.path])),
+            ...Object.fromEntries(p.containerPaths.map((entry, index) => [`container:${index}`, entry.path]))
+        },
+        ports: { base: p.ports.base, count: p.ports.stride }
+    }))
+});
+/** Both sides derive the descriptor bytes; a stale or edited bind must never certify a different pool. */
+export const assertTrainingSlotsDocument = (value, slots) => {
+    if (JSON.stringify(value) !== JSON.stringify(trainingSlotsDocument(slots)))
+        throw Error("Training slots manifest differs from the broker declaration");
+};
+/** Shared deny targets exist before worker attestation; no per-slot subtree is touched here. */
+export const TRAINING_OPTIONAL_DENY_DIRECTORIES = [
+    TRAINING_BROKER_TMPDIR, TRAINING_INFERENCE_DIRECTORY, TRAINING_TURN_STORE_ROOT,
+    TRAINING_SUPERVISOR_DIRECTORY, TRAINING_GRANT_HOME_ROOT,
+    DAIMON_GROK_TURN_USAGE_LEDGER.directoryPath, DAIMON_WAKE_FUSE_DIRECTORY
+];
+/**
+ * Deny entries a launch may legitimately not have mounted. Everything else must
+ * exist before a worker turn: Grok 1.0.34 creates an absent `deny` target as
+ * the worker uid inside bubblewrap and refuses the whole profile when it
+ * cannot, so provisioning materializes every other entry itself.
+ */
+export const TRAINING_DEFERRED_DENY_PATHS = [
+    // Docker materializes both host binds before the entrypoint runs. They sit on the read-only image
+    // root, which root cannot create into, so provisioning must tolerate a launch that declared neither.
+    TRAINING_RUN_ROOT,
+    TRAINING_SEALED_INPUTS_ROOT
+];
+/**
+ * Deny entries the launch binds from the host, named so a refusal can say why.
+ *
+ * Only the run root is one. The sealed inputs root is **not**: the launch binds
+ * each declared input at `/run/training/inputs/<id>`, so `/run/training/inputs`
+ * itself is the binds' *parent*, a directory on the read-only image root — and
+ * that is exactly what makes it sealable (see `TRAINING_SEALED_DENY_PATHS`).
+ *
+ * Being a host bind is no longer a waiver on its own. On Docker Desktop and
+ * Colima a bind lands on virtiofs/grpcfuse, which ignores `chown`, and there a
+ * worker-uid probe carries no information; on a Linux daemon over ext4, xfs,
+ * btrfs or overlay the same bind enforces ownership and the probe is real. The
+ * canary resolver asks the backing filesystem rather than this list, which is
+ * why the documented `refuse` default is reachable at all.
+ */
+export const TRAINING_HOST_BIND_DENY_PATHS = [TRAINING_RUN_ROOT, DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath];
+/**
+ * Deny entries whose denial must be proven by the kernel under **every**
+ * policy, because the experiment's central claim depends on them.
+ *
+ * `/run/training/inputs` holds the sealed train and test datasets. A bubblewrap
+ * `deny` mask is not a boundary the subject cannot lift: the pinned seccomp
+ * profile must allow `clone/unshare/mount/umount2/pivot_root/setns` for
+ * bubblewrap itself, and the container runs `apparmor=unconfined`, so a
+ * worker-uid process can open a user + mount namespace of its own and either
+ * detach the mask or bind the mask's parent somewhere fresh.
+ *
+ * What survives that is DAC on the datasets' ancestor. The image bakes this
+ * directory `0:<organization gid> 0750` on the read-only root, so:
+ *
+ *   - the worker uid is in neither the owner nor the group class and loses
+ *     search permission on the one directory every dataset read must traverse;
+ *   - the inode is owned by real uid 0, which a `unshare --map-root-user`
+ *     namespace does not map, so `CAP_DAC_OVERRIDE` there cannot override it;
+ *   - the root filesystem is read-only, so nothing in the container — root
+ *     included — can loosen the mode after the image is built.
+ *
+ * Provisioning asserts the mode and then attacks it as the worker uid, and the
+ * canary resolver refuses the slot rather than certifying one of these on the
+ * sandbox profile alone.
+ */
+export const TRAINING_SEALED_DENY_PATHS = [TRAINING_SEALED_INPUTS_ROOT];
+/** Spawnfile-owned per-route seal attestation, written by provisioning beside the slot preflight receipt. */
+/** The image-baked identity of the sealed inputs root, asserted before every slot and never writable at runtime. */
+export const TRAINING_SEALED_INPUTS_IDENTITY = { uid: 0, gid: DAIMON_ORGANIZATION_UID, mode: "750" };
+/**
+ * What a recycle removes outright, and what it only empties.
+ *
+ * Nothing the launch mounts may be at or below any of these. A mount point
+ * cannot be unlinked while it is mounted, so a wipe target holding one aborts
+ * the recycle — which is how the broker/relay `TMPDIR` took down a live launch
+ * when it still lived at `<control root>/tmp`. `paths.test.ts` enforces that
+ * against the launch's own mount list; the rendered shell skips mount points
+ * anyway, so an undeclared one degrades to "left in place and reported" rather
+ * than a failed provision.
+ */
+export const trainingWipeTargets = (slot) => {
+    const p = trainingSlotPaths(slot);
+    return [p.runtimeHome, p.state, p.preflightReceipt, p.sealedInputsAttestation];
+};
+/** Broker-owned turns and clocks are reset through its control seam, never by filesystem deletion. */
+export const trainingClearTargets = (slot) => {
+    const p = trainingSlotPaths(slot);
+    return [p.workspace, p.usageDirectory];
+};
+export const TRAINING_REALM_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.durableMountPath;
+export const TRAINING_BOOTSTRAP_MOUNT = DAIMON_GROK_SUBSCRIPTION_REALM.bootstrapMountPath;
+export { DAIMON_BROKER_UID, DAIMON_ORGANIZATION_UID };

package/dist/compiler/training/broker/processes.d.ts

@@ -0,0 +1,51 @@
+import { type ChildProcess } from "node:child_process";
+interface SocketIdentity {
+    dev: number;
+    ino: number;
+    ctimeMs: number;
+    isSocket(): boolean;
+}
+/** An existing placement socket is not readiness. Its inode (or reuse timestamp) must change. */
+export declare const isStartedBrokerSocket: (current: SocketIdentity | undefined, previous: SocketIdentity | undefined) => boolean;
+export declare const DAIMON_RUNTIME_ROOT = "/opt/spawnfile/runtime-installs/daimon";
+/** `00000000000000c1` = CHOWN|SETGID|SETUID, the launcher's bounding set after `setpriv`. */
+export declare const LAUNCHER_CAPABILITY_BOUND = "00000000000000c1";
+export declare const DROPPED_CAPABILITY_BOUND = "0000000000000000";
+export interface BrokerChild {
+    name: string;
+    pid: number;
+    child: ChildProcess;
+    uid: number;
+    capBnd: string;
+}
+/**
+ * The three broker processes, in the order and with the identities production
+ * starts them: the root launcher (which alone may `setuid` to a worker uid),
+ * the broker backend as uid 2100, and the native control relay as uid 2100.
+ * Each one runs with the same bounding set the production entrypoint verifies,
+ * and the broker and relay carry their own `TMPDIR` because shared `/tmp` is
+ * closed to every process outside the organization group.
+ */
+export declare const brokerProcessPlan: () => {
+    name: string;
+    argv: string[];
+    socket: string;
+    uid: number;
+    capBnd: string;
+}[];
+export interface StartBrokerOptions {
+    timeoutMs?: number;
+    pollMs?: number;
+    procRoot?: string;
+    log(line: string): void;
+}
+/**
+ * Starts all three, then proves each one reached its socket *and* its expected
+ * post-drop identity before the next starts. A broker that is alive but still
+ * root, or a relay whose socket never appeared, fails the slot here rather
+ * than at the first trial wake.
+ */
+export declare const startBrokerProcesses: (options: StartBrokerOptions) => Promise<BrokerChild[]>;
+/** Graceful stop in reverse start order: relay, backend, launcher. `SIGKILL` only after the grace window. */
+export declare const stopBrokerProcesses: (children: readonly BrokerChild[], log: (line: string) => void, graceMs?: number) => Promise<void>;
+export {};

package/dist/compiler/training/broker/processes.js

@@ -0,0 +1,102 @@
+import { spawn } from "node:child_process";
+import { readFile, stat } from "node:fs/promises";
+import { setTimeout as pause } from "node:timers/promises";
+import { SpawnfileError } from "../../../shared/index.js";
+import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js";
+import { DAIMON_BROKER_UID, TRAINING_BROKER_TMPDIR } from "./paths.js";
+/** An existing placement socket is not readiness. Its inode (or reuse timestamp) must change. */
+export const isStartedBrokerSocket = (current, previous) => current?.isSocket() === true && (!previous || current.dev !== previous.dev || current.ino !== previous.ino || current.ctimeMs !== previous.ctimeMs);
+export const DAIMON_RUNTIME_ROOT = "/opt/spawnfile/runtime-installs/daimon";
+/** `00000000000000c1` = CHOWN|SETGID|SETUID, the launcher's bounding set after `setpriv`. */
+export const LAUNCHER_CAPABILITY_BOUND = "00000000000000c1";
+export const DROPPED_CAPABILITY_BOUND = "0000000000000000";
+const setpriv = (args) => ["setpriv", ...args];
+/**
+ * The three broker processes, in the order and with the identities production
+ * starts them: the root launcher (which alone may `setuid` to a worker uid),
+ * the broker backend as uid 2100, and the native control relay as uid 2100.
+ * Each one runs with the same bounding set the production entrypoint verifies,
+ * and the broker and relay carry their own `TMPDIR` because shared `/tmp` is
+ * closed to every process outside the organization group.
+ */
+export const brokerProcessPlan = () => [
+    {
+        name: "engine broker launcher", socket: DAIMON_GROK_ENGINE_BROKER.launcherSocketPath, uid: 0, capBnd: LAUNCHER_CAPABILITY_BOUND,
+        argv: setpriv(["--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all,+chown,+setuid,+setgid", "--", DAIMON_GROK_ENGINE_BROKER.nativeExecutablePath])
+    },
+    {
+        name: "engine broker backend", socket: DAIMON_GROK_ENGINE_BROKER.backendSocketPath, uid: DAIMON_BROKER_UID, capBnd: DROPPED_CAPABILITY_BOUND,
+        argv: setpriv(["--clear-groups", `--reuid=${DAIMON_BROKER_UID}`, `--regid=${DAIMON_BROKER_UID}`, "--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all",
+            "--", "env", `TMPDIR=${TRAINING_BROKER_TMPDIR}`, `${DAIMON_RUNTIME_ROOT}/bin/daimon-runtime`, "engine-broker", "serve"])
+    },
+    {
+        name: "engine broker control relay", socket: DAIMON_GROK_ENGINE_BROKER.controlSocketPath, uid: DAIMON_BROKER_UID, capBnd: DROPPED_CAPABILITY_BOUND,
+        argv: setpriv(["--inh-caps=-all", "--ambient-caps=-all", "--bounding-set=-all,+chown,+setuid,+setgid,+setpcap", "--", "env", `TMPDIR=${TRAINING_BROKER_TMPDIR}`,
+            DAIMON_GROK_ENGINE_BROKER.nativeExecutablePath, "--relay"])
+    }
+];
+const processIdentity = async (pid, procRoot) => {
+    const status = await readFile(`${procRoot}/${pid}/status`, "utf8");
+    return {
+        uid: /^Uid:\s+(\d+)/mu.exec(status)?.[1] ?? "",
+        capBnd: /^CapBnd:\s+([0-9a-f]+)/mu.exec(status)?.[1] ?? ""
+    };
+};
+/**
+ * Starts all three, then proves each one reached its socket *and* its expected
+ * post-drop identity before the next starts. A broker that is alive but still
+ * root, or a relay whose socket never appeared, fails the slot here rather
+ * than at the first trial wake.
+ */
+export const startBrokerProcesses = async (options) => {
+    const timeoutMs = options.timeoutMs ?? 60_000, pollMs = options.pollMs ?? 100, procRoot = options.procRoot ?? "/proc";
+    const started = [];
+    const deadline = Date.now() + timeoutMs;
+    try {
+        for (const entry of brokerProcessPlan()) {
+            const previousSocket = await stat(entry.socket).catch(() => undefined);
+            const child = spawn(entry.argv[0], entry.argv.slice(1), { stdio: ["ignore", "inherit", "inherit"] });
+            if (child.pid === undefined)
+                throw new SpawnfileError("runtime_error", `${entry.name} did not start`);
+            let exited = false;
+            child.once("exit", (code) => { exited = true; options.log(`${entry.name} exited with status ${code ?? "signal"}`); });
+            started.push({ name: entry.name, pid: child.pid, child, uid: entry.uid, capBnd: entry.capBnd });
+            for (;;) {
+                if (exited)
+                    throw new SpawnfileError("runtime_error", `${entry.name} exited before readiness`);
+                if (Date.now() > deadline)
+                    throw new SpawnfileError("runtime_error", `${entry.name} readiness timed out`);
+                const socket = await stat(entry.socket).catch(() => undefined);
+                if (isStartedBrokerSocket(socket, previousSocket)) {
+                    const identity = await processIdentity(child.pid, procRoot);
+                    if (identity.uid === String(entry.uid) && identity.capBnd === entry.capBnd)
+                        break;
+                }
+                await pause(pollMs);
+            }
+            options.log(`${entry.name} ready pid=${child.pid} uid=${entry.uid} capbnd=${entry.capBnd}`);
+        }
+        return started;
+    }
+    catch (error) {
+        await stopBrokerProcesses(started, options.log);
+        throw error;
+    }
+};
+/** Graceful stop in reverse start order: relay, backend, launcher. `SIGKILL` only after the grace window. */
+export const stopBrokerProcesses = async (children, log, graceMs = 5_000) => {
+    for (const entry of [...children].reverse()) {
+        if (entry.child.exitCode !== null || entry.child.signalCode !== null)
+            continue;
+        entry.child.kill("SIGTERM");
+        const deadline = Date.now() + graceMs;
+        while (entry.child.exitCode === null && entry.child.signalCode === null && Date.now() < deadline)
+            await pause(25);
+        if (entry.child.exitCode === null && entry.child.signalCode === null) {
+            log(`${entry.name} did not stop gracefully; killing`);
+            entry.child.kill("SIGKILL");
+        }
+        while (entry.child.exitCode === null && entry.child.signalCode === null)
+            await pause(25);
+    }
+};

package/dist/compiler/training/broker/projection.d.ts

@@ -0,0 +1,52 @@
+import { type DaimonGrokBrokerModel, type DaimonGrokBrokerReasoningEffort } from "../../../runtime/daimon/contractManifest.js";
+import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js";
+import type { TrainingBrokerDeclaration } from "./declaration.js";
+/**
+ * The slice of Daimon's public `@noopolis/daimon/runtime` export the slot
+ * supervisor needs. Daimon stays the resolver: the slot preflight receipt binds
+ * `projection_sha256`, and a second implementation of that digest would fork
+ * the contract the moment either side changed a member.
+ */
+export interface DaimonProjectionModule {
+    resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: Record<string, unknown>): {
+        version: string;
+        profileSha256: string;
+        denyPaths: readonly string[];
+    };
+    grokBrokerProjectionSha256(projection: unknown): string;
+    GROK_ENGINE_BROKER: {
+        projectionVersion: string;
+        slotPreflightVersion: string;
+        nativeAbiVersion: number;
+        controlProtocolVersion: string;
+    };
+}
+export declare const loadDaimonProjectionModule: (load?: () => Promise<unknown>) => Promise<DaimonProjectionModule>;
+/**
+ * The one-agent organization runtime config the training slot's projection is
+ * resolved from.
+ *
+ * It is a projection *input*, never a runtime config Daimon hosts: the trial's
+ * real config is Paideia's. Only `agents[].id`, `workspacePath`,
+ * `runtimeHomePath`, the engine's declared model and effort, and the *absence*
+ * of peers reach `resolveOrganizationGrokBrokerProjection`, so this minimal
+ * config and Paideia's fuller one compute the same projection digest. A peer
+ * agent would change Daimon's protected set and the digest with it, which is
+ * why each slot resolves its own one-agent projection with explicit sibling deny paths.
+ */
+export declare const trainingOrganizationRuntimeConfig: (declaration: {
+    agentId: string;
+    model: DaimonGrokBrokerModel;
+    reasoningEffort: DaimonGrokBrokerReasoningEffort;
+}, slot?: number) => Record<string, unknown>;
+export interface TrainingGrokProjection {
+    projection: Record<string, unknown>;
+    projectionSha256: string;
+}
+/**
+ * Resolve the slot's public Grok broker projection through Daimon, over exactly
+ * the provisioned registration. A profile digest that differs from Daimon's own
+ * render is refused there, so the receipt can never certify a slot whose deny
+ * list drifted from the bytes provisioning wrote.
+ */
+export declare const resolveTrainingGrokProjection: (declaration: TrainingBrokerDeclaration, registration: DaimonGrokRegistration, daimon: DaimonProjectionModule) => Promise<TrainingGrokProjection>;

package/dist/compiler/training/broker/projection.js

@@ -0,0 +1,88 @@
+import { SpawnfileError } from "../../../shared/index.js";
+import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js";
+import { trainingSlotPaths } from "./paths.js";
+/** Indirect specifier: Daimon is an in-image runtime peer of the training container, never a Spawnfile source dependency. */
+const loadRuntime = async () => {
+    const packageName = "@noopolis/daimon/runtime";
+    return import(packageName);
+};
+export const loadDaimonProjectionModule = async (load = loadRuntime) => {
+    let daimon;
+    try {
+        daimon = await load();
+    }
+    catch {
+        throw new SpawnfileError("runtime_error", "The training image must install @noopolis/daimon with its public /runtime export");
+    }
+    if (typeof daimon.resolveOrganizationGrokBrokerProjection !== "function" || typeof daimon.grokBrokerProjectionSha256 !== "function"
+        || daimon.GROK_ENGINE_BROKER?.projectionVersion !== DAIMON_GROK_ENGINE_BROKER.projectionVersion
+        || daimon.GROK_ENGINE_BROKER.controlProtocolVersion !== DAIMON_GROK_ENGINE_BROKER.controlProtocolVersion
+        || daimon.GROK_ENGINE_BROKER.slotPreflightVersion !== DAIMON_GROK_ENGINE_BROKER.slotPreflightVersion
+        || daimon.GROK_ENGINE_BROKER.nativeAbiVersion !== DAIMON_GROK_ENGINE_BROKER.nativeAbiVersion) {
+        throw new SpawnfileError("runtime_error", "The installed Daimon Grok broker contract differs from this compiler's vendored manifest");
+    }
+    return daimon;
+};
+/**
+ * The one-agent organization runtime config the training slot's projection is
+ * resolved from.
+ *
+ * It is a projection *input*, never a runtime config Daimon hosts: the trial's
+ * real config is Paideia's. Only `agents[].id`, `workspacePath`,
+ * `runtimeHomePath`, the engine's declared model and effort, and the *absence*
+ * of peers reach `resolveOrganizationGrokBrokerProjection`, so this minimal
+ * config and Paideia's fuller one compute the same projection digest. A peer
+ * agent would change Daimon's protected set and the digest with it, which is
+ * why each slot resolves its own one-agent projection with explicit sibling deny paths.
+ */
+export const trainingOrganizationRuntimeConfig = (declaration, slot = 0) => ({
+    version: "noopolis.daimon.organization-runtime.v1",
+    host: { bindHost: "127.0.0.1", port: trainingSlotPaths(slot).ports.workerControl, controlTokenEnv: "SPAWNFILE_DAIMON_CONTROL_TOKEN" },
+    agents: [{
+            id: declaration.agentId,
+            name: declaration.agentId,
+            instructions: "training subject",
+            workspacePath: trainingSlotPaths(slot).workspace,
+            runtimeHomePath: trainingSlotPaths(slot).runtimeHome,
+            engine: { kind: "grok", model: declaration.model, reasoningEffort: declaration.reasoningEffort }
+        }]
+});
+/**
+ * Resolve the slot's public Grok broker projection through Daimon, over exactly
+ * the provisioned registration. A profile digest that differs from Daimon's own
+ * render is refused there, so the receipt can never certify a slot whose deny
+ * list drifted from the bytes provisioning wrote.
+ */
+export const resolveTrainingGrokProjection = async (declaration, registration, daimon) => {
+    const projection = daimon.resolveOrganizationGrokBrokerProjection(trainingOrganizationRuntimeConfig(declaration, registration.slot), declaration.agentId, {
+        slot: registration.slot,
+        workerConfigVariant: "clocked",
+        workerUid: registration.uid,
+        workerHomePath: registration.home,
+        architecture: declaration.architecture,
+        usageLedgerPath: registration.usageLedgerPath,
+        limits: declaration.limits,
+        // The mask that protects the wake-acceptance store is the slot state root it lives in: the store's
+        // own parent is `2000:2000 0700`, so bubblewrap could not materialize a deny target inside it as
+        // the worker uid. Every party that recomputes this projection must pass the same value.
+        acceptanceStorePath: trainingSlotPaths(registration.slot).state,
+        denyPaths: registration.denyPaths,
+        seccompProfileSha256: declaration.seccompProfileSha256,
+        profileSha256: registration.profileSha256
+    });
+    const clock = DAIMON_GROK_ENGINE_BROKER.worker.taskClock;
+    const expectedClock = { ...clock, directoryPath: clock.slotDirectoryPattern.replace("{slot}", String(registration.slot)),
+        timestampFilePath: clock.slotFilePattern.replace("{slot}", String(registration.slot)) };
+    const actualClock = projection.taskClock;
+    if (registration.configVariant !== "clocked" || projection.version !== DAIMON_GROK_ENGINE_BROKER.projectionVersion
+        || projection.profileSha256 !== registration.profileSha256 || actualClock === undefined
+        || Object.entries(expectedClock).some(([key, value]) => actualClock[key] !== value)
+        || projection.workerConfigVariant !== "clocked" || projection.workerConfigSha256 !== registration.configSha256) {
+        throw new SpawnfileError("runtime_error", "Daimon's clocked worker config differs from the provisioned registration");
+    }
+    const denied = [...projection.denyPaths];
+    if (denied.length !== registration.denyPaths.length || denied.some((entry, index) => entry !== registration.denyPaths[index])) {
+        throw new SpawnfileError("runtime_error", "Daimon's Grok broker projection deny list differs from the provisioned sandbox profile");
+    }
+    return { projection, projectionSha256: daimon.grokBrokerProjectionSha256(projection) };
+};

package/dist/compiler/training/broker/provisioning.d.ts

@@ -0,0 +1,78 @@
+import type { DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js";
+/**
+ * Every directory the training slot owns, with the identity that must hold it.
+ *
+ * The two ledger directories are setgid to the organization group on purpose:
+ * the broker writes its rows `0640` owned by the broker group, so without
+ * `2100:2000 2750` uid 2000 — Paideia, DSPy and every judge — cannot read a
+ * single subject usage row or inference row it paid for.
+ */
+export declare const trainingSlotDirectories: (slot?: number) => readonly {
+    path: string;
+    mode: string;
+    uid: number;
+    gid: number;
+}[];
+/**
+ * The fixed identities the slot needs, checked rather than created.
+ *
+ * The broker-capable training container runs on a read-only root, so
+ * `/etc/passwd` and `/etc/group` cannot be written at start-up the way the
+ * production Daimon entrypoint writes them. The image bakes uid/gid 2000, 2100
+ * and the worker uid instead, and this refuses to provision a slot in an image
+ * that did not.
+ */
+export declare const renderTrainingIdentities: (slots?: number) => string[];
+/** Full container provisioning runs exactly once, before any broker process starts. */
+export declare const renderTrainingBrokerProvisioning: (input: DaimonGrokRegistration | readonly DaimonGrokRegistration[], maxRunRequests?: number) => string[];
+/** Reuse the audited worker renderer; never touch service config, credentials, shared temps or siblings. */
+export declare const renderTrainingSlotProvisioning: (registration: DaimonGrokRegistration) => string[];
+/**
+ * The sealed-inputs seal, asserted and then attacked before any slot is used.
+ *
+ * The declared datasets are bind-mounted at `/run/training/inputs/<id>`, so
+ * `/run/training/inputs` is their common ancestor and the only inode in the
+ * chain the container controls: it lives on the read-only image root, the image
+ * bakes it `0:<organization gid> 0750`, and nothing in the container can widen
+ * it afterwards. The worker uid is in neither its owner nor its group class, so
+ * it loses *search* permission on the directory every dataset read must
+ * traverse.
+ *
+ * Four attacker routes, not a cooperating tool's read: a direct
+ * read/search/list; a user + mount namespace of the worker's own with the
+ * bubblewrap mask and its parent lazily unmounted; a fresh `mount --bind` of
+ * `/run/training` made inside that namespace; and, per dataset, a `mount --bind`
+ * of the dataset's *own* mount, which carries no mask and — unlike its parent —
+ * is not refused for locked children. A live control container left at Docker's
+ * own `0755 root:root` hands the held-out answer key to uid 2200 through routes
+ * one, two and four (`.runtime/sealed-inputs-dac/EVIDENCE.md`).
+ *
+ * Every route reports one verdict, and the verdicts are deliberately not
+ * interchangeable:
+ *
+ *  - `reachable` — the bytes were read. Refuse.
+ *  - `denied at-read` — the route ran and the kernel's permission check refused
+ *    the open or the list. This is the DAC seal doing the work.
+ *  - `denied at-mount` — the mount the route needs was refused although the
+ *    syscall is available.
+ *  - `unavailable seccomp` / `unavailable kernel` — the worker uid cannot open
+ *    the namespace the route needs at all, so the route provably cannot happen.
+ *    That is a *stronger* denial than DAC, and naming which layer refused is
+ *    the point: `EPERM` from a seccomp filter and `EPERM` from a kernel or LSM
+ *    policy are indistinguishable by errno, so the mechanism is derived from
+ *    the pinned profile Spawnfile ships and the declaration's digest binds
+ *    (`seccompRoutes.ts`), never guessed.
+ *  - anything else — no verdict. Refuse. "Provably cannot happen" and "could
+ *    not tell" must never collapse into one pass.
+ *
+ * The probe's workspace is the worker's **private** tmp, never `/tmp`: the
+ * shared temps are provisioned `root:<organization gid> 1774` so a worker lists
+ * names only, and probing from there is what made the first live run refuse
+ * every trial on `namespace-rebind reached no verdict` — the `mkdir` failed, not
+ * the seal. The private tmp is also the workspace the subject itself has.
+ */
+export declare const renderTrainingSealedInputsAssertions: (workerUid?: number, privateTmp?: string, mechanism?: "seccomp" | "kernel") => string[];
+/** Where the broker's own `service.json` lands, so the supervisor can prove the slot it restarted is the slot it provisioned. */
+export declare const TRAINING_SERVICE_CONFIG_PATH: "/etc/daimon-engine-broker/service.json";
+export declare const TRAINING_REGISTRATION_PATH: "/etc/daimon-engine-broker/registrations.bin";
+export declare const TRAINING_BROKER_CONTROL_ROOT: string;