spawnfile 0.1.14 → 0.1.17

package/dist/compiler/containerBackedMountRender.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Fail-closed guard for durable container state.
+ *
+ * Every path in `container.persistent_mounts[]` is state the compiler expects
+ * a host-backed named volume to hold. `spawnfile run`/`spawnfile up` and
+ * `spawnfile deploy` mount them; a hand-rolled `docker run` of the same image
+ * does not, and nothing in the image declares `VOLUME`, so the container would
+ * silently write its whole durable record onto the container's own writable
+ * layer and lose it on `docker rm`. That is exactly how a newsroom's message
+ * history was destroyed by a routine recreate.
+ *
+ * These checks make that case loud instead of silent: the entrypoint refuses
+ * to start unless each durable path is a real mount point, naming the mount
+ * id, the path, and the volume name to attach. `SPAWNFILE_ALLOW_EPHEMERAL_STATE=1`
+ * is the deliberate opt-out for throwaway containers.
+ *
+ * Detection reads `/proc/self/mountinfo` and looks for an EXACT mount point,
+ * rather than comparing `stat -c %d` against the parent directory: two nested
+ * durable volumes live on the same host filesystem and therefore share a
+ * device number, which would make a device comparison report a genuinely
+ * backed inner mount as unbacked.
+ *
+ * The kernel octal-escapes four characters in the mountinfo mount-point field
+ * (`show_mountinfo` passes `" \t\n\\"` to `seq_path_root`), so a declared
+ * `mount: "/var/lib/my store"` appears there as `/var/lib/my\040store`. The
+ * comparison is therefore made against the compiler-escaped form — computed
+ * here, where it is exactly testable — while the operator-facing message keeps
+ * the real path. Escaping at compile time rather than decoding in shell keeps
+ * the check a deterministic string compare with no dependence on how a
+ * particular `printf %b` implementation treats `\nnn` vs `\0nnn`. For a path
+ * containing none of those four characters both forms are identical.
+ */
+export declare const EPHEMERAL_STATE_OPT_OUT_ENV = "SPAWNFILE_ALLOW_EPHEMERAL_STATE";
+export interface BackedMountRequirement {
+    id: string;
+    mount_path: string;
+    volume_name: string;
+}
+/** The mount-point field exactly as `/proc/self/mountinfo` renders this path. */
+export declare const escapeMountInfoPath: (value: string) => string;
+export declare const createBackedMountShellFunction: () => string[];
+/**
+ * One check per durable mount, deduplicated by path and ordered by path so the
+ * rendered entrypoint is a deterministic function of the compile plan.
+ */
+export declare const createBackedMountChecks: (mounts: readonly BackedMountRequirement[]) => string[];
+/**
+ * The full guard — function plus checks — or nothing at all when the compile
+ * declares no durable mounts.
+ */
+export declare const createBackedMountGuard: (mounts: readonly BackedMountRequirement[]) => string[];
+/**
+ * Compile-time operator surface: how many durable mounts this organization
+ * requires a launcher to attach.
+ */
+export declare const describeBackedMountRequirement: (mounts: readonly BackedMountRequirement[]) => string | null;

package/dist/compiler/containerBackedMountRender.js

@@ -0,0 +1,101 @@
+/**
+ * Fail-closed guard for durable container state.
+ *
+ * Every path in `container.persistent_mounts[]` is state the compiler expects
+ * a host-backed named volume to hold. `spawnfile run`/`spawnfile up` and
+ * `spawnfile deploy` mount them; a hand-rolled `docker run` of the same image
+ * does not, and nothing in the image declares `VOLUME`, so the container would
+ * silently write its whole durable record onto the container's own writable
+ * layer and lose it on `docker rm`. That is exactly how a newsroom's message
+ * history was destroyed by a routine recreate.
+ *
+ * These checks make that case loud instead of silent: the entrypoint refuses
+ * to start unless each durable path is a real mount point, naming the mount
+ * id, the path, and the volume name to attach. `SPAWNFILE_ALLOW_EPHEMERAL_STATE=1`
+ * is the deliberate opt-out for throwaway containers.
+ *
+ * Detection reads `/proc/self/mountinfo` and looks for an EXACT mount point,
+ * rather than comparing `stat -c %d` against the parent directory: two nested
+ * durable volumes live on the same host filesystem and therefore share a
+ * device number, which would make a device comparison report a genuinely
+ * backed inner mount as unbacked.
+ *
+ * The kernel octal-escapes four characters in the mountinfo mount-point field
+ * (`show_mountinfo` passes `" \t\n\\"` to `seq_path_root`), so a declared
+ * `mount: "/var/lib/my store"` appears there as `/var/lib/my\040store`. The
+ * comparison is therefore made against the compiler-escaped form — computed
+ * here, where it is exactly testable — while the operator-facing message keeps
+ * the real path. Escaping at compile time rather than decoding in shell keeps
+ * the check a deterministic string compare with no dependence on how a
+ * particular `printf %b` implementation treats `\nnn` vs `\0nnn`. For a path
+ * containing none of those four characters both forms are identical.
+ */
+export const EPHEMERAL_STATE_OPT_OUT_ENV = "SPAWNFILE_ALLOW_EPHEMERAL_STATE";
+const shellQuote = (value) => `'${value.replace(/'/g, `'"'"'`)}'`;
+const MOUNTINFO_ESCAPES = {
+    "\\": "\\134",
+    " ": "\\040",
+    "\t": "\\011",
+    "\n": "\\012"
+};
+/** The mount-point field exactly as `/proc/self/mountinfo` renders this path. */
+export const escapeMountInfoPath = (value) => value.replace(/[\\ \t\n]/gu, (character) => MOUNTINFO_ESCAPES[character]);
+export const createBackedMountShellFunction = () => [
+    "require_backed_mount() {",
+    '  local mount_id="$1"',
+    // As mountinfo renders it (octal-escaped), for the comparison only.
+    '  local mount_field="$2"',
+    // The real path, for every operator-facing message.
+    '  local mount_target="$3"',
+    '  local mount_volume="$4"',
+    "  local mount_point",
+    `  if [ "\${${EPHEMERAL_STATE_OPT_OUT_ENV}:-}" = "1" ]; then`,
+    "    return 0",
+    "  fi",
+    "  if [ ! -r /proc/self/mountinfo ]; then",
+    '    echo "Durable mount $mount_id at $mount_target cannot be verified: /proc/self/mountinfo is unreadable" >&2',
+    "    exit 1",
+    "  fi",
+    "  while read -r _ _ _ _ mount_point _; do",
+    '    if [ "$mount_point" = "$mount_field" ]; then',
+    "      return 0",
+    "    fi",
+    "  done < /proc/self/mountinfo",
+    '  echo "Durable mount $mount_id is not backed by a volume at $mount_target; this container would lose that state when it is removed." >&2',
+    `  echo "Start this image with 'spawnfile run' or 'spawnfile up', or pass: --mount type=volume,source=\$mount_volume,target=\$mount_target" >&2`,
+    `  echo "Set ${EPHEMERAL_STATE_OPT_OUT_ENV}=1 to run this container without durable state." >&2`,
+    "  exit 1",
+    "}"
+];
+/**
+ * One check per durable mount, deduplicated by path and ordered by path so the
+ * rendered entrypoint is a deterministic function of the compile plan.
+ */
+export const createBackedMountChecks = (mounts) => {
+    const byPath = new Map();
+    for (const mount of mounts) {
+        if (!byPath.has(mount.mount_path))
+            byPath.set(mount.mount_path, mount);
+    }
+    return [...byPath.values()]
+        .sort((left, right) => left.mount_path.localeCompare(right.mount_path))
+        .map((mount) => `require_backed_mount ${shellQuote(mount.id)} ${shellQuote(escapeMountInfoPath(mount.mount_path))} ${shellQuote(mount.mount_path)} ${shellQuote(mount.volume_name)}`);
+};
+/**
+ * The full guard — function plus checks — or nothing at all when the compile
+ * declares no durable mounts.
+ */
+export const createBackedMountGuard = (mounts) => {
+    const checks = createBackedMountChecks(mounts);
+    return checks.length === 0 ? [] : [...createBackedMountShellFunction(), "", ...checks, ""];
+};
+/**
+ * Compile-time operator surface: how many durable mounts this organization
+ * requires a launcher to attach.
+ */
+export const describeBackedMountRequirement = (mounts) => {
+    const count = new Set(mounts.map((mount) => mount.mount_path)).size;
+    if (count === 0)
+        return null;
+    return `${count} durable mount${count === 1 ? "" : "s"} must be attached at start: use 'spawnfile run'/'spawnfile up', or pass matching --mount type=volume args. The entrypoint refuses to start otherwise (set ${EPHEMERAL_STATE_OPT_OUT_ENV}=1 to accept state loss).`;
+};

package/dist/compiler/containerDaimonBrokerRender.d.ts

@@ -0,0 +1,112 @@
+import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js";
+import { type DaimonGrokRegistration, type DaimonGrokServiceConfigOptions } from "./containerDaimonGrokWorkerRender.js";
+import { DAIMON_BROKER_UID, DAIMON_FIRST_WORKER_UID, DAIMON_ORGANIZATION_UID } from "../runtime/daimon/runtimeIdentity.js";
+export { DAIMON_BROKER_UID, DAIMON_FIRST_WORKER_UID, DAIMON_ORGANIZATION_UID };
+export declare const DAIMON_BROKER_EXECUTABLE = "/opt/daimon/bin/daimon-engine-broker";
+export declare const DAIMON_BROKER_REGISTRATIONS = "/etc/daimon-engine-broker/registrations.bin";
+export declare const DAIMON_BROKER_SOCKET = "/run/daimon-engine-broker/control.sock";
+export declare const DAIMON_BROKER_BACKEND_SOCKET = "/run/daimon-engine-broker/backend.sock";
+export declare const DAIMON_BROKER_LAUNCHER_SOCKET = "/run/daimon-engine-broker/launcher.sock";
+export declare const DAIMON_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json";
+export declare const DAIMON_BROKER_REALM = "/var/lib/spawnfile/daimon/grok-subscription-realm";
+/**
+ * Private temp for the broker and its relay (uid 2100, outside the organization
+ * group): shared `/tmp` and `/var/tmp` are `root:2000 1774` in a Grok
+ * organization, so any non-root process outside group 2000 needs its own
+ * `TMPDIR`. It lives in the broker's `/run` directory, which every worker denies.
+ */
+export declare const DAIMON_BROKER_TMPDIR = "/run/daimon-engine-broker/tmp";
+export { DAIMON_ORGANIZATION_STATE_DIRECTORY, DAIMON_WORKER_ROOT, resolveDaimonGrokRegistrations } from "./containerDaimonGrokWorkerRender.js";
+interface WorkspaceSecurityResource {
+    backingPath: string;
+    kind: "bundle" | "git" | "volume";
+    linkPath: string;
+    mode: "mutable" | "readonly";
+    resolvedIdentity: string | null;
+}
+export declare const renderDaimonWorkspaceResourceSecurity: (resources: WorkspaceSecurityResource[], owners?: {
+    linkUid: number;
+    linkGid: number;
+    readonlyUid: number;
+    readonlyGid: number;
+    privilegedUid: number;
+    privilegedGid: number;
+}, resourceRoot?: string) => string[];
+/**
+ * Fixes ownership and mode of the per-turn usage ledger directory for every
+ * Daimon organization, not just ones with a Grok agent. AGY and Codex both
+ * write here too (`onTurnUsage` in Daimon's `engineDispatcher.ts`), from the
+ * organization-uid runtime process (`uid`/`gid` below, set in
+ * `renderDaimonUidEntrypoint`) rather than the privileged broker — so this
+ * must run whether or not any Grok registration exists, and the directory
+ * must be group-writable (0770), not merely group-readable (0750): a mode
+ * that only lets the group list the directory silently defeated every
+ * AGY/Codex advisory usage write and, with it, Daimon's wake-fuse token
+ * ceiling, which now refuses to start at all if this ledger is missing or
+ * unreadable (`wakeFuse.ts`'s `ensureUsageLedgerReadable`).
+ *
+ * The directory itself is never created here: it is a persistent volume
+ * mount (`daimon-grok-usage-ledger` in `config.ts`, unconditional for every
+ * Daimon organization) that Docker always materializes before the entrypoint
+ * runs — exactly like `DAIMON_WAKE_FUSE_DIRECTORY`, whose own fix-up
+ * (`renderDaimonUidEntrypoint`) uses this same three-step order. `chown` to
+ * root first, `chmod` next, `chown` to the final owner last — never
+ * `install -d`'s create-then-chown-then-chmod order, and never a single
+ * `chown owner:group` followed by `chmod`: root only ever chmods a path it
+ * currently owns, so it never needs `CAP_FOWNER` (`runProject.ts`'s
+ * capability set grants `CAP_CHOWN` but not `CAP_FOWNER`) — chmod-ing
+ * *after* the final `chown` hands ownership to the broker uid fails with
+ * `EPERM` the moment root no longer owns the path.
+ */
+export declare const renderDaimonUsageLedgerProvisioning: () => string[];
+/**
+ * Empties a directory without touching a mount point inside it.
+ *
+ * `rm -rf` and `find -delete` both fail `EBUSY` on a mount point — it is
+ * another filesystem's root, and its entry cannot be unlinked while it is
+ * mounted. A deployment that mounts a tmpfs below a directory it later clears
+ * would otherwise abort provisioning on a bare `find` failure, which is exactly
+ * how a live training launch died. Mount points are left in place and descended
+ * into; anything else that cannot be removed is reported by name.
+ *
+ * Mount points come from `/proc/self/mountinfo` field 5. The kernel
+ * octal-escapes space, tab, newline and backslash there; every path this
+ * compiler provisions is free of all four, and a path that is not would simply
+ * fail to match and be treated as an ordinary directory.
+ */
+export declare const MOUNT_AWARE_CLEAR_HELPER: string[];
+export declare const renderDaimonBrokerProvisioning: (plans: RuntimeTargetPlan[]) => string[];
+/**
+ * The root broker provisioning program itself, over already-resolved
+ * registrations and workspace resources.
+ *
+ * Production reaches it through {@link renderDaimonBrokerProvisioning}; the
+ * broker-capable training container reaches it directly for its single fixed
+ * slot, with `serviceOptions` pointing the turn store at per-slot tmpfs and
+ * declaring the evaluator inference ledger. Both callers get byte-identical
+ * credential, registration, worker-home, temp and spill provisioning — the
+ * whole point of sharing it rather than writing a second root program.
+ *
+ * It is re-runnable: the shell wrapper removes `/etc/daimon-engine-broker` and
+ * `/run/daimon-engine-broker` first, so a slot recycle replays exactly the
+ * audited start-up path, credential-journal recovery included.
+ */
+export declare const renderDaimonBrokerProvisioningProgram: (registrations: readonly DaimonGrokRegistration[], workspaceResources: WorkspaceSecurityResource[], serviceOptions?: DaimonGrokServiceConfigOptions, 
+/**
+ * How the broker's `/etc` and `/run` roots are reset before provisioning.
+ * A production organization owns both as ordinary image directories and
+ * removes them outright. The training container mounts each as its own
+ * tmpfs, which cannot be unlinked, so it empties them instead — the same
+ * end state, reached the only way a mount point allows.
+ */
+rootReset?: "remove" | "clear", 
+/** Deny targets the worker provisioning creates when absent; see `renderDaimonGrokWorkerProvisioning`. */
+optionalDenyPaths?: readonly string[], 
+/**
+ * Where the broker and relay get their private `TMPDIR`. Production keeps it
+ * inside the control root, which it removes and recreates exactly once. A
+ * deployment that re-provisions that root — the training slot recycle — must
+ * pass a path outside it: a `tmp/` inside a cleared root is a mount point in
+ * the training launch, and clearing a mount point fails `EBUSY`.
+ */
+brokerTmpdir?: string) => string[];

package/dist/compiler/containerDaimonBrokerRender.js

@@ -0,0 +1,191 @@
+import { DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_TURN_USAGE_LEDGER } from "../runtime/daimon/contractManifest.js";
+import { resolveDaimonGrokRegistrations } from "./containerDaimonGrokWorkerRender.js";
+import { renderDaimonGrokWorkerProvisioning } from "./containerDaimonGrokWorkerProvisioning.js";
+import { DAIMON_BROKER_UID, DAIMON_FIRST_WORKER_UID, DAIMON_ORGANIZATION_UID } from "../runtime/daimon/runtimeIdentity.js";
+export { DAIMON_BROKER_UID, DAIMON_FIRST_WORKER_UID, DAIMON_ORGANIZATION_UID };
+export const DAIMON_BROKER_EXECUTABLE = "/opt/daimon/bin/daimon-engine-broker";
+export const DAIMON_BROKER_REGISTRATIONS = "/etc/daimon-engine-broker/registrations.bin";
+export const DAIMON_BROKER_SOCKET = "/run/daimon-engine-broker/control.sock";
+export const DAIMON_BROKER_BACKEND_SOCKET = "/run/daimon-engine-broker/backend.sock";
+export const DAIMON_BROKER_LAUNCHER_SOCKET = "/run/daimon-engine-broker/launcher.sock";
+export const DAIMON_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json";
+export const DAIMON_BROKER_REALM = "/var/lib/spawnfile/daimon/grok-subscription-realm";
+/**
+ * Private temp for the broker and its relay (uid 2100, outside the organization
+ * group): shared `/tmp` and `/var/tmp` are `root:2000 1774` in a Grok
+ * organization, so any non-root process outside group 2000 needs its own
+ * `TMPDIR`. It lives in the broker's `/run` directory, which every worker denies.
+ */
+export const DAIMON_BROKER_TMPDIR = "/run/daimon-engine-broker/tmp";
+export { DAIMON_ORGANIZATION_STATE_DIRECTORY, DAIMON_WORKER_ROOT, resolveDaimonGrokRegistrations } from "./containerDaimonGrokWorkerRender.js";
+export const renderDaimonWorkspaceResourceSecurity = (resources, owners = { linkUid: 2_000, linkGid: 2_000, readonlyUid: 2_000, readonlyGid: 2_000, privilegedUid: 0, privilegedGid: 0 }, resourceRoot = "/var/lib/spawnfile/resources/") => [
+    `const workspaceResources = ${JSON.stringify(resources)};`,
+    "const resourceByLink = new Map(workspaceResources.map((resource) => [resource.linkPath, resource])); if (resourceByLink.size !== workspaceResources.length) throw new Error('duplicate worker workspace resource link');",
+    `const validateResourceLink = (target, info) => { const resource = resourceByLink.get(target); if (!resource || info.uid !== ${owners.linkUid} || info.gid !== ${owners.linkGid} || info.nlink !== 1) throw new Error('unsafe worker workspace link'); const raw = fs.readlinkSync(target), normalized = require('node:path').posix.normalize(raw); if (!raw.startsWith('/') || normalized !== raw || raw !== resource.backingPath || !raw.startsWith(${JSON.stringify(resourceRoot)})) throw new Error('unsafe worker workspace link target'); const backing = fs.lstatSync(raw); if (!backing.isDirectory() || backing.isSymbolicLink()) throw new Error('unsafe worker workspace resource'); const mode = backing.mode & 0o777; if (resource.kind === 'volume') { const lifecycleOwner = (backing.uid === ${owners.privilegedUid} && backing.gid === ${owners.privilegedGid}) || (backing.uid === ${owners.linkUid} && backing.gid === ${owners.linkGid}); if (!lifecycleOwner || mode !== 0o755 || typeof resource.resolvedIdentity !== 'string') throw new Error('unsafe worker workspace volume'); const expected = Buffer.from(\`${"${resource.resolvedIdentity}"}\\n\`); if (expected.length !== 72) throw new Error('unsafe worker workspace volume identity'); const sentinel = \`${"${raw}"}/.spawnfile-resource-identity\`; let fd; try { fd = fs.openSync(sentinel, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); const before = fs.fstatSync(fd); if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.uid !== ${owners.privilegedUid} || before.gid !== ${owners.privilegedGid} || (before.mode & 0o777) !== 0o644 || before.size !== expected.length) throw new Error('unsafe worker workspace volume identity'); const bytes = fs.readFileSync(fd), after = fs.fstatSync(fd); if (!bytes.equals(expected) || after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs) throw new Error('unsafe worker workspace volume identity'); } finally { expected.fill(0); if (fd !== undefined) fs.closeSync(fd); } } else if (resource.mode === 'readonly') { if (backing.uid !== ${owners.readonlyUid} || backing.gid !== ${owners.readonlyGid} || mode !== 0o555) throw new Error('unsafe readonly worker workspace resource'); } else if (backing.uid !== ${owners.privilegedUid} || backing.gid !== ${owners.privilegedGid} || mode !== 0o755) throw new Error('unsafe mutable worker workspace resource'); };`,
+    "const secureWorkspace = (root, uid) => { const visit = (target) => { const info = fs.lstatSync(target); if (info.isSymbolicLink()) { validateResourceLink(target, info); return; } if (info.isDirectory()) { fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o750); for (const name of fs.readdirSync(target)) visit(`${target}/${name}`); fs.chownSync(target, 2000, uid); } else if (info.isFile()) { fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o640); fs.chownSync(target, 2000, uid); } else throw new Error('unsafe worker workspace node'); }; visit(root); };"
+];
+/**
+ * Fixes ownership and mode of the per-turn usage ledger directory for every
+ * Daimon organization, not just ones with a Grok agent. AGY and Codex both
+ * write here too (`onTurnUsage` in Daimon's `engineDispatcher.ts`), from the
+ * organization-uid runtime process (`uid`/`gid` below, set in
+ * `renderDaimonUidEntrypoint`) rather than the privileged broker — so this
+ * must run whether or not any Grok registration exists, and the directory
+ * must be group-writable (0770), not merely group-readable (0750): a mode
+ * that only lets the group list the directory silently defeated every
+ * AGY/Codex advisory usage write and, with it, Daimon's wake-fuse token
+ * ceiling, which now refuses to start at all if this ledger is missing or
+ * unreadable (`wakeFuse.ts`'s `ensureUsageLedgerReadable`).
+ *
+ * The directory itself is never created here: it is a persistent volume
+ * mount (`daimon-grok-usage-ledger` in `config.ts`, unconditional for every
+ * Daimon organization) that Docker always materializes before the entrypoint
+ * runs — exactly like `DAIMON_WAKE_FUSE_DIRECTORY`, whose own fix-up
+ * (`renderDaimonUidEntrypoint`) uses this same three-step order. `chown` to
+ * root first, `chmod` next, `chown` to the final owner last — never
+ * `install -d`'s create-then-chown-then-chmod order, and never a single
+ * `chown owner:group` followed by `chmod`: root only ever chmods a path it
+ * currently owns, so it never needs `CAP_FOWNER` (`runProject.ts`'s
+ * capability set grants `CAP_CHOWN` but not `CAP_FOWNER`) — chmod-ing
+ * *after* the final `chown` hands ownership to the broker uid fails with
+ * `EPERM` the moment root no longer owns the path.
+ */
+export const renderDaimonUsageLedgerProvisioning = () => {
+    const { directoryPath } = DAIMON_GROK_TURN_USAGE_LEDGER;
+    return [
+        `chown 0:0 ${directoryPath} && chmod 0770 ${directoryPath} && chown ${DAIMON_BROKER_UID}:${DAIMON_ORGANIZATION_UID} ${directoryPath}`,
+        `setpriv --clear-groups --reuid ${DAIMON_ORGANIZATION_UID} --regid ${DAIMON_ORGANIZATION_UID} --inh-caps=-all --ambient-caps=-all --bounding-set=-all -- bash -ceu 'probe=${directoryPath}/.daimon-usage-probe; umask 007; : > "$probe"; rm "$probe"'`
+    ];
+};
+/**
+ * Empties a directory without touching a mount point inside it.
+ *
+ * `rm -rf` and `find -delete` both fail `EBUSY` on a mount point — it is
+ * another filesystem's root, and its entry cannot be unlinked while it is
+ * mounted. A deployment that mounts a tmpfs below a directory it later clears
+ * would otherwise abort provisioning on a bare `find` failure, which is exactly
+ * how a live training launch died. Mount points are left in place and descended
+ * into; anything else that cannot be removed is reported by name.
+ *
+ * Mount points come from `/proc/self/mountinfo` field 5. The kernel
+ * octal-escapes space, tab, newline and backslash there; every path this
+ * compiler provisions is free of all four, and a path that is not would simply
+ * fail to match and be treated as an ordinary directory.
+ */
+export const MOUNT_AWARE_CLEAR_HELPER = [
+    "spawnfile_mount_points=$(awk '{print $5}' /proc/self/mountinfo)",
+    "spawnfile_is_mount() { printf '%s\\n' \"$spawnfile_mount_points\" | grep -qxF \"$1\"; }",
+    "spawnfile_holds_mount() { printf '%s\\n' \"$spawnfile_mount_points\" | grep -qE \"^$(printf '%s' \"$1\" | sed 's/[][\\.*^$/]/\\\\&/g')(/|$)\"; }",
+    "spawnfile_clear_tree() {",
+    "  spawnfile_root=$1",
+    "  [ -d \"$spawnfile_root\" ] || return 0",
+    "  for spawnfile_entry in \"$spawnfile_root\"/* \"$spawnfile_root\"/.[!.]* \"$spawnfile_root\"/..?*; do",
+    "    if [ ! -e \"$spawnfile_entry\" ] && [ ! -L \"$spawnfile_entry\" ]; then continue; fi",
+    "    if spawnfile_is_mount \"$spawnfile_entry\"; then continue; fi",
+    "    if spawnfile_holds_mount \"$spawnfile_entry\"; then spawnfile_clear_tree \"$spawnfile_entry\"; continue; fi",
+    "    rm -rf \"$spawnfile_entry\" || { echo \"cannot clear $spawnfile_entry: it is in use; a mount below a provisioned root must be declared outside it\" >&2; return 1; }",
+    "  done",
+    "}",
+    // Removes the target outright, unless it is or contains a mount point: then its contents go and the
+    // mount points and the directories holding them stay, because neither can be unlinked while mounted.
+    "spawnfile_remove_tree() {",
+    "  if [ ! -e \"$1\" ] && [ ! -L \"$1\" ]; then return 0; fi",
+    "  if spawnfile_holds_mount \"$1\"; then spawnfile_clear_tree \"$1\"; return $?; fi",
+    "  rm -rf \"$1\" || { echo \"cannot remove $1: it is in use\" >&2; return 1; }",
+    "}"
+];
+export const renderDaimonBrokerProvisioning = (plans) => {
+    const registrations = resolveDaimonGrokRegistrations(plans);
+    if (registrations.length === 0)
+        return [];
+    const workspaceResources = plans
+        .filter((plan) => plan.runtimeName === "daimon")
+        .flatMap((plan) => plan.resources ?? [])
+        .map((resource) => ({
+        backingPath: resource.backingPath,
+        kind: resource.kind,
+        linkPath: resource.linkPath,
+        mode: resource.mode,
+        resolvedIdentity: "resolvedIdentity" in resource && typeof resource.resolvedIdentity === "string"
+            ? resource.resolvedIdentity
+            : null
+    }))
+        .sort((left, right) => left.linkPath.localeCompare(right.linkPath));
+    return renderDaimonBrokerProvisioningProgram(registrations, workspaceResources);
+};
+/**
+ * The root broker provisioning program itself, over already-resolved
+ * registrations and workspace resources.
+ *
+ * Production reaches it through {@link renderDaimonBrokerProvisioning}; the
+ * broker-capable training container reaches it directly for its single fixed
+ * slot, with `serviceOptions` pointing the turn store at per-slot tmpfs and
+ * declaring the evaluator inference ledger. Both callers get byte-identical
+ * credential, registration, worker-home, temp and spill provisioning — the
+ * whole point of sharing it rather than writing a second root program.
+ *
+ * It is re-runnable: the shell wrapper removes `/etc/daimon-engine-broker` and
+ * `/run/daimon-engine-broker` first, so a slot recycle replays exactly the
+ * audited start-up path, credential-journal recovery included.
+ */
+export const renderDaimonBrokerProvisioningProgram = (registrations, workspaceResources, serviceOptions = {}, 
+/**
+ * How the broker's `/etc` and `/run` roots are reset before provisioning.
+ * A production organization owns both as ordinary image directories and
+ * removes them outright. The training container mounts each as its own
+ * tmpfs, which cannot be unlinked, so it empties them instead — the same
+ * end state, reached the only way a mount point allows.
+ */
+rootReset = "remove", 
+/** Deny targets the worker provisioning creates when absent; see `renderDaimonGrokWorkerProvisioning`. */
+optionalDenyPaths, 
+/**
+ * Where the broker and relay get their private `TMPDIR`. Production keeps it
+ * inside the control root, which it removes and recreates exactly once. A
+ * deployment that re-provisions that root — the training slot recycle — must
+ * pass a path outside it: a `tmp/` inside a cleared root is a mount point in
+ * the training launch, and clearing a mount point fails `EBUSY`.
+ */
+brokerTmpdir = DAIMON_BROKER_TMPDIR) => {
+    const program = [
+        "const crypto = require('node:crypto'); const fs = require('node:fs');",
+        `const registrations = ${JSON.stringify(registrations.map(({ agentId, home, slot, uid, workspace }) => ({ agentId, home, slot, uid, workspace })))};`,
+        `const executable = '${DAIMON_GROK_ENGINE_BROKER.grokExecutablePath}';`,
+        "const digest = crypto.createHash('sha256').update(fs.readFileSync(executable)).digest();",
+        `if (!${JSON.stringify([DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, DAIMON_GROK_ENGINE_BROKER.grokCliArtifacts.x64.sha256])}.includes(digest.toString('hex'))) throw new Error('Grok executable is not the manifest-pinned ${DAIMON_GROK_ENGINE_BROKER.grokCliVersion} build');`,
+        "const cString = (buffer, offset, length, value) => { const bytes = Buffer.from(value); if (bytes.length < 1 || bytes.length >= length || bytes.includes(0)) throw new Error('invalid broker registration'); bytes.copy(buffer, offset); };",
+        `const records = registrations.map((entry) => { const record = Buffer.alloc(692); record.writeUInt32LE(${DAIMON_GROK_ENGINE_BROKER.nativeAbiVersion}, 0); record.writeUInt32LE(entry.slot, 4); record.writeUInt32LE(entry.uid, 8); record.writeUInt32LE(entry.uid, 12); cString(record, 16, 129, entry.agentId); cString(record, 145, 256, entry.workspace); cString(record, 401, 256, entry.home); digest.copy(record, 657); return record; });`,
+        "fs.mkdirSync('/etc/daimon-engine-broker', { recursive: true, mode: 0o700 }); fs.chownSync('/etc/daimon-engine-broker', 0, 0); fs.chmodSync('/etc/daimon-engine-broker', 0o700);",
+        "fs.writeFileSync('/etc/daimon-engine-broker/registrations.bin', Buffer.concat(records), { mode: 0o400, flag: 'wx' });",
+        "fs.chownSync('/etc/daimon-engine-broker/registrations.bin', 0, 0); fs.chmodSync('/etc/daimon-engine-broker/registrations.bin', 0o400);",
+        `fs.mkdirSync('${DAIMON_BROKER_REALM}', { recursive: true, mode: 0o700 }); fs.chownSync('${DAIMON_BROKER_REALM}', 0, 0); fs.chmodSync('${DAIMON_BROKER_REALM}', 0o700);`,
+        // The usage ledger directory itself is provisioned unconditionally by
+        // `renderDaimonUsageLedgerProvisioning` (every Daimon organization writes
+        // here, not just Grok ones) before this script's caller reaches the
+        // broker startup this function guards; this script only ever reads the
+        // path below, for the sandbox denylist.
+        `const bootstrap = '/var/lib/spawnfile/daimon/grok-bootstrap-auth', authority = '${DAIMON_BROKER_REALM}/auth.json';`,
+        "const readSecure = (file, owner, label) => { const before = fs.lstatSync(file); if (!before.isFile() || before.isSymbolicLink() || (owner !== undefined && (before.uid !== owner || before.gid !== owner)) || (before.mode & 0o777) !== 0o600 || before.nlink !== 1 || before.size < 2 || before.size > 65536) throw new Error(`unsafe broker credential ${label}`); const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); try { const opened = fs.fstatSync(fd); if (opened.dev !== before.dev || opened.ino !== before.ino) throw new Error(`unsafe broker credential ${label}`); const bytes = Buffer.alloc(opened.size); let offset = 0; while (offset < bytes.length) { const count = fs.readSync(fd, bytes, offset, bytes.length - offset, offset); if (count < 1) throw new Error(`unsafe broker credential ${label}`); offset += count; } return bytes; } finally { fs.closeSync(fd); } };",
+        `const atomicOwned = (target, bytes) => { const temporary = \`${"${target}"}.\${process.pid}.\${crypto.randomUUID()}.tmp\`; try { const output = fs.openSync(temporary, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o600); try { let written = 0; while (written < bytes.length) written += fs.writeSync(output, bytes, written, bytes.length - written, written); fs.fchmodSync(output, 0o600); fs.fchownSync(output, 2100, 2100); fs.fsyncSync(output); } finally { fs.closeSync(output); } fs.renameSync(temporary, target); const directory = fs.openSync(require('node:path').dirname(target), fs.constants.O_RDONLY | fs.constants.O_DIRECTORY); try { fs.fsyncSync(directory); } finally { fs.closeSync(directory); } } catch (error) { try { fs.unlinkSync(temporary); } catch {} throw error; } };`,
+        "let existing; try { existing = fs.lstatSync(authority); } catch (error) { if (error.code !== 'ENOENT') throw error; } const bootstrapBytes = readSecure(bootstrap, undefined, 'bootstrap'); let bootstrapRecord; try { const root = JSON.parse(bootstrapBytes.toString('utf8')), rows = root && typeof root === 'object' && !Array.isArray(root) ? Object.entries(root).filter(([key, value]) => /^https:\\/\\/auth\\.x\\.ai::/.test(key) && value && typeof value === 'object' && !Array.isArray(value)).map(([, value]) => value) : []; if (rows.length !== 1 || typeof rows[0].key !== 'string' || !rows[0].key.trim() || typeof rows[0].refresh_token !== 'string' || !rows[0].refresh_token.trim() || typeof rows[0].expires_at !== 'string' || !Number.isFinite(Date.parse(rows[0].expires_at))) throw new Error(); bootstrapRecord = true; } catch { bootstrapBytes.fill(0); throw new Error('invalid broker credential bootstrap'); } if (!bootstrapRecord) throw new Error('invalid broker credential bootstrap'); const bootstrapDigest = crypto.createHash('sha256').update(bootstrapBytes).digest('hex');",
+        `const journalRoot = '${DAIMON_BROKER_REALM}/.daimon-broker'; let journalRootExists = false; try { const info = fs.lstatSync(journalRoot); if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== 2100 || info.gid !== 2100 || (info.mode & 0o777) !== 0o700) throw new Error('unsafe broker credential journal directory'); fs.chownSync(journalRoot, 0, 0); fs.chmodSync(journalRoot, 0o700); journalRootExists = true; } catch (error) { if (error.code !== 'ENOENT') throw error; }`,
+        `try { const journalPath = '${DAIMON_BROKER_REALM}/.daimon-broker/credential-journal.json'; let journal; try { const raw = readSecure(journalPath, 2100, 'recovery journal'); journal = JSON.parse(raw.toString('utf8')); raw.fill(0); } catch (error) { if (error.code !== 'ENOENT') throw error; } const stale = journal?.version === 'noopolis.daimon.broker-credential-journal.v1' && journal.state === 'stale'; const recover = () => { if (!stale || !Number.isSafeInteger(journal.generation) || journal.generation < 0 || !/^[a-f0-9]{64}$/.test(journal.sourceDigest) || journal.sourceDigest !== journal.promotedDigest || bootstrapDigest === journal.sourceDigest) throw new Error('unsafe broker credential recovery'); atomicOwned(authority, bootstrapBytes); const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } }; if (!existing) { if (stale) recover(); else atomicOwned(authority, bootstrapBytes); } else { const authorityBytes = readSecure(authority, 2100, 'authority'); try { const authorityDigest = crypto.createHash('sha256').update(authorityBytes).digest('hex'); if (stale) { if (authorityDigest !== journal.sourceDigest && authorityDigest !== bootstrapDigest) throw new Error('unsafe broker credential recovery'); if (authorityDigest === bootstrapDigest) { const recovered = Buffer.from(\`${"${JSON.stringify({ version: 'noopolis.daimon.broker-credential-journal.v1', state: 'promoted', generation: journal.generation + 1, sourceDigest: journal.sourceDigest, promotedDigest: bootstrapDigest })}"}\\n\`); try { atomicOwned(journalPath, recovered); } finally { recovered.fill(0); } } else recover(); } } finally { authorityBytes.fill(0); } } } finally { bootstrapBytes.fill(0); }`,
+        "if (journalRootExists) { fs.chownSync(journalRoot, 0, 0); fs.chmodSync(journalRoot, 0o700); fs.chownSync(journalRoot, 2100, 2100); }",
+        ...renderDaimonWorkspaceResourceSecurity(workspaceResources),
+        ...renderDaimonGrokWorkerProvisioning(registrations, serviceOptions, ...(optionalDenyPaths ? [optionalDenyPaths] : [])),
+        `fs.chownSync('${DAIMON_BROKER_REALM}', 0, 0); fs.chmodSync('${DAIMON_BROKER_REALM}', 0o700); fs.chownSync('${DAIMON_BROKER_REALM}', 2100, 2100);`,
+        "fs.chmodSync('/etc/daimon-engine-broker', 0o555);"
+    ].join("\n");
+    return [
+        "if [ -d /etc/daimon-engine-broker ]; then chmod u+rwx /etc/daimon-engine-broker; fi",
+        "if [ -d /run/daimon-engine-broker ]; then chmod u+rwx /run/daimon-engine-broker; fi",
+        ...(rootReset === "remove"
+            ? ["rm -rf /etc/daimon-engine-broker /run/daimon-engine-broker"]
+            : [...MOUNT_AWARE_CLEAR_HELPER, "for broker_root in /etc/daimon-engine-broker /run/daimon-engine-broker; do spawnfile_clear_tree \"$broker_root\"; done"]),
+        `install -d -o root -g ${DAIMON_BROKER_UID} -m 0731 /run/daimon-engine-broker`,
+        `install -d -o ${DAIMON_BROKER_UID} -g ${DAIMON_BROKER_UID} -m 0700 ${brokerTmpdir}`,
+        "node <<'SPAWNFILE_DAIMON_BROKER_PROVISION'",
+        program,
+        "SPAWNFILE_DAIMON_BROKER_PROVISION"
+    ];
+};

package/dist/compiler/containerDaimonGrokWorkerProvisioning.d.ts

@@ -0,0 +1,38 @@
+import { type DaimonGrokRegistration, type DaimonGrokServiceConfigOptions } from "./containerDaimonGrokWorkerRender.js";
+/**
+ * Root provisioning for every brokered Grok worker, as lines of the broker's
+ * node provisioning program (which already defines `crypto`, `fs`, and
+ * `secureWorkspace`).
+ *
+ * Layout per Daimon's `GROK_ENGINE_BROKER.worker.home` (attested by the broker
+ * before every turn): `$GROK_HOME` (`<home>/.grok`) and `$GROK_HOME/sessions`
+ * are `root:<worker> 1771`; `config.toml` (Daimon's renderer bytes for the
+ * declared model x effort), `sandbox.toml`, and the empty `trusted_folders.toml`,
+ * `managed_config.toml` and `requirements.toml` are `root:root 0444`, so the
+ * worker can neither write, rename nor unlink any file that decides a turn —
+ * trust included; `sessions/sandbox-events.jsonl` is `<worker>:<broker> 0640`.
+ *
+ * Every path handed to Daimon (workspace, home, profile, events, and each deny
+ * entry) must be canonical: it exists, is not a symlink, and resolves to itself.
+ * Daimon never resolves these paths, so a symlinked entry would silently mask
+ * the wrong inode. Root chmods only paths it owns at that moment (no
+ * `CAP_FOWNER`): reclaim, mode, then hand over.
+ */
+export declare const renderDaimonGrokWorkerProvisioning: (registrations: readonly DaimonGrokRegistration[], serviceOptions?: DaimonGrokServiceConfigOptions, 
+/**
+ * Deny targets this program creates root-owned `0700` when they are absent,
+ * so every mask always has an inode. The default is the production
+ * container's set; the training container passes its own, because its roots
+ * are tmpfs mounts and `/run/secrets` and the shared state roots do not
+ * exist there at all.
+ */
+optionalDenyPaths?: readonly string[]) => string[];
+/**
+ * Grok 1.0.34 runs every sandbox profile inside bubblewrap, which needs
+ * unprivileged user namespaces. Ubuntu 24.04+ hosts (and Colima's default VM)
+ * ship `kernel.apparmor_restrict_unprivileged_userns=1`, under which bubblewrap
+ * cannot create them even with the pinned seccomp profile — every worker turn
+ * would then fail attestation long after startup. The container sees the host
+ * kernel's sysctls read-only, so this refuses to start with the fix named.
+ */
+export declare const renderDaimonGrokHostPreflight: (procSys?: string) => string[];