spawnfile 0.1.14 → 0.1.17

package/dist/compiler/training/context.js

@@ -0,0 +1,93 @@
+import { createHash } from "node:crypto";
+import { readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { SpawnfileError } from "../../shared/index.js";
+import { getManifestPath } from "../../filesystem/index.js";
+import { buildCompilePlan } from "../buildCompilePlan.js";
+import { stableStringify } from "../helpers.js";
+import { resolveEffectiveModelTarget } from "../modelEnv.js";
+import { TRAINING_CONTEXT_VERSION, trainingContextSchema } from "./contract.js";
+const sha256 = (value) => `sha256:${createHash("sha256").update(value).digest("hex")}`;
+const invalid = (message) => { throw new SpawnfileError("validation_error", message); };
+const inside = (root, file) => {
+    const relative = path.relative(root, file);
+    return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+};
+/** Resolves the complete project, preserving inherited selection and source provenance. */
+export const createTrainingContext = async (inputPath, options) => {
+    const plan = await buildCompilePlan(await realpath(getManifestPath(inputPath)));
+    const agents = plan.nodes.filter((node) => node.kind === "agent");
+    const selected = options.agent === undefined
+        ? agents.length === 1 ? agents[0] : undefined
+        : agents.find((node) => node.id === options.agent);
+    if (!selected)
+        invalid(options.agent === undefined
+            ? "Training requires --agent with an exact node id when the project does not contain exactly one agent"
+            : `No canonical agent matches ${options.agent}`);
+    const agent = selected.value;
+    const manifest = await realpath(plan.root);
+    const root = path.dirname(manifest);
+    const pins = new Map();
+    const pin = async (source, expectedContent) => {
+        const sourcePath = path.resolve(source);
+        if (!inside(root, sourcePath) || !inside(root, await realpath(sourcePath))) {
+            invalid("Training context v1 requires source files inside the canonical project root");
+        }
+        const info = await stat(sourcePath);
+        if (!info.isFile() || info.size > 16 * 1024 * 1024)
+            invalid("Training source must be a regular file no larger than 16 MiB");
+        const bytes = await readFile(sourcePath);
+        if (expectedContent !== undefined && bytes.toString("utf8") !== expectedContent) {
+            invalid("Training source changed during canonical graph resolution");
+        }
+        const next = { sourcePath, destinationPath: path.relative(root, sourcePath).split(path.sep).join("/"), sha256: sha256(bytes) };
+        const previous = pins.get(sourcePath);
+        if (previous && previous.sha256 !== next.sha256)
+            invalid("Training source changed while its context was captured");
+        pins.set(sourcePath, next);
+        return next;
+    };
+    await pin(manifest);
+    // Manifests for inline nodes live at sourcePath; their synthetic node source is not a file.
+    for (const node of plan.nodes) {
+        await pin(node.value.kind === "agent" ? node.value.sourcePath ?? node.value.source : node.value.source);
+        for (const document of node.value.docs)
+            await pin(document.sourcePath, document.content);
+        const skills = node.value.kind === "agent" ? node.value.skills : node.value.shared.skills;
+        for (const skill of skills)
+            await pin(skill.sourcePath, skill.content);
+    }
+    const documents = await Promise.all(agent.docs.map(async (document) => ({
+        ...await pin(document.sourcePath, document.content), role: document.role
+    })));
+    const skills = await Promise.all(agent.skills.map(async (skill) => ({
+        ...await pin(skill.sourcePath, skill.content), name: skill.name, ref: skill.ref, requiresMcp: skill.requiresMcp
+    })));
+    // A manifest can change after graph resolution but before its first pin. Re-resolve
+    // through the compiler owner, then seal all captured bytes against that graph.
+    if (stableStringify(plan) !== stableStringify(await buildCompilePlan(manifest))) {
+        invalid("Training source changed between canonical resolution and provenance capture");
+    }
+    for (const source of [...pins.keys()])
+        await pin(source);
+    const sources = [...pins.values()].sort((left, right) => left.destinationPath < right.destinationPath ? -1 : left.destinationPath > right.destinationPath ? 1 : 0);
+    const declaredPrimary = agent.execution?.model?.primary;
+    const primary = declaredPrimary ? resolveEffectiveModelTarget(declaredPrimary, agent.execution) : undefined;
+    return trainingContextSchema.parse({
+        version: TRAINING_CONTEXT_VERSION,
+        producer: { package: "spawnfile", version: options.packageVersion },
+        project: { root, manifest, sourceDigest: sha256(JSON.stringify(sources.map(({ destinationPath, sha256: hash }) => ({ destinationPath, sha256: hash })))) },
+        agent: {
+            id: selected.id, name: agent.name, source: path.resolve(agent.sourcePath ?? agent.source), runtime: agent.runtime.name,
+            engine: typeof agent.runtime.options.engine === "string" ? agent.runtime.options.engine : null,
+            model: primary ? { provider: primary.provider, name: primary.name, authMethod: primary.auth.method } : null
+        },
+        sources, documents, skills,
+        resources: (agent.workspaceResources ?? []).map((resource) => ({
+            id: resource.id, kind: resource.kind, mount: resource.mount, mode: resource.mode, sharing: resource.sharing,
+            definitionDigest: sha256(stableStringify(resource)),
+            pin: resource.kind === "bundle" ? resource.sha256 : resource.kind === "git" ? resource.ref ?? null : null
+        })),
+        requirements: { nativeCompilation: true, isolatedPreparation: true }
+    });
+};

package/dist/compiler/training/contract.d.ts

@@ -0,0 +1,142 @@
+import { z } from "zod";
+export declare const TRAINING_CONTEXT_VERSION: "spawnfile.training-context.v1";
+export declare const trainingSourceSchema: z.ZodObject<{
+    sourcePath: z.ZodString;
+    destinationPath: z.ZodString;
+    sha256: z.ZodString;
+}, z.core.$strict>;
+/** Trusted compiler metadata only; this is neither a prompt nor permission to launch. */
+export declare const trainingContextSchema: z.ZodObject<{
+    version: z.ZodLiteral<"spawnfile.training-context.v1">;
+    producer: z.ZodObject<{
+        package: z.ZodLiteral<"spawnfile">;
+        version: z.ZodString;
+    }, z.core.$strict>;
+    project: z.ZodObject<{
+        root: z.ZodString;
+        manifest: z.ZodString;
+        sourceDigest: z.ZodString;
+    }, z.core.$strict>;
+    agent: z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+        source: z.ZodString;
+        runtime: z.ZodString;
+        engine: z.ZodNullable<z.ZodString>;
+        model: z.ZodNullable<z.ZodObject<{
+            provider: z.ZodString;
+            name: z.ZodString;
+            authMethod: z.ZodString;
+        }, z.core.$strict>>;
+    }, z.core.$strict>;
+    sources: z.ZodArray<z.ZodObject<{
+        sourcePath: z.ZodString;
+        destinationPath: z.ZodString;
+        sha256: z.ZodString;
+    }, z.core.$strict>>;
+    documents: z.ZodArray<z.ZodObject<{
+        sourcePath: z.ZodString;
+        destinationPath: z.ZodString;
+        sha256: z.ZodString;
+        role: z.ZodString;
+    }, z.core.$strict>>;
+    skills: z.ZodArray<z.ZodObject<{
+        sourcePath: z.ZodString;
+        destinationPath: z.ZodString;
+        sha256: z.ZodString;
+        name: z.ZodString;
+        ref: z.ZodString;
+        requiresMcp: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+    resources: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        kind: z.ZodEnum<{
+            volume: "volume";
+            git: "git";
+            bundle: "bundle";
+        }>;
+        mount: z.ZodString;
+        mode: z.ZodEnum<{
+            readonly: "readonly";
+            mutable: "mutable";
+        }>;
+        sharing: z.ZodEnum<{
+            per_agent: "per_agent";
+            team: "team";
+        }>;
+        definitionDigest: z.ZodString;
+        pin: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    requirements: z.ZodObject<{
+        nativeCompilation: z.ZodLiteral<true>;
+        isolatedPreparation: z.ZodLiteral<true>;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export type TrainingContext = z.infer<typeof trainingContextSchema>;
+export type TrainingSource = z.infer<typeof trainingSourceSchema>;
+export declare const trainingContextJsonSchema: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
+    version: z.ZodLiteral<"spawnfile.training-context.v1">;
+    producer: z.ZodObject<{
+        package: z.ZodLiteral<"spawnfile">;
+        version: z.ZodString;
+    }, z.core.$strict>;
+    project: z.ZodObject<{
+        root: z.ZodString;
+        manifest: z.ZodString;
+        sourceDigest: z.ZodString;
+    }, z.core.$strict>;
+    agent: z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+        source: z.ZodString;
+        runtime: z.ZodString;
+        engine: z.ZodNullable<z.ZodString>;
+        model: z.ZodNullable<z.ZodObject<{
+            provider: z.ZodString;
+            name: z.ZodString;
+            authMethod: z.ZodString;
+        }, z.core.$strict>>;
+    }, z.core.$strict>;
+    sources: z.ZodArray<z.ZodObject<{
+        sourcePath: z.ZodString;
+        destinationPath: z.ZodString;
+        sha256: z.ZodString;
+    }, z.core.$strict>>;
+    documents: z.ZodArray<z.ZodObject<{
+        sourcePath: z.ZodString;
+        destinationPath: z.ZodString;
+        sha256: z.ZodString;
+        role: z.ZodString;
+    }, z.core.$strict>>;
+    skills: z.ZodArray<z.ZodObject<{
+        sourcePath: z.ZodString;
+        destinationPath: z.ZodString;
+        sha256: z.ZodString;
+        name: z.ZodString;
+        ref: z.ZodString;
+        requiresMcp: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+    resources: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        kind: z.ZodEnum<{
+            volume: "volume";
+            git: "git";
+            bundle: "bundle";
+        }>;
+        mount: z.ZodString;
+        mode: z.ZodEnum<{
+            readonly: "readonly";
+            mutable: "mutable";
+        }>;
+        sharing: z.ZodEnum<{
+            per_agent: "per_agent";
+            team: "team";
+        }>;
+        definitionDigest: z.ZodString;
+        pin: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    requirements: z.ZodObject<{
+        nativeCompilation: z.ZodLiteral<true>;
+        isolatedPreparation: z.ZodLiteral<true>;
+    }, z.core.$strict>;
+}, z.core.$strict>>;

package/dist/compiler/training/contract.js

@@ -0,0 +1,34 @@
+import { z } from "zod";
+export const TRAINING_CONTEXT_VERSION = "spawnfile.training-context.v1";
+const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/u);
+const absolutePath = z.string().min(1).regex(/^(?:\/|[A-Za-z]:[\\/])/u);
+const relativePath = z.string().min(1).regex(/^(?!\/)(?!.*(?:^|\/)\.\.(?:\/|$))[^\\]+$/u);
+const text = z.string().min(1);
+export const trainingSourceSchema = z.object({
+    sourcePath: absolutePath,
+    destinationPath: relativePath,
+    sha256: digest
+}).strict();
+/** Trusted compiler metadata only; this is neither a prompt nor permission to launch. */
+export const trainingContextSchema = z.object({
+    version: z.literal(TRAINING_CONTEXT_VERSION),
+    producer: z.object({ package: z.literal("spawnfile"), version: text }).strict(),
+    project: z.object({ root: absolutePath, manifest: absolutePath, sourceDigest: digest }).strict(),
+    agent: z.object({
+        id: text, name: text, source: absolutePath, runtime: text,
+        engine: text.nullable(),
+        model: z.object({ provider: text, name: text, authMethod: text }).strict().nullable()
+    }).strict(),
+    sources: z.array(trainingSourceSchema).min(1).max(10_000),
+    documents: z.array(trainingSourceSchema.extend({ role: text }).strict()).max(128),
+    skills: z.array(trainingSourceSchema.extend({
+        name: text, ref: text, requiresMcp: z.array(text)
+    }).strict()).max(1_000),
+    resources: z.array(z.object({
+        id: text, kind: z.enum(["bundle", "git", "volume"]), mount: text,
+        mode: z.enum(["mutable", "readonly"]), sharing: z.enum(["per_agent", "team"]),
+        definitionDigest: digest, pin: text.nullable()
+    }).strict()).max(1_000),
+    requirements: z.object({ nativeCompilation: z.literal(true), isolatedPreparation: z.literal(true) }).strict()
+}).strict();
+export const trainingContextJsonSchema = z.toJSONSchema(trainingContextSchema);

package/dist/compiler/training/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./contract.js";
+export * from "./context.js";
+export { parseTrainingMappedPreparation } from "./preparation/contract.js";
+export type { TrainingMappedPreparation } from "./preparation/contract.js";

package/dist/compiler/training/index.js

@@ -0,0 +1,3 @@
+export * from "./contract.js";
+export * from "./context.js";
+export { parseTrainingMappedPreparation } from "./preparation/contract.js";

package/dist/compiler/training/preparation/assets/Dockerfile

@@ -0,0 +1,98 @@
+ARG NATIVE_IMAGE
+ARG PYTHON_IMAGE
+FROM ${PYTHON_IMAGE} AS python
+FROM ${NATIVE_IMAGE} AS training
+
+# Both parents must use the same architecture and Debian release.
+COPY --from=python /usr/local /opt/python
+COPY --from=python /usr/lib/*-linux-gnu/libsqlite3.so.0* /opt/python/lib/
+# The bridge intentionally clears ambient environment variables. Register native
+# libraries in the image so Python extensions also load in that clean process.
+RUN echo /opt/python/lib > /etc/ld.so.conf.d/training-python.conf && ldconfig
+
+# The broker-capable container runs on a read-only root, so the fixed Daimon
+# identities are baked here instead of being created by the entrypoint the way
+# the production organization image creates them. It depends on nothing from the
+# build context, so it sits first: it is the least frequently changing layer.
+RUN for fixed_uid in 2000 2100 2200; do \
+      groupadd -K GID_MIN=1 --gid "$fixed_uid" "daimon-$fixed_uid" \
+   && useradd -K UID_MIN=1 --no-create-home --no-log-init --uid "$fixed_uid" --gid "$fixed_uid" \
+        --home-dir /nonexistent --shell /usr/sbin/nologin "daimon-$fixed_uid"; \
+    done
+
+# The sealed train and test datasets are bind-mounted UNDER /run/training/inputs, so this directory is
+# the ancestor every read of them must traverse, and it is the only inode in that chain the image owns.
+# Root-owned with no world bits, on the read-only root: uid 2200 is in neither the owner nor the group
+# class, so it loses search permission; a `unshare --map-root-user` namespace maps only uid 2200, so
+# CAP_DAC_OVERRIDE there cannot override an inode owned by real uid 0; and a fresh bind of the parent
+# re-exposes this same directory rather than the bytes under it. Nothing at runtime can widen it,
+# because the container root is read-only — the slot provisioning asserts the mode and refuses otherwise.
+RUN mkdir -p /run/training/inputs /run/training/output \
+ && chown 0:0 /run/training /run/training/output && chmod 0755 /run/training /run/training/output \
+ && chown 0:2000 /run/training/inputs && chmod 0750 /run/training/inputs
+
+# Layers run from least to most frequently changing: locked dependencies, the
+# DSPy venv, bridge source, pinned executables, then distributions. Staged context
+# modes are already `a+rX`-closed (train/train-broker 0555), so late COPY layers need no RUN.
+COPY claude/package.json claude/package-lock.json /opt/training/claude/
+RUN cd /opt/training/claude && npm ci --omit=dev
+
+WORKDIR /opt/training/paideia
+COPY paideia/package.json paideia/package-lock.json ./
+RUN npm ci --omit=dev --omit=peer --ignore-scripts \
+ && mkdir -p node_modules/@noopolis \
+ && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon node_modules/@noopolis/daimon
+
+WORKDIR /opt/training/spawnfile
+COPY spawnfile/package.json spawnfile/package-lock.json ./
+RUN npm ci --omit=dev --ignore-scripts
+
+WORKDIR /opt/training/compiler
+COPY compiler/package.json compiler/package-lock.json ./
+RUN npm ci --omit=dev --ignore-scripts
+
+COPY bridge/requirements.lock /opt/training/paideia/bridges/dspy/requirements.lock
+RUN /opt/python/bin/python3 -m venv /opt/training/paideia/bridges/dspy/.venv \
+ && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir -r /opt/training/paideia/bridges/dspy/requirements.lock \
+ && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir setuptools==80.9.0
+
+# Exactly `chmod -R a+rX /opt/training`, but only entries that need a change are
+# touched, so unchanged lower-layer files are never copied up into a new layer.
+# This layer also puts the native parent's own pinned Grok and engine broker at the two fixed paths
+# `GROK_ENGINE_BROKER` names, exactly as a generated organization image does (`src/runtime/container.ts`).
+# No second build is introduced: both are installed from the parent's runtime install, and the slot
+# provisioning and the slot supervisor both refuse an executable that is not a manifest-pinned one.
+RUN find /opt/training \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l \( ! -perm -0444 -o \( \( -perm -0100 -o -perm -0010 -o -perm -0001 \) ! -perm -0111 \) \) \) \) -exec chmod a+rX {} + \
+ && mkdir -m 0755 /opt/training/bin /opt/training/integration \
+ && mkdir -p /opt/daimon/bin \
+ && install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/grok /usr/local/bin/grok \
+ && install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/daimon-engine-broker /opt/daimon/bin/daimon-engine-broker \
+ && ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude \
+ && ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia \
+ && ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile \
+ && mkdir -p /opt/training/integration/node_modules/@noopolis \
+ && ln -s /opt/training/paideia /opt/training/integration/node_modules/@noopolis/paideia \
+ && ln -s /opt/training/spawnfile /opt/training/integration/node_modules/spawnfile \
+ && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon \
+ && mkdir -p /opt/training/spawnfile/node_modules/@noopolis \
+ && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/spawnfile/node_modules/@noopolis/daimon
+
+COPY bridge /opt/training/paideia/bridges/dspy
+RUN /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-deps --no-build-isolation -e /opt/training/paideia/bridges/dspy \
+ && find /opt/training/paideia/bridges/dspy \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l \( ! -perm -0444 -o \( \( -perm -0100 -o -perm -0010 -o -perm -0001 \) ! -perm -0111 \) \) \) \) -exec chmod a+rX {} +
+
+# No Grok binary: `spawnfile.training-container.v3` runs the native parent's pinned
+# /usr/local/bin/grok through the broker, so a second copy could only be an unattested build.
+COPY train /opt/training/bin/train
+COPY train-broker /opt/training/bin/train-broker
+COPY compiler/runtimes.yaml compiler/moltnet-releases.json /opt/training/compiler/
+COPY spawnfile/runtimes.yaml spawnfile/moltnet-releases.json /opt/training/spawnfile/
+COPY compiler/dist /opt/training/compiler/dist
+COPY integration /opt/training/integration
+COPY bootstrap /opt/training/bootstrap
+COPY paideia/dist /opt/training/paideia/dist
+COPY spawnfile/dist /opt/training/spawnfile/dist
+ENV PATH=/opt/training/bin:/opt/training/paideia/bridges/dspy/.venv/bin:/opt/spawnfile/runtime-installs/daimon/bin:/usr/local/bin:/usr/bin:/bin
+ENV HOME=/home/training
+WORKDIR /work
+ENTRYPOINT ["/opt/training/bin/train"]