spawnfile 0.1.14 → 0.1.18

package/dist/compiler/training/plan/targets.d.ts

@@ -0,0 +1,11 @@
+import type { TrainingPlan } from "./contract.js";
+import type { LoadedTrainingProject, LoadedTrainingTarget, TrainingAgentNode } from "./types.js";
+export declare const trainingAgents: (project: LoadedTrainingProject) => TrainingAgentNode[];
+export declare const planGrader: (project: LoadedTrainingProject, name: string, reference: {
+    sourcePath: string;
+    declaredBy: string;
+}) => TrainingPlan["graders"][number];
+export declare const resolveTrainingTarget: (project: LoadedTrainingProject, target: LoadedTrainingTarget, selectedId: string) => {
+    target: TrainingPlan["targets"][number];
+    grader: TrainingPlan["graders"][number];
+} | null;

package/dist/compiler/training/plan/targets.js

@@ -0,0 +1,71 @@
+import path from "node:path";
+import { invalidTraining, lexical, resolveTrainingOverride } from "./references.js";
+const inside = (root, file) => {
+    const relative = path.relative(root, file);
+    return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+};
+export const trainingAgents = (project) => project.compilePlan.nodes.filter((node) => node.value.kind === "agent");
+const targetFile = (file, target, agents) => {
+    const context = `Training target ${file.sourcePath} declared by ${target.declaredBy}`;
+    if (file.error)
+        invalidTraining(`${context}: ${file.error}`);
+    if (!file.realPath || !inside(target.directory, file.sourcePath) || !inside(target.realDirectory, file.realPath)) {
+        invalidTraining(`${context} must be inside its declaring manifest directory (no symlink escape)`);
+    }
+    if (!file.regular || !file.sourcePath.endsWith(".md"))
+        invalidTraining(`${context} must be an existing regular .md file`);
+    if (file.nestedManifests.length > 0)
+        invalidTraining(`${context} belongs to a nested Spawnfile: ${file.nestedManifests.join(", ")}`);
+    const consumers = agents.filter((agent) => [...agent.value.docs, ...agent.value.skills]
+        .some((entry) => entry.sourcePath === file.sourcePath)).map((agent) => agent.id).sort(lexical);
+    if (consumers.length === 0)
+        invalidTraining(`${context} is not consumed by any agent document or skill`);
+    return { sourcePath: file.sourcePath, path: file.path, consumers };
+};
+export const planGrader = (project, name, reference) => {
+    const grader = project.graders[reference.sourcePath]
+        ?? invalidTraining(`Training grader ${name} declared by ${reference.declaredBy} was not loaded: ${reference.sourcePath}`);
+    return {
+        name, declaredBy: reference.declaredBy, sourcePath: grader.sourcePath, path: grader.path
+    };
+};
+const datasetFor = (project, target, agent, sourcePath) => {
+    const dataset = project.datasets[sourcePath]
+        ?? invalidTraining(`Training dataset ${sourcePath} declared by ${target.declaredBy} was not loaded`);
+    return {
+        sourcePath: dataset.sourcePath, path: dataset.path,
+        examples: dataset.value.examples.map((example) => ({
+            id: example.id, at: example.at, wake: { ...example.wake },
+            overrides: ["resources", "memory", "networks"].flatMap((section) => (example[section] ?? []).map((entry) => resolveTrainingOverride(project.compilePlan, agent, section, entry.id, `Training dataset ${sourcePath} example ${example.id} declared by ${target.declaredBy} for ${agent.id} ${section}`))).sort((left, right) => lexical(`${left.section}:${left.id}:${left.declaredBy}`, `${right.section}:${right.id}:${right.declaredBy}`))
+        })).sort((left, right) => lexical(left.id, right.id))
+    };
+};
+export const resolveTrainingTarget = (project, target, selectedId) => {
+    const agents = trainingAgents(project);
+    const files = target.files.map((file) => targetFile(file, target, agents)).sort((left, right) => lexical(left.path, right.path));
+    const declaringNode = project.compilePlan.nodes.find((node) => node.id === target.declaredBy)
+        ?? invalidTraining(`Unknown training declaring node ${target.declaredBy}`);
+    const registry = declaringNode.value.training?.graders ?? {};
+    const reference = (Object.hasOwn(registry, target.declaration.grader) ? registry[target.declaration.grader] : undefined)
+        ?? invalidTraining(`Training target declared by ${target.declaredBy} references unknown grader ${target.declaration.grader}`);
+    const grader = planGrader(project, target.declaration.grader, reference);
+    const consumerIds = new Set(files.flatMap((file) => file.consumers));
+    const checkedAgents = target.kind === "team"
+        ? agents.filter((agent) => consumerIds.has(agent.id))
+        : agents.filter((agent) => agent.id === target.declaredBy);
+    const { train, validation, test } = target.declaration.datasets;
+    let selectedDatasets;
+    // Resolve every consumer even when the caller selects a different one.
+    for (const agent of checkedAgents) {
+        const datasets = {
+            train: datasetFor(project, target, agent, train),
+            validation: validation === undefined ? null : datasetFor(project, target, agent, validation),
+            test: datasetFor(project, target, agent, test)
+        };
+        if (agent.id === selectedId)
+            selectedDatasets = datasets;
+    }
+    return selectedDatasets ? {
+        target: { declaredBy: target.declaredBy, grader: target.declaration.grader, files, datasets: selectedDatasets }, grader
+    } : null;
+};

package/dist/compiler/training/plan/types.d.ts

@@ -0,0 +1,36 @@
+import type { TrainingDataset, TrainingTarget } from "../../../manifest/index.js";
+import type { CompilePlan, CompilePlanNode, ResolvedAgentNode } from "../../types.js";
+export interface TrainingFileSource {
+    sourcePath: string;
+    path: string;
+}
+/** Filesystem observations only; ownership decisions belong to the pure resolver. */
+export interface TrainingTargetFile extends TrainingFileSource {
+    realPath: string | null;
+    regular: boolean;
+    nestedManifests: string[];
+    error?: string;
+}
+export interface LoadedTrainingTarget {
+    declaredBy: string;
+    kind: "agent" | "team";
+    directory: string;
+    realDirectory: string;
+    declaration: TrainingTarget;
+    files: TrainingTargetFile[];
+}
+export interface LoadedTrainingDataset extends TrainingFileSource {
+    value: TrainingDataset;
+    sha256: string;
+}
+export interface LoadedTrainingProject {
+    root: string;
+    manifest: string;
+    compilePlan: CompilePlan;
+    targets: LoadedTrainingTarget[];
+    datasets: Record<string, LoadedTrainingDataset>;
+    graders: Record<string, TrainingFileSource>;
+}
+export type TrainingAgentNode = CompilePlanNode & {
+    value: ResolvedAgentNode;
+};

package/dist/compiler/training/plan/types.js

@@ -0,0 +1 @@
+export {};

package/dist/compiler/training/preparation/assets/Dockerfile

@@ -0,0 +1,109 @@
+ARG NATIVE_IMAGE
+ARG PYTHON_IMAGE
+FROM ${PYTHON_IMAGE} AS python
+FROM ${NATIVE_IMAGE} AS training
+
+# Both parents must use the same architecture and Debian release.
+COPY --from=python /usr/local /opt/python
+COPY --from=python /usr/lib/*-linux-gnu/libsqlite3.so.0* /opt/python/lib/
+# The bridge intentionally clears ambient environment variables. Register native
+# libraries in the image so Python extensions also load in that clean process.
+RUN echo /opt/python/lib > /etc/ld.so.conf.d/training-python.conf && ldconfig
+
+# Bookworm's distro library, shared by broker shell tools and direct clocked
+# Daimon children. Resolve the multiarch path at build time; never preload the image.
+RUN . /etc/os-release && test "$VERSION_CODENAME" = bookworm \
+ && apt-get update && apt-get install --yes --no-install-recommends libfaketime=0.9.10-2.1 \
+ && library="$(dpkg-query -L libfaketime | sed -n '\|/faketime/libfaketime\.so\.1$|p')" \
+ && test -f "$library" \
+ && install -d -o root -g root -m 0755 /usr/local/lib/daimon \
+ && ln -s "$library" /usr/local/lib/daimon/libfaketime.so.1 \
+ && chown -h root:root /usr/local/lib/daimon/libfaketime.so.1 \
+ && rm -rf /var/lib/apt/lists/*
+
+# The broker-capable container runs on a read-only root, so the fixed Daimon
+# identities are baked here instead of being created by the entrypoint the way
+# the production organization image creates them. It depends on nothing from the
+# build context, so it sits first: it is the least frequently changing layer.
+RUN for fixed_uid in 2000 2100 2200 2201 2202 2203 2204 2205 2206 2207; do \
+      groupadd -K GID_MIN=1 --gid "$fixed_uid" "daimon-$fixed_uid" \
+   && useradd -K UID_MIN=1 --no-create-home --no-log-init --uid "$fixed_uid" --gid "$fixed_uid" \
+        --home-dir /nonexistent --shell /usr/sbin/nologin "daimon-$fixed_uid"; \
+    done
+
+# The sealed train and test datasets are bind-mounted UNDER /run/training/inputs, so this directory is
+# the ancestor every read of them must traverse, and it is the only inode in that chain the image owns.
+# Root-owned with no world bits, on the read-only root: uid 2200 is in neither the owner nor the group
+# class, so it loses search permission; a `unshare --map-root-user` namespace maps only uid 2200, so
+# CAP_DAC_OVERRIDE there cannot override an inode owned by real uid 0; and a fresh bind of the parent
+# re-exposes this same directory rather than the bytes under it. Nothing at runtime can widen it,
+# because the container root is read-only — the slot provisioning asserts the mode and refuses otherwise.
+RUN mkdir -p /run/training/inputs /run/training/output /run/training/slots \
+ && chown 0:0 /run/training /run/training/output && chmod 0755 /run/training /run/training/output \
+ && chown 0:2000 /run/training/inputs && chmod 0750 /run/training/inputs
+
+# Layers run from least to most frequently changing: locked dependencies, the
+# DSPy venv, bridge source, pinned executables, then distributions. Staged context
+# modes are already `a+rX`-closed (train/train-broker 0555), so late COPY layers need no RUN.
+COPY claude/package.json claude/package-lock.json /opt/training/claude/
+RUN cd /opt/training/claude && npm ci --omit=dev
+
+WORKDIR /opt/training/paideia
+COPY paideia/package.json paideia/package-lock.json ./
+RUN npm ci --omit=dev --omit=peer --ignore-scripts \
+ && mkdir -p node_modules/@noopolis \
+ && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon node_modules/@noopolis/daimon
+
+WORKDIR /opt/training/spawnfile
+COPY spawnfile/package.json spawnfile/package-lock.json ./
+RUN npm ci --omit=dev --ignore-scripts
+
+WORKDIR /opt/training/compiler
+COPY compiler/package.json compiler/package-lock.json ./
+RUN npm ci --omit=dev --ignore-scripts
+
+COPY bridge/requirements.lock /opt/training/paideia/bridges/dspy/requirements.lock
+RUN /opt/python/bin/python3 -m venv /opt/training/paideia/bridges/dspy/.venv \
+ && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir -r /opt/training/paideia/bridges/dspy/requirements.lock \
+ && /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-cache-dir setuptools==80.9.0
+
+# Exactly `chmod -R a+rX /opt/training`, but only entries that need a change are
+# touched, so unchanged lower-layer files are never copied up into a new layer.
+# This layer also puts the native parent's own pinned Grok and engine broker at the two fixed paths
+# `GROK_ENGINE_BROKER` names, exactly as a generated organization image does (`src/runtime/container.ts`).
+# No second build is introduced: both are installed from the parent's runtime install, and the slot
+# provisioning and the slot supervisor both refuse an executable that is not a manifest-pinned one.
+RUN find /opt/training \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l \( ! -perm -0444 -o \( \( -perm -0100 -o -perm -0010 -o -perm -0001 \) ! -perm -0111 \) \) \) \) -exec chmod a+rX {} + \
+ && mkdir -m 0755 /opt/training/bin /opt/training/integration \
+ && mkdir -p /opt/daimon/bin \
+ && install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/grok /usr/local/bin/grok \
+ && install -o root -g root -m 0555 /opt/spawnfile/runtime-installs/daimon/bin/daimon-engine-broker /opt/daimon/bin/daimon-engine-broker \
+ && ln -s /opt/training/claude/node_modules/.bin/claude /opt/training/bin/claude \
+ && ln -s /opt/training/paideia/dist/src/cli/main.js /opt/training/bin/paideia \
+ && ln -s /opt/training/spawnfile/dist/cli/index.js /opt/training/bin/spawnfile \
+ && mkdir -p /opt/training/integration/node_modules/@noopolis \
+ && ln -s /opt/training/paideia /opt/training/integration/node_modules/@noopolis/paideia \
+ && ln -s /opt/training/spawnfile /opt/training/integration/node_modules/spawnfile \
+ && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/integration/node_modules/@noopolis/daimon \
+ && mkdir -p /opt/training/spawnfile/node_modules/@noopolis \
+ && ln -s /opt/spawnfile/runtime-installs/daimon/node_modules/@noopolis/daimon /opt/training/spawnfile/node_modules/@noopolis/daimon
+
+COPY bridge /opt/training/paideia/bridges/dspy
+RUN /opt/training/paideia/bridges/dspy/.venv/bin/pip install --no-deps --no-build-isolation -e /opt/training/paideia/bridges/dspy \
+ && find /opt/training/paideia/bridges/dspy \( \( -type d ! -perm -0555 \) -o \( ! -type d ! -type l \( ! -perm -0444 -o \( \( -perm -0100 -o -perm -0010 -o -perm -0001 \) ! -perm -0111 \) \) \) \) -exec chmod a+rX {} +
+
+# No Grok binary: `spawnfile.training-container.v3` runs the native parent's pinned
+# /usr/local/bin/grok through the broker, so a second copy could only be an unattested build.
+COPY train /opt/training/bin/train
+COPY train-broker /opt/training/bin/train-broker
+COPY compiler/runtimes.yaml compiler/moltnet-releases.json /opt/training/compiler/
+COPY spawnfile/runtimes.yaml spawnfile/moltnet-releases.json /opt/training/spawnfile/
+COPY compiler/dist /opt/training/compiler/dist
+COPY integration /opt/training/integration
+COPY bootstrap /opt/training/bootstrap
+COPY paideia/dist /opt/training/paideia/dist
+COPY spawnfile/dist /opt/training/spawnfile/dist
+ENV PATH=/opt/training/bin:/opt/training/paideia/bridges/dspy/.venv/bin:/opt/spawnfile/runtime-installs/daimon/bin:/usr/local/bin:/usr/bin:/bin
+ENV HOME=/home/training
+WORKDIR /work
+ENTRYPOINT ["/opt/training/bin/train"]