spawnfile 0.1.14 → 0.1.18

package/dist/compiler/training/repair/witness.d.ts

@@ -0,0 +1,352 @@
+import type { TrainingContext } from "../contract.js";
+import { type TrainingExecution } from "../execution.js";
+import type { TrainingPreparationConfig } from "../preparation/contract.js";
+import type { TrainingImagePlan } from "../preparation/image.js";
+import type { PlannedInput } from "../preparation/inputs.js";
+export declare function readTrainingWitness(file: string): Promise<{
+    path: string;
+    manifest: {
+        schema: "spawnfile.training-legacy-witness.v1" | "spawnfile.training-witness.v1";
+        parentImage: string;
+        parentPreparationDigest: string;
+        imagePlanDigest: string;
+        config: {
+            version: "spawnfile.training-container.v3" | "spawnfile.training-container.v4" | "spawnfile.training-container.v2";
+            slots: number;
+            dockerContext: string;
+            image: {
+                ref: string;
+            } | {
+                build: {
+                    recipe: "daimon-dspy.v1";
+                    nativeImage: string;
+                    pythonImage: string;
+                    platform: "linux/amd64" | "linux/arm64";
+                    paideia: string;
+                    bridge: string;
+                    claude: string;
+                    integration: {
+                        source: string;
+                        entry: string;
+                        execution?: "spawnfile.training-execution.v1" | undefined;
+                    };
+                    compiler?: string | undefined;
+                    grok?: {
+                        source: string;
+                        sha256: string;
+                    } | undefined;
+                    bootstrap?: string | undefined;
+                };
+            };
+            integration: {
+                settings: {
+                    input: string;
+                    path: string;
+                };
+            };
+            inputs: {
+                id: string;
+                source: string;
+                destination: string;
+                include?: string[] | undefined;
+                git?: {
+                    revision: string;
+                    overlays: {
+                        source: string;
+                        path: string;
+                        sha256: string;
+                    }[];
+                } | undefined;
+            }[];
+            output: {
+                source: string;
+                destination: "/run/training/output";
+            };
+            auth: {
+                source: string;
+                provider: "codex" | "claude";
+            }[];
+            evaluatorRoutes?: {
+                purpose: "decisions" | "extraction" | "proposals";
+                provider: "codex-subscription" | "typesafe";
+                model: string;
+                maxRequests: number;
+                maxSpendUsd?: number | undefined;
+            }[] | undefined;
+            evaluatorSecrets?: {
+                provider: "typesafe";
+                source: string;
+            }[] | undefined;
+            subjectIsolation?: "codex-native-sandbox" | undefined;
+            broker?: {
+                maxRunRequests: number;
+                engine: "grok";
+                agentId: string;
+                model: "grok-4.6" | "grok-4.5" | "grok-build";
+                reasoningEffort: "high" | "medium" | "low";
+                architecture: "arm64" | "x64";
+                limits: {
+                    maxRequests: number;
+                    maxTokens: number;
+                    timeoutMs: number;
+                };
+                realmVolume: string;
+                bootstrap: string;
+                unenforcedBindPolicy?: "refuse" | "profile-only" | undefined;
+            } | undefined;
+        };
+        context: {
+            version: "spawnfile.training-context.v1";
+            producer: {
+                package: "spawnfile";
+                version: string;
+            };
+            project: {
+                root: string;
+                manifest: string;
+                sourceDigest: string;
+            };
+            agent: {
+                id: string;
+                name: string;
+                source: string;
+                runtime: string;
+                engine: string | null;
+                model: {
+                    provider: string;
+                    name: string;
+                    authMethod: string;
+                } | null;
+            };
+            sources: {
+                sourcePath: string;
+                destinationPath: string;
+                sha256: string;
+            }[];
+            documents: {
+                sourcePath: string;
+                destinationPath: string;
+                sha256: string;
+                role: string;
+            }[];
+            skills: {
+                sourcePath: string;
+                destinationPath: string;
+                sha256: string;
+                name: string;
+                ref: string;
+                requiresMcp: string[];
+            }[];
+            resources: {
+                id: string;
+                kind: "volume" | "git" | "bundle";
+                mount: string;
+                mode: "readonly" | "mutable";
+                sharing: "per_agent" | "team";
+                definitionDigest: string;
+                pin: string | null;
+            }[];
+            requirements: {
+                nativeCompilation: true;
+                isolatedPreparation: true;
+            };
+        };
+        command: {
+            args: string[];
+            env?: Record<string, string> | undefined;
+        };
+        inputs: {
+            id: string;
+            source: string;
+            destination: string;
+            digest: string;
+            staged: string;
+            snapshotDigest: string | null;
+        }[];
+        image: {
+            build: {
+                recipe: "daimon-dspy.v1";
+                nativeImage: string;
+                pythonImage: string;
+                platform: "linux/amd64" | "linux/arm64";
+                paideia: string;
+                bridge: string;
+                claude: string;
+                integration: {
+                    source: string;
+                    entry: string;
+                    execution?: "spawnfile.training-execution.v1" | undefined;
+                };
+                compiler?: string | undefined;
+                grok?: {
+                    source: string;
+                    sha256: string;
+                } | undefined;
+                bootstrap?: string | undefined;
+            };
+            dockerfile: string;
+            entry: string;
+            brokerEntry: string;
+            files: {
+                destination: string;
+                sha256: string;
+                mode: number;
+                size: number;
+            }[];
+        };
+        execution?: {
+            version: "spawnfile.training-execution.v1";
+            authoring: {
+                version: "spawnfile.training-plan.v1";
+                producer: {
+                    package: "spawnfile";
+                    version: string;
+                };
+                project: {
+                    root: string;
+                    manifest: string;
+                };
+                agent: {
+                    id: string;
+                    name: string;
+                };
+                record: {
+                    keep: string | null;
+                    volumes: {
+                        id: string;
+                        declaredBy: string;
+                    }[];
+                } | null;
+                optimizer: {
+                    method: string;
+                    model: string;
+                    variants: number;
+                    keep: number;
+                    budget: {
+                        rounds: number;
+                    };
+                } | null;
+                graders: {
+                    sourcePath: string;
+                    path: string;
+                    name: string;
+                    declaredBy: string;
+                }[];
+                targets: {
+                    declaredBy: string;
+                    grader: string;
+                    files: {
+                        sourcePath: string;
+                        path: string;
+                        consumers: string[];
+                    }[];
+                    datasets: {
+                        train: {
+                            sourcePath: string;
+                            path: string;
+                            examples: {
+                                id: string;
+                                at: string;
+                                wake: {
+                                    surface: "moltnet";
+                                    room: string;
+                                    from: string;
+                                    text: string;
+                                };
+                                overrides: {
+                                    id: string;
+                                    declaredBy: string;
+                                    section: "memory" | "resources" | "networks";
+                                }[];
+                            }[];
+                        };
+                        validation: {
+                            sourcePath: string;
+                            path: string;
+                            examples: {
+                                id: string;
+                                at: string;
+                                wake: {
+                                    surface: "moltnet";
+                                    room: string;
+                                    from: string;
+                                    text: string;
+                                };
+                                overrides: {
+                                    id: string;
+                                    declaredBy: string;
+                                    section: "memory" | "resources" | "networks";
+                                }[];
+                            }[];
+                        } | null;
+                        test: {
+                            sourcePath: string;
+                            path: string;
+                            examples: {
+                                id: string;
+                                at: string;
+                                wake: {
+                                    surface: "moltnet";
+                                    room: string;
+                                    from: string;
+                                    text: string;
+                                };
+                                overrides: {
+                                    id: string;
+                                    declaredBy: string;
+                                    section: "memory" | "resources" | "networks";
+                                }[];
+                            }[];
+                        };
+                    };
+                }[];
+                deadlines?: {
+                    experiment_ms: number;
+                    trial_ms?: number | undefined;
+                } | undefined;
+                preparation?: {
+                    hook: string;
+                    history?: string | undefined;
+                } | undefined;
+            };
+            datasetPins: {
+                sourcePath: string;
+                sha256: string;
+            }[];
+            optimizer: {
+                method: string;
+                model: string;
+                variants: number;
+                keep: number;
+                budget: {
+                    rounds: number;
+                };
+            };
+            args: string[];
+            output: string;
+        } | undefined;
+        repair?: {
+            witness: string;
+            parent: string;
+        } | undefined;
+    };
+    digest: string;
+}>;
+/** Future repairs need sealed original bytes, not a mutable checkout or caller assertion. */
+export declare function writeTrainingWitness(options: {
+    directory: string;
+    digest: string;
+    image: string;
+    config: TrainingPreparationConfig;
+    context: TrainingContext;
+    args: readonly string[];
+    execution?: TrainingExecution;
+    inputs: PlannedInput[];
+    staged: string[];
+    snapshots: (string | null)[];
+    plan: TrainingImagePlan;
+    repair?: {
+        witness: string;
+        parent: string;
+    };
+}): Promise<void>;

package/dist/compiler/training/repair/witness.js

@@ -0,0 +1,48 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { trainingExecutionIdentity } from "../execution.js";
+import { copySealed, exactPath, fileIdentity, hashJson, sealTree } from "../preparation/files.js";
+import { witnessEnvelopeSchema } from "./contract.js";
+export async function readTrainingWitness(file) {
+    const canonical = await exactPath(file), bytes = await readFile(canonical, "utf8");
+    if (Buffer.byteLength(bytes) > 8 * 1024 * 1024)
+        throw Error("Training witness exceeds 8 MiB");
+    const raw = JSON.parse(bytes);
+    if (hashJson(raw.manifest) !== raw.digest)
+        throw Error("Training witness digest mismatch");
+    const parsed = witnessEnvelopeSchema.parse(raw), witness = parsed.manifest;
+    const files = await sealTree(path.join(path.dirname(canonical), "image-files"), "");
+    const sorted = (value) => [...value].sort((a, b) => a.destination.localeCompare(b.destination));
+    if (hashJson(sorted(fileIdentity(files))) !== hashJson(sorted(witness.image.files)))
+        throw Error("Training witness image bytes changed");
+    const { recipe, nativeImage, pythonImage, platform } = witness.image.build;
+    if (hashJson({ recipe, nativeImage, pythonImage, platform, files: witness.image.files,
+        dockerfile: witness.image.dockerfile, entry: witness.image.entry, brokerEntry: witness.image.brokerEntry }) !== witness.imagePlanDigest)
+        throw Error("Training witness recipe identity mismatch");
+    // Validate with the schemas above, but hash the original property order used by preparation.
+    const manifest = raw.manifest;
+    const expected = hashJson({ config: manifest.config,
+        sources: witness.inputs.map(input => ({ id: input.id, digest: input.digest })),
+        image: witness.imagePlanDigest, canonical: witness.context.project.sourceDigest,
+        ...(manifest.execution ? { execution: trainingExecutionIdentity(manifest.execution) } : {}),
+        ...(witness.repair ? { repair: witness.repair } : {}) });
+    if (expected !== witness.parentPreparationDigest)
+        throw Error("Training witness preparation identity mismatch");
+    return { ...parsed, path: canonical };
+}
+/** Future repairs need sealed original bytes, not a mutable checkout or caller assertion. */
+export async function writeTrainingWitness(options) {
+    const directory = path.join(options.directory, "witness");
+    await mkdir(directory, { mode: 0o700 });
+    await copySealed(options.plan.files, path.join(directory, "image-files"));
+    const manifest = { schema: "spawnfile.training-witness.v1", parentImage: options.image,
+        parentPreparationDigest: options.digest, imagePlanDigest: options.plan.digest, config: options.config,
+        ...(options.execution ? { execution: options.execution } : {}),
+        ...(options.repair ? { repair: options.repair } : {}),
+        context: options.context, command: { args: [...options.args] },
+        inputs: options.inputs.map((input, index) => ({ id: input.id, source: input.source, destination: input.destination,
+            digest: input.digest, staged: options.staged[index], snapshotDigest: options.snapshots[index] })),
+        image: { build: options.plan.build, files: fileIdentity(options.plan.files), dockerfile: options.plan.dockerfile,
+            entry: options.plan.entry, brokerEntry: options.plan.brokerEntry } };
+    await writeFile(path.join(directory, "manifest.json"), JSON.stringify({ manifest, digest: hashJson(manifest) }), { flag: "wx", mode: 0o400 });
+}

package/dist/compiler/training/resolution.d.ts

@@ -0,0 +1,25 @@
+import type { TrainingDeclaration, TrainingTarget } from "../../manifest/index.js";
+import type { CompilePlanNode } from "../types.js";
+export interface ResolvedTrainingGrader {
+    sourcePath: string;
+    /** A source key during traversal; a stable node ID in the finished compile plan. */
+    declaredBy: string;
+}
+export interface ResolvedTraining {
+    deadlines?: TrainingDeclaration["deadlines"];
+    preparation?: {
+        hook: string;
+        history?: string;
+    };
+    record?: TrainingDeclaration["record"];
+    optimizer?: TrainingDeclaration["optimizer"];
+    graders: Record<string, ResolvedTrainingGrader>;
+    /** Only this node's targets, with absolute file and dataset paths. */
+    targets: TrainingTarget[];
+}
+/** Resolve declaration-relative paths before any inheritance or shadowing. */
+export declare const resolveTrainingDeclaration: (declaration: TrainingDeclaration | undefined, manifestPath: string, declaredBy: string) => ResolvedTraining | undefined;
+/** Direct team membership only: field merges, local arrays, local targets. */
+export declare const mergeTrainingDeclarations: (inherited: ResolvedTraining | undefined, local: ResolvedTraining | undefined) => ResolvedTraining | undefined;
+/** Provenance uses the same stable IDs as every finished graph consumer. */
+export declare const resolvePlanTrainingSources: (nodes: CompilePlanNode[]) => void;

package/dist/compiler/training/resolution.js

@@ -0,0 +1,69 @@
+import path from "node:path";
+import { SpawnfileError } from "../../shared/index.js";
+/** Resolve declaration-relative paths before any inheritance or shadowing. */
+export const resolveTrainingDeclaration = (declaration, manifestPath, declaredBy) => {
+    if (declaration === undefined)
+        return undefined;
+    const resolve = (value) => path.resolve(path.dirname(manifestPath), value);
+    return {
+        ...(declaration.deadlines === undefined ? {} : { deadlines: { ...declaration.deadlines } }),
+        ...(declaration.preparation === undefined ? {} : { preparation: { ...declaration.preparation,
+                ...(declaration.preparation.history === undefined ? {} : { history: resolve(declaration.preparation.history) }) } }),
+        ...(declaration.record === undefined ? {} : { record: { ...declaration.record,
+                ...(declaration.record.volumes === undefined ? {} : { volumes: [...declaration.record.volumes] }) } }),
+        ...(declaration.optimizer === undefined ? {} : { optimizer: { ...declaration.optimizer,
+                ...(declaration.optimizer.budget === undefined ? {} : { budget: { ...declaration.optimizer.budget } }) } }),
+        graders: Object.fromEntries(Object.entries(declaration.graders ?? {})
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([name, value]) => [name, { sourcePath: resolve(value), declaredBy }])),
+        targets: (declaration.targets ?? []).map((target) => ({
+            files: target.files.map(resolve),
+            grader: target.grader,
+            datasets: {
+                train: resolve(target.datasets.train),
+                ...(target.datasets.validation === undefined ? {} : { validation: resolve(target.datasets.validation) }),
+                test: resolve(target.datasets.test)
+            }
+        }))
+    };
+};
+/** Direct team membership only: field merges, local arrays, local targets. */
+export const mergeTrainingDeclarations = (inherited, local) => {
+    const deadlines = inherited?.deadlines === undefined && local?.deadlines === undefined
+        ? undefined : { ...inherited?.deadlines, ...local?.deadlines };
+    const record = inherited?.record === undefined && local?.record === undefined
+        ? undefined : { ...inherited?.record, ...local?.record };
+    const budget = inherited?.optimizer?.budget === undefined && local?.optimizer?.budget === undefined
+        ? undefined : { ...inherited?.optimizer?.budget, ...local?.optimizer?.budget };
+    const optimizer = inherited?.optimizer === undefined && local?.optimizer === undefined
+        ? undefined : { ...inherited?.optimizer, ...local?.optimizer, ...(budget === undefined ? {} : { budget }) };
+    const graders = { ...inherited?.graders, ...local?.graders };
+    const targets = local?.targets ?? [];
+    const preparation = local?.preparation ?? inherited?.preparation;
+    if (deadlines === undefined && preparation === undefined && record === undefined && optimizer === undefined && Object.keys(graders).length === 0 && targets.length === 0) {
+        return undefined;
+    }
+    return {
+        ...(deadlines === undefined ? {} : { deadlines }),
+        ...(preparation === undefined ? {} : { preparation }),
+        ...(record === undefined ? {} : { record }),
+        ...(optimizer === undefined ? {} : { optimizer }),
+        graders: Object.fromEntries(Object.entries(graders).sort(([left], [right]) => left.localeCompare(right))),
+        targets
+    };
+};
+/** Provenance uses the same stable IDs as every finished graph consumer. */
+export const resolvePlanTrainingSources = (nodes) => {
+    const idBySource = new Map(nodes.map((node) => [node.value.source, node.id]));
+    for (const node of nodes) {
+        if (!node.value.training)
+            continue;
+        node.value.training.graders = Object.fromEntries(Object.entries(node.value.training.graders).map(([name, grader]) => {
+            const declaredBy = idBySource.get(grader.declaredBy);
+            if (declaredBy === undefined) {
+                throw new SpawnfileError("compile_error", `Training grader ${name} has unknown declaring source ${grader.declaredBy}`);
+            }
+            return [name, { ...grader, declaredBy }];
+        }));
+    }
+};