spawnfile 0.1.14 → 0.1.18

package/dist/compiler/training/execution.js

@@ -0,0 +1,160 @@
+import { randomUUID } from "node:crypto";
+import { lstat, mkdir, realpath } from "node:fs/promises";
+import path from "node:path";
+import { z } from "zod";
+import { defaultTrainingOutputRoot } from "./output.js";
+import { SpawnfileError } from "../../shared/index.js";
+import { trainingPlanSchema, trainingPlanOptimizerSchema } from "./plan/contract.js";
+import { trainingPreparationSchema } from "./preparation/contract.js";
+import { readBoundedJson } from "./preparation/inputs.js";
+import { trainingContainerConfigSchema, trainingImageSchema } from "./container/contract.js";
+export const TRAINING_EXECUTION_FILE = "/run/paideia/execution.json";
+export { TRAINING_EXECUTION_CAPABILITY_LABEL, TRAINING_EXECUTION_VERSION } from "./contract.js";
+export const trainingExecutionSchema = z.object({
+    version: z.literal("spawnfile.training-execution.v1"),
+    authoring: trainingPlanSchema,
+    datasetPins: z.array(z.object({ sourcePath: z.string().min(1), sha256: z.string().regex(/^sha256:[a-f0-9]{64}$/u) }).strict()),
+    optimizer: trainingPlanOptimizerSchema,
+    args: z.array(z.string().min(1)),
+    output: z.string().min(1)
+}).strict();
+const invalid = (message) => { throw new SpawnfileError("validation_error", message); };
+const option = (args, flag) => {
+    const index = args.indexOf(flag);
+    return index < 0 ? undefined : args[index + 1];
+};
+const exists = async (file) => {
+    try {
+        await lstat(file);
+        return true;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return false;
+        throw error;
+    }
+};
+/** Resolve local declaration and authoring defaults; no writes, auth reads or execution. */
+export const lowerTrainingExecution = async (options) => {
+    const { project, plan } = options;
+    if (!plan.targets.length || !plan.optimizer)
+        return invalid(`Agent ${plan.agent.id} has no executable training targets`);
+    const agent = project.compilePlan.nodes.find(node => node.id === plan.agent.id);
+    const agentDirectory = path.dirname(agent.value.kind === "agent" ? agent.value.sourcePath ?? agent.value.source : agent.value.source);
+    const candidates = [...new Set([agentDirectory, project.root].map(root => path.join(root, "training/container.json")))];
+    let trainingConfig = options.trainingConfig === undefined ? undefined : path.resolve(options.trainingConfig);
+    if (trainingConfig === undefined)
+        for (const file of candidates)
+            if (await exists(file)) {
+                trainingConfig = file;
+                break;
+            }
+    if (trainingConfig === undefined)
+        return invalid(`Training requires a container declaration at ${candidates.join(" or ")}, or --training-config`);
+    trainingConfig = await realpath(trainingConfig);
+    const raw = await readBoundedJson(trainingConfig);
+    const preparation = raw.version === "spawnfile.training-container.v2" ||
+        (["spawnfile.training-container.v3", "spawnfile.training-container.v4"].includes(String(raw.version)) && raw.image !== undefined);
+    const declaration = preparation ? trainingPreparationSchema.parse(raw) : trainingContainerConfigSchema.parse(raw);
+    const trainingImage = options.trainingImage === undefined ? undefined : trainingImageSchema.parse(options.trainingImage);
+    if (!preparation && trainingImage === undefined) {
+        return invalid("A lowered container launch configuration requires --training-image; use a v2/v4 preparation declaration for a declared image");
+    }
+    const args = [...options.args];
+    if (!args.includes("--experiment-timeout-ms") && !args.includes("--timeout-ms")) {
+        args.push("--experiment-timeout-ms", String(plan.deadlines?.experiment_ms ?? 43_200_000));
+    }
+    if (!args.includes("--trial-timeout-ms") && plan.deadlines?.trial_ms !== undefined) {
+        args.push("--trial-timeout-ms", String(plan.deadlines.trial_ms));
+    }
+    // These are authoring locators. The receipt consumer lowers ALL target datasets into one file per role.
+    for (const role of ["train", "validation", "test"]) {
+        if (option(args, `--${role}`) !== undefined)
+            continue;
+        const sources = [...new Set(plan.targets.flatMap(target => target.datasets[role] ? [target.datasets[role].sourcePath] : []))];
+        if (role === "validation" && sources.length === 0)
+            continue;
+        args.push(`--${role}`, sources[0]);
+    }
+    if (!args.includes("--editable"))
+        for (const file of [...new Set(plan.targets.flatMap(target => target.files.map(file => file.path)))]) {
+            args.push("--editable", file);
+        }
+    if (!args.includes("--optimizer-model"))
+        args.push("--optimizer-model", plan.optimizer.model);
+    if (args.includes("--resume") && option(args, "--out") === undefined)
+        return invalid("--resume requires --out identifying the exact prior experiment");
+    const output = path.resolve(option(args, "--out") ?? path.join(defaultTrainingOutputRoot(project.root, declaration.inputs.map(input => path.resolve(path.dirname(trainingConfig), input.source))), encodeURIComponent(plan.agent.id), `${new Date().toISOString().replace(/[:.]/gu, "-")}-${randomUUID().slice(0, 8)}`));
+    if (!args.includes("--out"))
+        args.push("--out", output);
+    else
+        args[args.indexOf("--out") + 1] = output;
+    const datasetPins = [...new Set(plan.targets.flatMap(target => Object.values(target.datasets)
+            .filter(dataset => dataset !== null).map(dataset => dataset.sourcePath)))].map(sourcePath => {
+        const loaded = project.datasets[sourcePath];
+        if (!loaded)
+            return invalid(`Authored dataset was not captured during plan loading: ${sourcePath}`);
+        return { sourcePath, sha256: loaded.sha256 };
+    });
+    const execution = trainingExecutionSchema.parse({ version: "spawnfile.training-execution.v1", authoring: plan, datasetPins,
+        optimizer: { ...plan.optimizer, model: option(args, "--optimizer-model") }, args, output });
+    return { execution, trainingConfig, trainingImage, declaration };
+};
+/** Allocation is separate so dry-run never creates a directory, and aliases fail before writes. */
+export const allocateTrainingOutputParent = async (output) => {
+    const parent = path.dirname(path.resolve(output));
+    const missing = [];
+    let ancestor = parent;
+    while (!await exists(ancestor)) {
+        missing.unshift(ancestor);
+        ancestor = path.dirname(ancestor);
+    }
+    if (await realpath(ancestor) !== ancestor || !(await lstat(ancestor)).isDirectory())
+        invalid("Training output parent must be a canonical directory");
+    for (const directory of missing) {
+        try {
+            await mkdir(directory, { mode: 0o700 });
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+        }
+        if (await realpath(directory) !== directory || !(await lstat(directory)).isDirectory())
+            invalid("Training output parent must be a canonical directory");
+    }
+};
+/** Preserve identities and all dataset roles while projecting only filesystem addresses. */
+export const mapTrainingExecution = (execution, map) => {
+    const source = (value) => ({ ...value, sourcePath: map(value.sourcePath) });
+    const args = [...execution.args];
+    for (let index = 0; index < args.length; index++) {
+        if (["--train", "--validation", "--test", "--out", "--cost-config"].includes(args[index]))
+            args[++index] = map(args[index]);
+        else if (args[index] === "--resource") {
+            const value = args[++index], split = value.indexOf("=");
+            args[index] = `${value.slice(0, split)}=${map(value.slice(split + 1))}`;
+        }
+    }
+    return trainingExecutionSchema.parse({ ...execution, datasetPins: execution.datasetPins.map(source), args, output: map(execution.output), authoring: {
+            ...execution.authoring,
+            ...(execution.authoring.preparation ? { preparation: { ...execution.authoring.preparation,
+                    ...(execution.authoring.preparation.history ? { history: map(execution.authoring.preparation.history) } : {}) } } : {}),
+            project: { root: map(execution.authoring.project.root), manifest: map(execution.authoring.project.manifest) },
+            graders: execution.authoring.graders.map(source),
+            targets: execution.authoring.targets.map(target => ({ ...target, files: target.files.map(source), datasets: {
+                    train: source(target.datasets.train), test: source(target.datasets.test),
+                    validation: target.datasets.validation === null ? null : source(target.datasets.validation)
+                } }))
+        } });
+};
+/** Resume changes position/presence of --resume, not the experiment's effective inputs. */
+export const trainingExecutionIdentity = (execution) => {
+    const flags = [];
+    for (let index = 0; index < execution.args.length; index++) {
+        const flag = execution.args[index];
+        if (flag === "--resume")
+            continue;
+        flags.push(flag === "--dry-run" ? [flag] : [flag, execution.args[++index]]);
+    }
+    return { ...execution, args: flags.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0) };
+};

package/dist/compiler/training/index.d.ts

@@ -0,0 +1,12 @@
+export * from "./contract.js";
+export * from "./context.js";
+export * from "./plan/index.js";
+export * from "./rewind/index.js";
+export type { ResolvedTraining, ResolvedTrainingGrader } from "./resolution.js";
+export { parseTrainingMappedPreparation } from "./preparation/contract.js";
+export type { TrainingMappedPreparation } from "./preparation/contract.js";
+export { TRAINING_SLOTS_FILE, TRAINING_SLOTS_VERSION, trainingSlotPaths, trainingSlotDescriptors, trainingSlotsDocument, type TrainingSlotsDocument, type TrainingSlotPaths, type TrainingSlotDescriptor } from "./broker/paths.js";
+export * from "./execution.js";
+export * from "./moments.js";
+export * from "./compiledAgent.js";
+export { TRAINING_FAKETIME_LIBRARY_PATH } from "./container/contract.js";

package/dist/compiler/training/index.js

@@ -0,0 +1,10 @@
+export * from "./contract.js";
+export * from "./context.js";
+export * from "./plan/index.js";
+export * from "./rewind/index.js";
+export { parseTrainingMappedPreparation } from "./preparation/contract.js";
+export { TRAINING_SLOTS_FILE, TRAINING_SLOTS_VERSION, trainingSlotPaths, trainingSlotDescriptors, trainingSlotsDocument } from "./broker/paths.js";
+export * from "./execution.js";
+export * from "./moments.js";
+export * from "./compiledAgent.js";
+export { TRAINING_FAKETIME_LIBRARY_PATH } from "./container/contract.js";

package/dist/compiler/training/moments.d.ts

@@ -0,0 +1,80 @@
+import type { LoadedTrainingProject, TrainingPlan } from "./plan/index.js";
+import { type MaterializedMoment } from "./rewind/contract.js";
+export type TrainingDatasetRole = "train" | "validation" | "test";
+type Example = TrainingPlan["targets"][number]["datasets"]["train"]["examples"][number];
+export interface TrainingMomentCase {
+    version: "spawnfile.training-moment-case.v1";
+    projectRoot: string;
+    agent: string;
+    example: string;
+    role: TrainingDatasetRole;
+    dataset: string;
+    at: string;
+    effectiveAt: string;
+    wake: Example["wake"];
+    group: string;
+    lineage: string;
+    momentRoot: string;
+    historyDir?: string;
+    hook: string;
+}
+export interface PreparedTrainingCase {
+    setup?: Record<string, unknown>;
+    context?: Record<string, unknown>;
+    reference?: Record<string, unknown>;
+    outputs?: Record<string, string>;
+    checks?: unknown[];
+    stages?: unknown[];
+}
+export interface TrainingCasePreparation {
+    moment: MaterializedMoment;
+    momentRoot: string;
+    example: Example;
+    role: TrainingDatasetRole;
+    dataset: string;
+    group: string;
+    lineage: string;
+}
+export interface LowerTrainingMomentsOptions {
+    project: LoadedTrainingProject;
+    plan: TrainingPlan;
+    out: string;
+    historyDir?: string;
+    /** Named integration's case authoring only; never credentials or live subject execution. */
+    prepareCase?: (context: TrainingCasePreparation) => Promise<PreparedTrainingCase>;
+}
+export interface TrainingMomentLoweringEntry {
+    id: string;
+    role: TrainingDatasetRole;
+    dataset: string;
+    at: string;
+    wake: Example["wake"];
+    group: string;
+    lineage: string | null;
+    momentRoot: string;
+    status: "materialized" | "refused";
+    moment?: MaterializedMoment;
+    reason?: string;
+}
+/** Rewind each authored example exactly once; repeated target references share one case. */
+export declare const lowerTrainingMoments: (options: LowerTrainingMomentsOptions) => Promise<{
+    datasets: {
+        train: string;
+        validation?: string;
+        test: string;
+    };
+    report: {
+        version: "spawnfile.training-moment-lowering.v1";
+        agent: string;
+        entries: TrainingMomentLoweringEntry[];
+    };
+}>;
+/** Trial hooks call this on fresh paths; historical identity must match the accepted lowering. */
+export declare const materializeTrainingCase: (options: {
+    project: LoadedTrainingProject;
+    plan: TrainingPlan;
+    moment: TrainingMomentCase;
+    out: string;
+    expected?: MaterializedMoment;
+}) => Promise<MaterializedMoment>;
+export {};

package/dist/compiler/training/moments.js

@@ -0,0 +1,118 @@
+import { createHash } from "node:crypto";
+import { lstat, mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { materializeTrainingMoment } from "./rewind/materialize.js";
+import { normalizeInstant } from "./rewind/instant.js";
+import { materializedMomentSchema } from "./rewind/contract.js";
+const roles = ["train", "validation", "test"];
+const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+const save = (file, value) => writeFile(file, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx", mode: 0o600 });
+/** Rewind each authored example exactly once; repeated target references share one case. */
+export const lowerTrainingMoments = async (options) => {
+    const { project, plan } = options;
+    const out = path.resolve(options.out);
+    // A fresh root is intentional: resume must use the persisted lowered dataset, never overwrite it.
+    await mkdir(out, { mode: 0o700 });
+    const entries = [];
+    const records = { train: [], validation: [], test: [] };
+    const seen = new Set(), declared = new Set();
+    const historyDir = options.historyDir ?? plan.preparation?.history;
+    for (const role of roles)
+        for (const target of plan.targets) {
+            const dataset = target.datasets[role];
+            if (!dataset)
+                continue;
+            declared.add(role);
+            for (const example of dataset.examples) {
+                const identity = JSON.stringify([role, dataset.sourcePath, example.id]);
+                if (seen.has(identity))
+                    continue;
+                seen.add(identity);
+                // Group follows the authored example's identity, not its split, preventing split leakage.
+                const group = `moment:${hash([plan.agent.id, example.id])}`;
+                const momentRoot = path.join(out, "moments", role, hash(identity));
+                const entry = { id: example.id, role, dataset: dataset.path, at: example.at,
+                    wake: example.wake, group, lineage: null, momentRoot, status: "refused" };
+                entries.push(entry);
+                try {
+                    const lineage = `moment:${hash([plan.agent.id, normalizeInstant(example.at), example.wake])}`;
+                    entry.lineage = lineage;
+                    // Scope the selector to this dataset; different datasets may use the same example ID.
+                    const scoped = { ...plan, targets: [{ ...target, datasets: { ...target.datasets, [role]: dataset } }] };
+                    const moment = await materializeTrainingMoment({ project, plan: scoped, example: `${role}:${example.id}`, historyDir, out: momentRoot });
+                    entry.moment = moment;
+                    if (moment.status === "refused") {
+                        entry.reason = moment.inputs.flatMap(input => input.refusal ? [input.refusal] : []).join("; ") || "Rewind leak check refused the example";
+                        continue;
+                    }
+                    const prepared = await options.prepareCase?.({ moment, momentRoot, example, role, dataset: dataset.path, group, lineage }) ?? {};
+                    if (!prepared.checks?.length && !prepared.stages?.length)
+                        throw new Error("Moment case hook must supply terminal checks or stages");
+                    const trainingMoment = { version: "spawnfile.training-moment-case.v1", projectRoot: project.root,
+                        agent: plan.agent.id, example: `${role}:${example.id}`, role, dataset: dataset.path, at: moment.at, effectiveAt: moment.effectiveAt,
+                        wake: example.wake, group, lineage, momentRoot, ...(historyDir ? { historyDir } : {}),
+                        hook: plan.preparation?.hook ?? "spawnfile/rewind" };
+                    records[role].push({ ...prepared, id: example.id, group, lineage: [lineage],
+                        input: { message: { network: example.wake.surface, sender: example.wake.from,
+                                target: example.wake.room, text: example.wake.text } },
+                        setup: { ...prepared.setup, trainingMoment }, context: { ...prepared.context, trainingMomentProvenance: moment } });
+                    entry.status = "materialized";
+                }
+                catch (error) {
+                    entry.reason = error instanceof Error ? error.message : String(error);
+                }
+            }
+        }
+    const datasets = {};
+    for (const role of roles)
+        if (declared.has(role)) {
+            datasets[role] = path.join(out, `${role}.paideia.json`);
+            await save(datasets[role], { schema: "paideia.dataset.v1", id: `${plan.agent.id}:${role}`, subject: plan.agent.name,
+                setup: { profile: plan.preparation?.hook ?? "spawnfile/rewind" }, records: records[role] });
+        }
+    const report = { version: "spawnfile.training-moment-lowering.v1", agent: plan.agent.id, entries };
+    await save(path.join(out, "lowering.json"), report);
+    // Keep every refusal on disk, including an all-refused role. Never substitute another split.
+    for (const role of declared)
+        if (records[role].length === 0)
+            throw new Error(`No executable ${role} moments; see ${path.join(out, "lowering.json")}`);
+    for (const role of roles) {
+        const ids = records[role].map(record => record.id);
+        if (new Set(ids).size !== ids.length)
+            throw new Error(`Duplicate executable ${role} example IDs across datasets; use unique IDs`);
+    }
+    return { datasets: datasets, report };
+};
+/** Trial hooks call this on fresh paths; historical identity must match the accepted lowering. */
+export const materializeTrainingCase = async (options) => {
+    const { project, plan, moment } = options;
+    if (project.root !== moment.projectRoot || plan.agent.id !== moment.agent)
+        throw new Error("Trial moment project or agent differs from lowering");
+    const targets = plan.targets.filter(target => target.datasets[moment.role]?.path === moment.dataset);
+    if (!targets.length)
+        throw new Error("Trial moment dataset is not declared");
+    const example = targets[0].datasets[moment.role].examples.find(value => `${moment.role}:${value.id}` === moment.example);
+    if (!example || !isDeepStrictEqual(example.wake, moment.wake) || normalizeInstant(example.at) !== normalizeInstant(moment.at)) {
+        throw new Error("Trial moment wake or instant differs from lowering");
+    }
+    const file = path.join(moment.momentRoot, "moment.json");
+    if (!options.expected) {
+        const stat = await lstat(file);
+        if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 2_000_000)
+            throw new Error("Accepted moment provenance must be a bounded regular file");
+    }
+    const expected = materializedMomentSchema.parse(options.expected ?? JSON.parse(await readFile(file, "utf8")));
+    if (expected.status !== "materialized" || expected.agent !== moment.agent || expected.example.id !== example.id ||
+        expected.example.role !== moment.role || expected.example.dataset !== moment.dataset || expected.at !== moment.at ||
+        expected.effectiveAt !== moment.effectiveAt || !expected.leakCheck.ok)
+        throw new Error("Accepted moment provenance differs from lowering");
+    const result = await materializeTrainingMoment({ project, plan: { ...plan, targets }, example: moment.example,
+        historyDir: moment.historyDir, out: options.out });
+    if (result.status !== "materialized" || result.effectiveAt !== moment.effectiveAt)
+        throw new Error("Trial rewind refused or historical start instant changed");
+    if (!isDeepStrictEqual(result.inputs, expected.inputs) || !isDeepStrictEqual(result.record, expected.record)) {
+        throw new Error("Trial rewind provenance differs from accepted lowering");
+    }
+    return result;
+};

package/dist/compiler/training/output.d.ts

@@ -0,0 +1,2 @@
+/** Durable default namespace disjoint from every declared input, even a project ancestor. */
+export declare const defaultTrainingOutputRoot: (projectRoot: string, inputs?: readonly string[]) => string;

package/dist/compiler/training/output.js

@@ -0,0 +1,20 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { within } from "./preparation/files.js";
+/** Durable default namespace disjoint from every declared input, even a project ancestor. */
+export const defaultTrainingOutputRoot = (projectRoot, inputs = []) => {
+    const sources = [projectRoot, ...inputs].map(source => path.resolve(source));
+    if (sources.includes(path.parse(projectRoot).root))
+        throw Error("No training output location exists outside declared inputs; supply --out");
+    const identity = createHash("sha256").update(projectRoot).digest("hex").slice(0, 16);
+    let ancestor = path.dirname(projectRoot);
+    for (;;) {
+        const candidate = path.join(ancestor, ".spawn-training", `${path.basename(projectRoot)}-${identity}`);
+        if (!sources.some(source => within(source, candidate) || within(candidate, source)))
+            return candidate;
+        const parent = path.dirname(ancestor);
+        if (parent === ancestor)
+            throw Error("No training output location exists outside declared inputs; supply --out");
+        ancestor = parent;
+    }
+};