stageflow 0.25.0 → 0.30.0

package/dist/runstore/runManifest.js

@@ -0,0 +1,288 @@
+import { createHash } from "node:crypto";
+import { PACKAGE_VERSION, BUILD_SHA } from "../package-meta.js";
+import { redact, } from "../logging/redact.js";
+import { getNamedSecrets } from "../logging/namedSecrets.js";
+import { CURRENT_SCHEMA_VERSION } from "./sqlite/migrations/index.js";
+export const RUN_MANIFEST_VERSION = 1;
+export function sha256Hex(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}
+export function sha256Digest(bytes) {
+    return `sha256:${sha256Hex(bytes)}`;
+}
+export function digestSkillFiles(files) {
+    const names = Object.keys(files).sort();
+    const hash = createHash("sha256");
+    for (const name of names) {
+        hash.update(name);
+        hash.update("\0");
+        hash.update(files[name] ?? "");
+        hash.update("\0");
+    }
+    return { digest: `sha256:${hash.digest("hex")}`, files: names };
+}
+export function skillsFromPayload(payload) {
+    if (payload === undefined)
+        return [];
+    const out = [];
+    for (const [name, files] of Object.entries(payload)) {
+        const dig = digestSkillFiles(files);
+        out.push({
+            name,
+            origin: "run",
+            digest: dig.digest,
+            files: dig.files,
+        });
+    }
+    return out.sort((a, b) => a.name.localeCompare(b.name));
+}
+export function toolchainFromChecks(checks) {
+    return checks.map((c) => ({
+        tool: c.tool,
+        ...(c.required !== undefined ? { required: c.required } : {}),
+        ...(c.found !== undefined ? { resolved: c.found } : {}),
+        ...(c.path !== undefined ? { path: c.path } : {}),
+        origin: c.origin,
+        status: c.status,
+    }));
+}
+export function bindingFromFields(fields) {
+    if (fields.repository != null && fields.repository !== "") {
+        return {
+            kind: "repository",
+            repository: fields.repository,
+            ...(fields.ref !== undefined ? { ref: fields.ref } : {}),
+            ...(fields.resolvedSha !== undefined
+                ? { resolved_sha: fields.resolvedSha }
+                : {}),
+            ...(fields.runBranch !== undefined ? { branch: fields.runBranch } : {}),
+            ...(fields.checkoutRoot !== undefined
+                ? { worktree_path: fields.checkoutRoot }
+                : {}),
+        };
+    }
+    if (fields.checkoutRoot != null && fields.checkoutRoot !== "") {
+        return {
+            kind: "checkout",
+            ...(fields.resolvedSha !== undefined
+                ? { resolved_sha: fields.resolvedSha }
+                : {}),
+            worktree_path: fields.checkoutRoot,
+        };
+    }
+    return { kind: "unbound" };
+}
+export function stagesFromLoaded(stages) {
+    return stages.map((stage) => ({
+        stage_id: stage.id,
+        model: {
+            authored: stage.model,
+            authored_tier: stage.model_tier ?? "stage",
+        },
+        mcp_servers: [],
+    }));
+}
+export function buildHostIdentity(env = process.env) {
+    const digest = env.STAGEFLOW_IMAGE_DIGEST?.trim();
+    return {
+        stageflow_version: PACKAGE_VERSION,
+        build_sha: BUILD_SHA,
+        image_digest: digest && digest.length > 0 ? digest : null,
+        schema_version: CURRENT_SCHEMA_VERSION,
+    };
+}
+export function buildInitialRunManifest(input) {
+    const pipelineBody = input.pipelineSource === "inline" && input.pipelineBody
+        ? (() => {
+            try {
+                return JSON.parse(input.pipelineBody);
+            }
+            catch {
+                return input.pipelineBody;
+            }
+        })()
+        : null;
+    const pipelineBytes = input.pipelineSource === "inline" && input.pipelineBody
+        ? input.pipelineBody
+        : input.pipelinePath ?? "";
+    const taskSource = input.taskPath !== undefined && input.taskPath.length > 0
+        ? "path"
+        : "inline";
+    const raw = {
+        manifest_version: RUN_MANIFEST_VERSION,
+        run_id: input.runId,
+        created_at: input.createdAt,
+        host: buildHostIdentity(),
+        caller: {
+            caller_id: input.callerId,
+            surface: input.surface,
+        },
+        binding: input.binding,
+        pipeline: {
+            source: input.pipelineSource,
+            path: input.pipelinePath ?? null,
+            bytes_sha256: sha256Hex(pipelineBytes),
+            body: pipelineBody,
+        },
+        task: {
+            source: taskSource,
+            path: input.taskPath ?? null,
+            bytes_sha256: sha256Hex(input.taskYaml),
+            body: input.taskYaml,
+        },
+        skills: skillsFromPayload(input.skills),
+        stages: stagesFromLoaded(input.stages),
+        toolchain: toolchainFromChecks(input.toolchain),
+    };
+    return redactRunManifest(raw, {
+        namedSecrets: input.namedSecrets ?? getNamedSecrets(),
+    });
+}
+export function parseRunManifest(value) {
+    if (value === null || value === undefined)
+        return null;
+    if (typeof value !== "object" || Array.isArray(value))
+        return null;
+    const version = value.manifest_version;
+    if (version === 1) {
+        return value;
+    }
+    return null;
+}
+export function collectMcpEnvNamedSecrets(servers) {
+    if (servers === undefined)
+        return [];
+    const out = [];
+    for (const [serverName, cfg] of Object.entries(servers)) {
+        const env = cfg.env;
+        if (env === null || typeof env !== "object" || Array.isArray(env))
+            continue;
+        for (const [key, value] of Object.entries(env)) {
+            if (typeof value === "string" && value.length > 0) {
+                out.push({ name: `${serverName}.${key}`, value });
+            }
+        }
+    }
+    return out;
+}
+export function collectDeclaredSecretValues(names, env = process.env) {
+    const out = [];
+    for (const name of names) {
+        const plain = env[name];
+        if (typeof plain === "string" && plain.length > 0) {
+            out.push({ name, value: plain });
+        }
+    }
+    return out;
+}
+export function redactMcpServers(servers, namedSecrets, origins) {
+    const out = [];
+    for (const [name, cfg] of Object.entries(servers)) {
+        const redacted = redact({ ...cfg }, { namedSecrets });
+        out.push({
+            name,
+            ...redacted,
+            ...(origins?.get(name) !== undefined
+                ? { origin: origins.get(name) }
+                : {}),
+        });
+    }
+    return out;
+}
+export function redactRunManifest(manifest, options = {}) {
+    const namedSecrets = [
+        ...(options.namedSecrets ?? getNamedSecrets()),
+    ];
+    return redact({ ...manifest }, {
+        ...options,
+        namedSecrets,
+    });
+}
+/** Defense-in-depth redaction for get_run / export views (pipeline body included). */
+export function redactRunManifestForRead(value, options = {}) {
+    const parsed = parseRunManifest(value);
+    if (parsed === null)
+        return null;
+    if (parsed.manifest_version === 1) {
+        return redactRunManifest(parsed, options);
+    }
+    return null;
+}
+export function withStageResolvedModel(manifest, stageId, resolved) {
+    const stages = manifest.stages.map((stage) => {
+        if (stage.stage_id !== stageId)
+            return stage;
+        return {
+            ...stage,
+            model: {
+                ...stage.model,
+                resolved: resolved.model,
+                ...(resolved.thinkingLevel !== undefined
+                    ? { thinking_level: resolved.thinkingLevel }
+                    : {}),
+            },
+        };
+    });
+    const has = stages.some((s) => s.stage_id === stageId);
+    if (!has) {
+        stages.push({
+            stage_id: stageId,
+            model: {
+                authored: resolved.model,
+                authored_tier: "stage",
+                resolved: resolved.model,
+                ...(resolved.thinkingLevel !== undefined
+                    ? { thinking_level: resolved.thinkingLevel }
+                    : {}),
+            },
+            mcp_servers: [],
+        });
+    }
+    return { ...manifest, stages };
+}
+export function withStageMcpServers(manifest, stageId, mcpServers) {
+    const stages = manifest.stages.map((stage) => {
+        if (stage.stage_id !== stageId)
+            return stage;
+        return { ...stage, mcp_servers: mcpServers };
+    });
+    const has = stages.some((s) => s.stage_id === stageId);
+    if (!has) {
+        stages.push({
+            stage_id: stageId,
+            model: { authored: "", authored_tier: "stage" },
+            mcp_servers: mcpServers,
+        });
+    }
+    return { ...manifest, stages };
+}
+export function withSkillEntry(manifest, skill) {
+    const skills = [...manifest.skills];
+    const idx = skills.findIndex((s) => s.name === skill.name);
+    if (idx >= 0)
+        skills[idx] = skill;
+    else
+        skills.push(skill);
+    skills.sort((a, b) => a.name.localeCompare(b.name));
+    return { ...manifest, skills };
+}
+export function finaliseRunManifest(manifest, finalisedAt = new Date().toISOString()) {
+    return { ...manifest, finalised_at: finalisedAt };
+}
+export async function readParsedRunManifest(store, runId) {
+    const meta = await store.readRunMeta(runId);
+    return parseRunManifest(meta.run_manifest);
+}
+export async function patchRunManifest(store, runId, mutate, namedSecrets) {
+    const current = await readParsedRunManifest(store, runId);
+    if (current === null || current.manifest_version !== 1)
+        return null;
+    const next = redactRunManifest(mutate(current), {
+        namedSecrets: namedSecrets ?? getNamedSecrets(),
+    });
+    await store.updateRunManifest(runId, next);
+    return next;
+}
+export async function finaliseStoredRunManifest(store, runId, finalisedAt = new Date().toISOString()) {
+    await patchRunManifest(store, runId, (m) => m.finalised_at !== undefined ? m : finaliseRunManifest(m, finalisedAt));
+}

package/dist/runstore/runProjection.d.ts

@@ -1,9 +1,11 @@
-import type { FeedbackLoopHistory, FeedbackLoopRecord, RunDetail, RunMeta, RunPipelineDagSnapshot, RunSummary, StageSnapshot } from "./port.js";
+import type { FeedbackLoopHistory, FeedbackLoopRecord, RunBindingCompact, RunBindingDetail, RunDetail, RunMeta, RunPipelineDagSnapshot, RunSummary, StageSnapshot } from "./port.js";
 export { syntheticPendingSnapshot } from "./syntheticStageSnapshot.js";
 export type FeedbackProjectionInput = {
     feedback_loops?: FeedbackLoopHistory[];
     active_feedback_loop?: FeedbackLoopRecord;
 };
+export declare function projectRunBindingCompact(meta: RunMeta): RunBindingCompact;
+export declare function projectRunBindingDetail(meta: RunMeta): RunBindingDetail;
 export declare function overlayPlannedStages(stageIds: string[], snapshots: StageSnapshot[], dag?: Pick<RunPipelineDagSnapshot, "nodes"> | null): StageSnapshot[];
 export declare function orderStageSnapshots(stageIds: string[], snapshots: StageSnapshot[]): StageSnapshot[];
 export declare function projectRunSummary(meta: RunMeta, stages: StageSnapshot[], feedback?: Pick<FeedbackProjectionInput, "active_feedback_loop">): RunSummary;

package/dist/runstore/runProjection.js

@@ -3,6 +3,47 @@ import { linearCompatDagSnapshot } from "./pipelineDagSnapshot.js";
 import { syntheticPendingSnapshot } from "./syntheticStageSnapshot.js";
 import { buildPipelineTrack } from "./trackProjection.js";
 export { syntheticPendingSnapshot } from "./syntheticStageSnapshot.js";
+export function projectRunBindingCompact(meta) {
+    if (meta.repository != null && meta.repository !== "") {
+        return {
+            kind: "repository",
+            repository: meta.repository,
+            ...(meta.ref !== undefined ? { ref: meta.ref } : {}),
+            ...(meta.resolved_sha !== undefined
+                ? { resolved_sha: meta.resolved_sha }
+                : {}),
+        };
+    }
+    if (meta.checkout_root != null && meta.checkout_root !== "") {
+        return {
+            kind: "checkout",
+            ...(meta.resolved_sha !== undefined
+                ? { resolved_sha: meta.resolved_sha }
+                : {}),
+        };
+    }
+    return { kind: "unbound" };
+}
+export function projectRunBindingDetail(meta) {
+    const compact = projectRunBindingCompact(meta);
+    if (compact.kind === "unbound")
+        return compact;
+    if (compact.kind === "checkout") {
+        return {
+            ...compact,
+            ...(meta.checkout_root !== undefined
+                ? { checkout_root: meta.checkout_root }
+                : {}),
+        };
+    }
+    return {
+        ...compact,
+        ...(meta.run_branch !== undefined ? { run_branch: meta.run_branch } : {}),
+        ...(meta.checkout_root !== undefined
+            ? { checkout_root: meta.checkout_root }
+            : {}),
+    };
+}
 export function overlayPlannedStages(stageIds, snapshots, dag) {
     const byId = new Map(snapshots.map((s) => [s.stage_id, s]));
     const nodeById = new Map((dag?.nodes ?? []).map((n) => [n.id, n]));
@@ -85,7 +126,7 @@ function failedFieldsFromStages(stages, dag) {
 function resolveListedStatus(stages, meta, dag) {
     if (stages.length === 0)
         return meta.status ?? "created";
-    const derived = deriveStatusFromStages(stages, dag);
+    const derived = deriveStatusFromStages(stages, dag, meta.status);
     if (meta.status === "succeeded" && derived === "running") {
         const hasActiveStage = stages.some((s) => s.status === "running" || s.status === "waiting_for_input");
         if (!hasActiveStage)
@@ -113,9 +154,13 @@ export function projectRunSummary(meta, stages, feedback) {
         ...(meta.pipeline_path !== undefined ? { pipeline_path: meta.pipeline_path } : {}),
         ...(meta.task_path !== undefined ? { task_path: meta.task_path } : {}),
         ...(meta.project_root !== undefined ? { project_root: meta.project_root } : {}),
+        ...(meta.pipeline_source !== undefined
+            ? { pipeline_source: meta.pipeline_source }
+            : {}),
         status: resolveListedStatus(stages, meta, dag),
         created_at: meta.created_at,
         updated_at: meta.updated_at,
+        binding: projectRunBindingCompact(meta),
         stages: compactStages(stages),
         ...waiting,
         ...failedFieldsFromStages(stages, dag),
@@ -123,6 +168,15 @@ export function projectRunSummary(meta, stages, feedback) {
             ? { active_feedback_loop: feedback.active_feedback_loop }
             : {}),
         ...(cost !== undefined ? { total_cost_usd: cost } : {}),
+        ...(meta.cancel_reason !== undefined
+            ? { cancel_reason: meta.cancel_reason }
+            : {}),
+        ...(meta.finished_at !== undefined ? { finished_at: meta.finished_at } : {}),
+        ...(meta.slimmed_at !== undefined ? { slimmed_at: meta.slimmed_at } : {}),
+        ...(meta.disk_bytes !== undefined ? { disk_bytes: meta.disk_bytes } : {}),
+        ...(meta.disk_measured_at !== undefined
+            ? { disk_measured_at: meta.disk_measured_at }
+            : {}),
     };
 }
 export function projectRunDetail(meta, stages, task_yaml, dagSnapshot, feedback) {
@@ -139,9 +193,17 @@ export function projectRunDetail(meta, stages, task_yaml, dagSnapshot, feedback)
     });
     return {
         ...summary,
+        binding: projectRunBindingDetail(dag ? { ...meta, pipeline_dag: dag } : meta),
         task_yaml,
         stages: ordered,
         pipeline_track,
         feedback_loops: feedback?.feedback_loops ?? [],
+        ...(meta.config_origins !== undefined && meta.config_origins.length > 0
+            ? { config_origins: meta.config_origins }
+            : {}),
+        ...(meta.caller_id !== undefined ? { caller_id: meta.caller_id } : {}),
+        ...(meta.run_manifest !== undefined
+            ? { run_manifest: meta.run_manifest }
+            : {}),
     };
 }

package/dist/runstore/sqlite/SqliteRunStore.d.ts

@@ -1,15 +1,22 @@
 import Database from "better-sqlite3";
 import type { StageEnvelope } from "../../types/envelope.js";
 import type { StageLogLine } from "../../agent/activity.js";
-import { type CreateFeedbackLoopInput, type CreateFeedbackReplayInput, type CreateFeedbackReplayStagePassInput, type CreateForkGenerationInput, type CreateRunInput, type CreatedRun, type FeedbackLoopPatch, type FeedbackLoopRecord, type FeedbackLoopState, type FeedbackReplayPatch, type FeedbackReplayRecord, type FeedbackReplayStagePassPatch, type FeedbackReplayStagePassRecord, type ForkGenerationPatch, type ForkGenerationRecord, type ListRunsFilter, type RunDetail, type RunMeta, type RunPipelineDagSnapshot, type RunStatus, type RunStore, type RunSummary, type StageExecution, type StageExecutionPatch, type StageLogEvent, type VerificationCheckResult, type VerificationCheckResultPatch } from "../port.js";
+import { type CreateFeedbackLoopInput, type CreateFeedbackReplayInput, type CreateFeedbackReplayStagePassInput, type CreateForkGenerationInput, type CreateRunInput, type CreatedRun, type FeedbackLoopPatch, type FeedbackLoopRecord, type FeedbackLoopState, type FeedbackReplayPatch, type FeedbackReplayRecord, type FeedbackReplayStagePassPatch, type FeedbackReplayStagePassRecord, type ForkGenerationPatch, type ForkGenerationRecord, type ListRunsFilter, type RunDetail, type RunMeta, type RunPipelineDagSnapshot, type RunStatus, type RunStore, type RunSummary, type StageExecution, type StageExecutionPatch, type StageLogEvent, type TriggerRecord, type UpsertTriggerInput, type VerificationCheckResult, type VerificationCheckResultPatch, type ConfigOriginRecord } from "../port.js";
 import { type RunSubmissionRecord } from "../submission.js";
+type SqliteRunStoreOpenerMode = "assert" | "migrate";
 export declare class SqliteRunStore implements RunStore {
     private readonly storeRoot;
     private readonly db;
     private migratePromise;
-    constructor(storeRoot: string);
+    constructor(storeRoot: string, options?: {
+        openerMode?: SqliteRunStoreOpenerMode;
+    });
     /** Await one-shot disk import before catalog reads (create paths also wait). */
     ready(): Promise<void>;
+    close(): Promise<void>;
+    snapshotInto(destPath: string): Promise<{
+        userVersion: number;
+    }>;
     /**
      * The connection this store owns, for the one caller allowed to share it: the composition root
      * wiring the A2A tables into the same `state.db` file. Not part of the `RunStore` interface.
@@ -20,9 +27,24 @@ export declare class SqliteRunStore implements RunStore {
     private readSubmission;
     getRunBySubmission(key: string): Promise<RunSubmissionRecord | null>;
     updateRunStatus(runId: string, status: RunStatus): Promise<void>;
+    tryUpdateRunStatus(runId: string, status: RunStatus, expectedStatus: RunStatus): Promise<boolean>;
+    patchRunWorkspaceBinding(runId: string, patch: {
+        checkoutRoot?: string;
+        repository?: string;
+        ref?: string;
+        resolvedSha?: string;
+        runBranch?: string;
+    }): Promise<void>;
+    appendConfigOrigins(runId: string, origins: ConfigOriginRecord[]): Promise<void>;
+    setCancelReason(runId: string, reason: string): Promise<void>;
+    setRunDiskUsage(runId: string, diskBytes: number, measuredAt: string): Promise<void>;
+    setSlimmedAt(runId: string, slimmedAt: string): Promise<void>;
+    deleteRun(runId: string): Promise<void>;
     updatePipelineDag(runId: string, dag: RunPipelineDagSnapshot): Promise<void>;
     readRunMeta(runId: string): Promise<RunMeta>;
     readTaskYaml(runId: string): Promise<string>;
+    readPipelineBody(runId: string): Promise<string | null>;
+    updateRunManifest(runId: string, manifest: unknown): Promise<void>;
     ensureStageWorkspace(runId: string, stageId: string): Promise<void>;
     ensureAttemptWorkspace(runId: string, stageId: string, attempt: number): Promise<void>;
     createStageExecution(runId: string, stageId: string): Promise<StageExecution>;
@@ -30,6 +52,7 @@ export declare class SqliteRunStore implements RunStore {
     getLatestStageExecution(runId: string, stageId: string): Promise<StageExecution | null>;
     countStageAttempts(runId: string, stageId: string): Promise<number>;
     getStageExecution(runId: string, stageId: string, attempt: number): Promise<StageExecution>;
+    listInterruptedStageExecutions(): Promise<StageExecution[]>;
     updateStageExecution(runId: string, stageId: string, attempt: number, patch: StageExecutionPatch): Promise<void>;
     upsertVerificationCheckResult(runId: string, stageId: string, result: VerificationCheckResultPatch, options?: {
         attempt?: number;
@@ -52,7 +75,15 @@ export declare class SqliteRunStore implements RunStore {
     private updateStageStatusDenorm;
     listStageEvents(runId: string, stageId: string, attempt?: number): Promise<StageLogEvent[]>;
     listRuns(filter?: ListRunsFilter): Promise<RunSummary[]>;
-    listProjectRoots(): Promise<string[]>;
+    ensureProject(absPath: string): Promise<string>;
+    listRegisteredProjects(): Promise<string[]>;
+    upsertTrigger(input: UpsertTriggerInput): Promise<TriggerRecord>;
+    getTrigger(id: string): Promise<TriggerRecord | null>;
+    listTriggers(): Promise<TriggerRecord[]>;
+    recordTriggerFired(id: string, runId: string): Promise<void>;
+    setTriggerNextRun(id: string, nextRunAt: string): Promise<void>;
+    getTriggerAdapterState(triggerId: string, key: string): Promise<string | null>;
+    setTriggerAdapterState(triggerId: string, key: string, value: string): Promise<void>;
     readRun(runId: string): Promise<RunDetail>;
     createFeedbackLoop(runId: string, input: CreateFeedbackLoopInput): Promise<FeedbackLoopRecord>;
     getFeedbackLoop(runId: string, loopId: string): Promise<FeedbackLoopRecord>;
@@ -81,3 +112,4 @@ export declare class SqliteRunStore implements RunStore {
     private loadStageSnapshots;
 }
 export declare function openSqliteRunStore(storeRoot: string): SqliteRunStore;
+export {};