stageflow 0.2.0 → 0.8.0

package/dist/runtime/credentialBinding.js

@@ -1,14 +1,15 @@
-import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { storeRootFor } from "../runstore/paths.js";
-import { parseCredentialSource, readCredentialSourceFromFile, writeCredentialSourceToFile, } from "./settingsFile.js";
-export { parseCredentialSource, readCredentialSourceFromFile, writeCredentialSourceToFile, };
-export function sfOwnedAgentDir(cwd) {
-    return path.join(storeRootFor(cwd), "agent");
+import { ensureGlobalHome } from "../project/globalHome.js";
+import { resolveProjectContext, } from "../project/resolveProjectContext.js";
+import { parseCredentialSource, readCredentialSourceFromContext, writeCredentialSourceToContext, } from "./settingsFile.js";
+export { parseCredentialSource, readCredentialSourceFromContext, writeCredentialSourceToContext, };
+export function sfOwnedAgentDir() {
+    return path.join(ensureGlobalHome(), "agent");
 }
-export function sfOwnedAuthPath(cwd) {
-    return path.join(sfOwnedAgentDir(cwd), "auth.json");
+export function sfOwnedAuthPath() {
+    return path.join(sfOwnedAgentDir(), "auth.json");
 }
 export function piHomeAuthPath(override) {
     return (override ??
@@ -25,18 +26,9 @@ export function isUsableAuthFile(authPath) {
         return false;
     }
 }
-export function ensureSfOwnedAuthStore(cwd) {
-    const storeRoot = storeRootFor(cwd);
-    mkdirSync(storeRoot, { recursive: true });
-    const agentDir = sfOwnedAgentDir(cwd);
-    mkdirSync(agentDir, { recursive: true });
-    try {
-        chmodSync(agentDir, 0o700);
-    }
-    catch {
-        // best-effort on non-POSIX
-    }
-    const authPath = sfOwnedAuthPath(cwd);
+export function ensureSfOwnedAuthStore() {
+    ensureGlobalHome();
+    const authPath = sfOwnedAuthPath();
     if (!existsSync(authPath)) {
         writeFileSync(authPath, "{}\n", { encoding: "utf8", mode: 0o600 });
     }
@@ -48,14 +40,15 @@ export function ensureSfOwnedAuthStore(cwd) {
     }
     return authPath;
 }
-export function resolveCredentialBinding(cwd, options = {}) {
+export function resolveCredentialBinding(ctx, options = {}) {
+    const projectCtx = typeof ctx === "string" ? resolveProjectContext(ctx) : ctx;
     const piHome = piHomeAuthPath(options.piHomeAuthPath);
-    const persisted = readCredentialSourceFromFile(cwd);
+    const persisted = readCredentialSourceFromContext(projectCtx);
     if (persisted !== undefined) {
         if (persisted === "sf_owned") {
             return {
                 source: "sf_owned",
-                authPath: ensureSfOwnedAuthStore(cwd),
+                authPath: ensureSfOwnedAuthStore(),
                 provisional: false,
             };
         }
@@ -74,7 +67,8 @@ export function resolveCredentialBinding(cwd, options = {}) {
     }
     return {
         source: "sf_owned",
-        authPath: ensureSfOwnedAuthStore(cwd),
+        authPath: ensureSfOwnedAuthStore(),
         provisional: true,
     };
 }
+export { readCredentialSourceFromFile, writeCredentialSourceToFile, } from "./settingsFile.js";

package/dist/runtime/envelopeRouting.d.ts

@@ -2,11 +2,12 @@ import type { RunStore, StageSnapshot } from "../runstore/port.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
 import type { StageConfig } from "../types/stage.js";
-export declare function buildCompletedEnvelopesFromRun(store: RunStore, runId: string, stages?: StageSnapshot[]): Promise<Map<string, StageEnvelope>>;
+export declare function buildCompletedEnvelopesFromRun(store: RunStore, runId: string, stages?: StageSnapshot[], dag?: ResolvedPipelineDag): Promise<Map<string, StageEnvelope>>;
 export declare function buildStageConfigById(loaded: LoadedPipeline): Map<string, StageConfig>;
 export type ResolvePriorEnvelopeResult = {
     ok: true;
     prior: StageEnvelope | null;
+    joinPriors?: StageEnvelope[];
 } | {
     ok: false;
     reason: string;

package/dist/runtime/envelopeRouting.js

@@ -1,15 +1,34 @@
-export async function buildCompletedEnvelopesFromRun(store, runId, stages) {
+import { instancesOfDefinition, } from "../runstore/pipelineDagSnapshot.js";
+import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
+function definitionInstances(dag, catalogId) {
+    const snapshot = dag;
+    if (Array.isArray(snapshot.stage_ids)) {
+        return instancesOfDefinition(snapshot, catalogId);
+    }
+    return dag.nodes.some((n) => n.id === catalogId) ? [catalogId] : [];
+}
+export async function buildCompletedEnvelopesFromRun(store, runId, stages, dag) {
     const completedEnvelopes = new Map();
     const snapshotStages = stages ?? (await store.readRun(runId)).stages;
     for (const snap of snapshotStages) {
-        if (snap.status !== "succeeded")
+        const isCloneInstance = dag !== undefined &&
+            definitionIdForInstance(dag, snap.stage_id) !==
+                snap.stage_id;
+        if (snap.status !== "succeeded" && !(snap.status === "failed" && isCloneInstance)) {
             continue;
+        }
         try {
             const envelope = snap.envelope ?? (await store.readEnvelope(runId, snap.stage_id));
             completedEnvelopes.set(snap.stage_id, envelope);
         }
         catch {
-            // envelope may be missing for corrupt runs; leave map empty
+            if (snap.status === "failed" && isCloneInstance) {
+                completedEnvelopes.set(snap.stage_id, {
+                    status: "failure",
+                    summary: "stage failed",
+                    artifacts: [],
+                });
+            }
         }
     }
     return completedEnvelopes;
@@ -35,15 +54,41 @@ export async function resolvePriorEnvelope(options) {
     if (node.needs === null) {
         return { ok: true, prior: null };
     }
+    const joinInstances = definitionInstances(options.dag, node.needs);
+    if (joinInstances.length > 1) {
+        const joinPriors = [];
+        for (const id of joinInstances) {
+            const fromMap = options.completedEnvelopes.get(id);
+            if (fromMap !== undefined) {
+                joinPriors.push(structuredClone(fromMap));
+                continue;
+            }
+            if (options.store !== undefined && options.runId !== undefined) {
+                try {
+                    const envelope = await options.store.readEnvelope(options.runId, id);
+                    joinPriors.push(structuredClone(envelope));
+                    continue;
+                }
+                catch {
+                    return {
+                        ok: false,
+                        reason: `missing envelope for clone instance "${id}"`,
+                    };
+                }
+            }
+            return {
+                ok: false,
+                reason: `missing envelope for clone instance "${id}"`,
+            };
+        }
+        return { ok: true, prior: null, joinPriors };
+    }
     const parentId = node.needs;
     const fromMap = options.completedEnvelopes.get(parentId);
-    if (fromMap !== undefined) {
-        return { ok: true, prior: structuredClone(fromMap) };
-    }
-    if (options.store !== undefined && options.runId !== undefined) {
+    let parent = fromMap;
+    if (parent === undefined && options.store !== undefined && options.runId !== undefined) {
         try {
-            const envelope = await options.store.readEnvelope(options.runId, parentId);
-            return { ok: true, prior: structuredClone(envelope) };
+            parent = await options.store.readEnvelope(options.runId, parentId);
         }
         catch {
             return {
@@ -52,8 +97,30 @@ export async function resolvePriorEnvelope(options) {
             };
         }
     }
-    return {
-        ok: false,
-        reason: `missing envelope for upstream stage "${parentId}"`,
-    };
+    if (parent === undefined) {
+        return {
+            ok: false,
+            reason: `missing envelope for upstream stage "${parentId}"`,
+        };
+    }
+    for (const item of parent.clone_forks ?? []) {
+        if (item.action === "once" && item.successor_id === options.stageId) {
+            return { ok: true, prior: structuredClone(item.envelope) };
+        }
+        if (item.action === "fanout") {
+            const instanceIds = definitionInstances(options.dag, item.successor_id);
+            const index = instanceIds.indexOf(options.stageId);
+            if (index >= 0) {
+                const clone = item.clones[index];
+                if (clone === undefined) {
+                    return {
+                        ok: false,
+                        reason: `missing clone envelope for instance "${options.stageId}"`,
+                    };
+                }
+                return { ok: true, prior: structuredClone(clone.envelope) };
+            }
+        }
+    }
+    return { ok: true, prior: structuredClone(parent) };
 }

package/dist/runtime/pipelineRunner.d.ts

@@ -31,6 +31,7 @@ export type PreparedPipeline = {
     agent: AgentPort;
     store: RunStore;
     cwd: string;
+    projectRoot: string;
     checkoutRoot?: string;
     hitl?: StageHitlController;
     executionMode?: StageExecutionMode;
@@ -55,6 +56,7 @@ export declare function runPipeline(options: {
     taskYaml?: string;
     pipeline: string;
     cwd?: string;
+    projectRoot?: string;
     checkoutOverride?: string;
     hitl?: StageHitlController;
     maxActiveStagesPerRun?: number;
@@ -71,6 +73,7 @@ export declare function startPipeline(options: {
     taskYaml?: string;
     pipeline: string;
     cwd?: string;
+    projectRoot?: string;
     checkoutOverride?: string;
     gitSha?: string;
     ciPrUrl?: string;

package/dist/runtime/pipelineRunner.js

@@ -1,10 +1,11 @@
 import { readFile } from "node:fs/promises";
 import path from "node:path";
-import { loadPipelineValidated } from "../config/loadPipeline.js";
+import { loadPipelineValidated } from "../config/validateCatalog.js";
 import { loadTaskFromYaml } from "../config/loadTask.js";
 import { buildValidationResult } from "../config/validateCatalog.js";
 import { resolveAndValidateCheckout } from "./stageRoots.js";
 import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
+import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
 import { runPipelineDag } from "./pipelineScheduler.js";
 import { readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
@@ -41,6 +42,11 @@ async function preparePipeline(options) {
     const label = options.taskPath ?? "task.yaml";
     const task = loadTaskFromYaml(taskYaml, `task file ${label}`);
     const checkoutRoot = await resolveAndValidateCheckout(task, options.checkoutOverride, options.cwd);
+    const pipelinePath = normalizeCatalogPath(loaded.pipelinePath);
+    const taskPath = options.taskPath
+        ? normalizeCatalogPath(path.resolve(options.cwd, options.taskPath))
+        : undefined;
+    const projectRoot = normalizeCatalogPath(options.projectRoot ?? options.cwd);
     const run = await options.store.createRun({
         pipelineId: loaded.pipeline.id,
         taskYaml,
@@ -50,6 +56,9 @@ async function preparePipeline(options) {
         ciPrUrl: options.ciPrUrl,
         ciJobUrl: options.ciJobUrl,
         pipelineDag: buildPipelineDagSnapshotFromLoaded(loaded),
+        pipelinePath,
+        taskPath,
+        projectRoot,
     });
     const executionMode = readStageExecutionMode(process.env, options.executionMode);
     const stageProcessLauncher = resolveStageProcessLauncher(executionMode, options.stageProcessLauncher);
@@ -60,6 +69,7 @@ async function preparePipeline(options) {
         agent: options.agent,
         store: options.store,
         cwd: options.cwd,
+        projectRoot: options.projectRoot ?? options.cwd,
         checkoutRoot,
         hitl: options.hitl,
         executionMode,
@@ -86,6 +96,7 @@ export async function executeStages(prepared, options) {
 }
 export async function runPipeline(options) {
     const cwd = options.cwd ?? process.cwd();
+    const projectRoot = options.projectRoot ?? cwd;
     const prepared = await preparePipeline({
         agent: options.agent,
         store: options.store,
@@ -93,6 +104,7 @@ export async function runPipeline(options) {
         taskYaml: options.taskYaml,
         pipeline: options.pipeline,
         cwd,
+        projectRoot,
         checkoutOverride: options.checkoutOverride,
         hitl: options.hitl,
         executionMode: options.executionMode,
@@ -109,6 +121,7 @@ export async function runPipeline(options) {
 /** Create the run immediately, then execute stages in the returned promise. */
 export async function startPipeline(options) {
     const cwd = options.cwd ?? process.cwd();
+    const projectRoot = options.projectRoot ?? cwd;
     const prepared = await preparePipeline({
         agent: options.agent,
         store: options.store,
@@ -116,6 +129,7 @@ export async function startPipeline(options) {
         taskYaml: options.taskYaml,
         pipeline: options.pipeline,
         cwd,
+        projectRoot,
         checkoutOverride: options.checkoutOverride,
         gitSha: options.gitSha,
         ciPrUrl: options.ciPrUrl,

package/dist/runtime/pipelineScheduler.d.ts

@@ -1,8 +1,9 @@
 import type { AgentPort } from "../agent/port.js";
-import type { RunStore, StageSnapshot } from "../runstore/port.js";
+import type { RunPipelineDagSnapshot, RunStore, StageSnapshot } from "../runstore/port.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
 import type { TaskFile } from "../types/task.js";
+import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule } from "./cloneSchedule.js";
 import type { PipelineRunResult } from "./pipelineRunner.js";
 import type { StageProcessLauncher } from "./stageProcessLauncher.js";
 import type { StageExecutionMode } from "./stageConcurrency.js";
@@ -18,6 +19,7 @@ type SchedulerPreparedPipeline = {
     agent: AgentPort;
     store: RunStore;
     cwd: string;
+    projectRoot?: string;
     checkoutRoot?: string;
     hitl?: StageHitlController;
     operatorCatalog?: OperatorCatalog;
@@ -38,13 +40,16 @@ type HasWorkOutsideBranchContext = {
     retryRoots?: ReadonlyMap<string, number>;
     pendingDeltaStageIds?: ReadonlySet<string>;
 };
-type StageScheduleState = "pending" | "active" | "waiting" | "succeeded" | "failed" | "skipped";
+export type StageScheduleState = "pending" | "active" | "waiting" | "succeeded" | "failed" | "skipped";
 export type HydratedSchedule = {
     states: Map<string, StageScheduleState>;
     completedEnvelopes: Map<string, StageEnvelope>;
     schedulingHalted: boolean;
     firstFailureReason: string | undefined;
+    dag?: RunPipelineDagSnapshot;
 };
+export { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, };
+export type { ApplyCloneForksResult } from "./cloneSchedule.js";
 export declare function snapshotToScheduleState(snap: {
     status: StageSnapshot["status"];
 }, overrides: ReadonlyMap<string, StageScheduleState>, stageId: string, executionMode?: StageExecutionMode): StageScheduleState;
@@ -86,6 +91,7 @@ export type RetryRunOptions = {
     onRetryRootTerminal?: OnRetryRootTerminal;
 };
 export declare function retryRun(options: RetryRunOptions): Promise<PipelineRunResult>;
+export declare function applyForkSkipsFromEnvelopes(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): void;
 export declare function applyRetryRootDelta(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>, retryRoots: Map<string, number>, stageId: string, attempt: number, clearSchedulingHalt?: () => void): void;
 export declare function hasWorkOutsideBranch(dag: ResolvedPipelineDag, branchRootId: string, states: Map<string, StageScheduleState>, ctx?: HasWorkOutsideBranchContext): boolean;
 export declare function runPipelineDag(options: RunPipelineDagOptions): Promise<PipelineRunResult>;