stageflow 0.27.0 → 0.31.0

package/dist/runtime/runManager.js

@@ -17,10 +17,17 @@ import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnaps
 import { InlinePipelineTooLargeError, PipelinePreflightError, PipelineValidationError, QueuedRunActivationAborted, startPipeline, } from "./pipelineRunner.js";
 import { INLINE_PIPELINE_TOO_LARGE, pipelineBodyBytes, pipelinePersistenceForStart, } from "./startPayload.js";
 import { materializeRunSkills, validateSkillsPayload, } from "./runSkills.js";
-import { hydrateScheduleFromStore, hydratedScheduleHasRunnableWork, resumeRun, runPipelineDag, } from "./pipelineScheduler.js";
+import { hydrateScheduleFromStore, hydratedScheduleHasRunnableWork, resumeRun, retryRun, runPipelineDag, } from "./pipelineScheduler.js";
 import { resolveFeedbackLoopDecision, } from "./feedbackLoopDecision.js";
 import { RunRetryCoordinator, readRetryRootWaitTimeoutMs, } from "./runRetryCoordinator.js";
 import { readMaxActiveStageProcesses, readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
+import { isHumanLoginStage } from "../browser/humanLogin.js";
+import { defaultStageBrowserSupport, resolveStageBrowserEnv, } from "../browser/stageBrowserEnv.js";
+import { teardownRunBrowsers, teardownStageBrowser, } from "../browser/browserTeardown.js";
+import { createRunLiveness } from "../browser/runLiveness.js";
+import { acquireStageProfile, stageProfileLock, } from "../browser/stageProfileLock.js";
+import { sweepOrphanBrowserSessions } from "../browser/browserSweep.js";
+import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
 import { logger as rootLogger } from "../logging/logger.js";
 import { registerNamedSecrets } from "../logging/namedSecrets.js";
@@ -584,6 +591,23 @@ export class RunManager {
             .getActiveStageProcesses()
             .some((entry) => entry.runId === runId && entry.stageId === stageId);
     }
+    async sweepBrowserSessions() {
+        const support = this.options.browser ?? defaultStageBrowserSupport();
+        const isRunLive = createRunLiveness(this.options.store);
+        await stageProfileLock(support)
+            .reclaimStale(isRunLive)
+            .catch(() => 0);
+        return sweepOrphanBrowserSessions({
+            isRunLive,
+            ...(support.runner !== undefined ? { runner: support.runner } : {}),
+            ...(support.closeWaitMs !== undefined
+                ? { closeWaitMs: support.closeWaitMs }
+                : {}),
+            ...(support.socketRoot !== undefined
+                ? { socketRoot: support.socketRoot }
+                : {}),
+        });
+    }
     async reconcileOrphanedStages() {
         const reconciled = [];
         const runs = await this.options.store.listRuns();
@@ -847,6 +871,12 @@ export class RunManager {
                 await this.stageProcessLauncher.cancelRun(runId);
             }
         }
+        await teardownStageBrowser(this.options.browser ?? defaultStageBrowserSupport(), {
+            runId,
+            runDir: this.options.store.getWorkspaceDir(runId),
+            stageId,
+            events: () => this.options.store.listStageEvents(runId, stageId),
+        }).catch(() => undefined);
         await markStageInterrupted({
             store: this.options.store,
             runId,
@@ -861,6 +891,14 @@ export class RunManager {
         await syncRunStatusFromStages(this.options.store, runId);
         const after = await this.options.store.readRun(runId);
         const hasWaiting = after.stages.some((s) => s.status === "waiting_for_input");
+        if (!hasWaiting &&
+            !(await createRunLiveness(this.options.store)(runId).catch(() => true))) {
+            await teardownRunBrowsers(this.options.browser ?? defaultStageBrowserSupport(), {
+                runId,
+                runDir: this.options.store.getWorkspaceDir(runId),
+                events: (stageId) => this.options.store.listStageEvents(runId, stageId),
+            }).catch(() => undefined);
+        }
         if (!hasWaiting && this.active.has(runId)) {
             this.removeActiveEntry(runId, false);
         }
@@ -912,6 +950,11 @@ export class RunManager {
         if (this.stageProcessLauncher !== undefined) {
             await this.stageProcessLauncher.cancelRun(runId);
         }
+        await teardownRunBrowsers(this.options.browser ?? defaultStageBrowserSupport(), {
+            runId,
+            runDir: this.options.store.getWorkspaceDir(runId),
+            events: (stageId) => this.options.store.listStageEvents(runId, stageId),
+        });
         const detail = await this.options.store.readRun(runId);
         for (const stage of detail.stages) {
             if (stage.status !== "pending" &&
@@ -1478,6 +1521,9 @@ export class RunManager {
                 agent: this.options.agent,
                 cwd: meta.project_root ?? this.projectRoot,
                 operatorCatalog: this.options.operatorCatalog,
+                ...(this.options.browser !== undefined
+                    ? { browser: this.options.browser }
+                    : {}),
                 maxActiveStagesPerRun: this.maxActiveStagesPerRun,
                 executionMode: this.executionMode,
                 stageProcessLauncher: this.stageProcessLauncher,
@@ -1841,6 +1887,9 @@ export class RunManager {
                     checkoutRoot: meta.checkout_root,
                     hitl: this.hitl,
                     operatorCatalog: this.options.operatorCatalog,
+                    ...(this.options.browser !== undefined
+                        ? { browser: this.options.browser }
+                        : {}),
                 },
                 maxActiveStagesPerRun: this.maxActiveStagesPerRun,
                 executionMode: this.executionMode,
@@ -1897,16 +1946,40 @@ export class RunManager {
                 runWorkspaceDir: workspaceDir,
                 hostEnv: process.env,
             });
+            const resumedStage = loaded.stages.find((s) => s.id === definitionIdForInstance(meta.pipeline_dag, stageId));
+            if (resumedStage?.browser?.profile !== undefined) {
+                await acquireStageProfile(this.options.browser ?? defaultStageBrowserSupport(), {
+                    profile: resumedStage.browser.profile,
+                    owner: { runId, stageId },
+                    isRunLive: createRunLiveness(store),
+                    halted: () => true,
+                }).then((outcome) => {
+                    if (outcome === "halted") {
+                        throw new Error(`browser profile "${resumedStage.browser?.profile}" is held by another run`);
+                    }
+                });
+            }
+            const browserEnv = await resolveStageBrowserEnv(this.options.browser ?? defaultStageBrowserSupport(), {
+                runId,
+                stageId,
+                runDir: workspaceDir,
+                browser: resumedStage?.browser,
+                attempt,
+                humanLogin: isHumanLoginStage(meta.pipeline_dag, definitionIdForInstance(meta.pipeline_dag, stageId), resumedStage?.browser),
+                resuming: true,
+            });
             const launchResult = await launcher.launch({
                 runId,
                 stageId,
                 rootDir: runProjectRoot,
+                ...(browserEnv !== undefined ? { browserEnv } : {}),
                 mode: "resume",
                 resumeAnswer: opaqueAnswer,
                 attempt,
                 sessionFilePath,
                 env: stageBinding.env,
                 bindingKind: stageBinding.kind,
+                ...(resumedStage !== undefined ? { model: resumedStage.model } : {}),
                 ...(this.options.operatorCatalog !== undefined
                     ? { operatorCatalog: this.options.operatorCatalog }
                     : {}),
@@ -1914,12 +1987,53 @@ export class RunManager {
             if (launchResult.type === "waiting") {
                 return { ok: true };
             }
+            await teardownStageBrowser(this.options.browser ?? defaultStageBrowserSupport(), {
+                runId,
+                runDir: workspaceDir,
+                stageId,
+                events: () => store.listStageEvents(runId, stageId),
+            }).catch(() => undefined);
             if (launchResult.type === "failed") {
+                const nextRepairAttempt = await this.scheduleRepairAfterVerifyFailure(runId, stageId, meta.pipeline_dag);
+                if (nextRepairAttempt !== undefined) {
+                    await store.updateRunStatus(runId, "running");
+                    const repaired = await retryRun({
+                        prepared: {
+                            task,
+                            loaded,
+                            run: { runId, workspaceDir: store.getWorkspaceDir(runId) },
+                            agent: this.options.agent,
+                            store,
+                            cwd: meta.project_root ?? this.projectRoot,
+                            projectRoot: meta.project_root ?? this.projectRoot,
+                            checkoutRoot: meta.checkout_root,
+                            hitl: this.hitl,
+                            operatorCatalog: this.options.operatorCatalog,
+                            ...(this.options.browser !== undefined
+                                ? { browser: this.options.browser }
+                                : {}),
+                        },
+                        retryRoots: new Map([[stageId, nextRepairAttempt]]),
+                        maxActiveStagesPerRun: this.maxActiveStagesPerRun,
+                        executionMode: this.executionMode,
+                        stageProcessLauncher: launcher,
+                        schedulingHalt: this.ensureSchedulingHalt(runId),
+                    });
+                    if (repaired.outcome === "failed") {
+                        return { ok: false, reason: repaired.reason };
+                    }
+                    return { ok: true, verification: "failed_retrying" };
+                }
                 await store.appendStageEvent(runId, stageId, {
                     event: "failed",
                     reason: launchResult.reason,
                 }, eventOptions);
                 await store.updateRunStatus(runId, "failed");
+                await teardownRunBrowsers(this.options.browser ?? defaultStageBrowserSupport(), {
+                    runId,
+                    runDir: workspaceDir,
+                    events: (stageId) => store.listStageEvents(runId, stageId),
+                }).catch(() => undefined);
                 return { ok: false, reason: launchResult.reason };
             }
             let initialPrior = null;
@@ -1941,6 +2055,9 @@ export class RunManager {
                     checkoutRoot: meta.checkout_root,
                     hitl: this.hitl,
                     operatorCatalog: this.options.operatorCatalog,
+                    ...(this.options.browser !== undefined
+                        ? { browser: this.options.browser }
+                        : {}),
                 },
                 maxActiveStagesPerRun: this.maxActiveStagesPerRun,
                 resumeFromStageId: stageId,
@@ -1964,6 +2081,11 @@ export class RunManager {
                     reason,
                 }, eventOptions);
                 await store.updateRunStatus(runId, "failed");
+                await teardownRunBrowsers(this.options.browser ?? defaultStageBrowserSupport(), {
+                    runId,
+                    runDir: store.getWorkspaceDir(runId),
+                    events: (stageId) => store.listStageEvents(runId, stageId),
+                }).catch(() => undefined);
             }
             catch {
                 // ignore secondary failures
@@ -1974,6 +2096,24 @@ export class RunManager {
             this.attachedWaiting.delete(`${runId}\0${stageId}`);
         }
     }
+    /**
+     * A resumed stage whose after-phase verification failed is repaired by the
+     * same policy the scheduler applies to a first-launch failure. Returns the
+     * freshly created attempt, or undefined when the failure is final.
+     */
+    async scheduleRepairAfterVerifyFailure(runId, stageId, dag) {
+        const store = this.options.store;
+        const recovery = dag?.nodes.find((node) => node.id === stageId)?.recovery;
+        if (recovery?.mode !== "repair")
+            return undefined;
+        const latest = await store.getLatestStageExecution(runId, stageId);
+        if (latest?.verification_outcome !== "failed")
+            return undefined;
+        const attempts = await store.countStageAttempts(runId, stageId);
+        if (attempts >= recovery.max_attempts)
+            return undefined;
+        return (await store.createStageExecution(runId, stageId)).attempt;
+    }
     async reconstructAndContinue(runId, stageId, opaqueAnswer) {
         try {
             return await resumeReconstructAndContinue({
@@ -2248,6 +2388,9 @@ export class RunManager {
             const schedulingHalt = this.ensureSchedulingHalt(materialized.runId);
             const started = await startPipeline({
                 submission,
+                ...(this.options.browser !== undefined
+                    ? { browser: this.options.browser }
+                    : {}),
                 agent: this.options.agent,
                 store: this.options.store,
                 taskYaml,
@@ -2364,17 +2507,26 @@ export class RunManager {
         };
     }
     async checkDiskFloorAdmission() {
+        const raw = process.env.STAGEFLOW_MIN_FREE_DISK_BYTES;
+        if (raw === undefined || raw.trim() === "") {
+            return undefined;
+        }
         try {
             const readFree = this.options.freeSpaceReader ?? readFilesystemSize;
             const size = await readFree(globalStageflowHome());
-            const minFreeBytes = resolveMinFreeDiskFloor(process.env.STAGEFLOW_MIN_FREE_DISK_BYTES, size.totalBytes);
+            const minFreeBytes = resolveMinFreeDiskFloor(raw, size.totalBytes);
+            if (minFreeBytes === 0) {
+                return undefined;
+            }
             if (size.freeBytes < minFreeBytes) {
                 return this.insufficientDiskFailure(size.freeBytes, minFreeBytes);
             }
             return undefined;
         }
         catch (err) {
-            // Default floor is always active via resolveMinFreeDiskFloor — fail closed.
+            if (resolveMinFreeDiskFloor(raw, 0) === 0 && !raw.trim().endsWith("%")) {
+                return undefined;
+            }
             return {
                 ok: false,
                 reason: `Disk free-space check failed: ${err instanceof Error ? err.message : String(err)}`,
@@ -2738,6 +2890,9 @@ export class RunManager {
             const schedulingHalt = this.ensureSchedulingHalt(runId);
             const started = await startPipeline({
                 submission,
+                ...(this.options.browser !== undefined
+                    ? { browser: this.options.browser }
+                    : {}),
                 agent: this.options.agent,
                 store: this.options.store,
                 taskYaml,
@@ -2842,6 +2997,9 @@ export class RunManager {
                     checkoutRoot: loadedMeta.checkout_root,
                     hitl: this.hitl,
                     operatorCatalog: this.options.operatorCatalog,
+                    ...(this.options.browser !== undefined
+                        ? { browser: this.options.browser }
+                        : {}),
                 },
                 maxActiveStagesPerRun: this.maxActiveStagesPerRun,
                 executionMode: this.executionMode,

package/dist/runtime/runRetryCoordinator.d.ts

@@ -3,6 +3,7 @@ import { type RunDetail, type RunPipelineDagSnapshot, type RunStatus, type RunSt
 import type { PipelineRunResult } from "./pipelineRunner.js";
 import { type SchedulerPreparedPipeline } from "./pipelineScheduler.js";
 import type { StageExecutionMode } from "./stageConcurrency.js";
+import type { StageBrowserSupport } from "../browser/browserHost.js";
 import type { StageProcessLauncher } from "./stageProcessLauncher.js";
 import type { StageHitlController } from "./stageHitl.js";
 import type { OperatorCatalog } from "./stageAttemptBootstrap.js";
@@ -58,6 +59,7 @@ export type RetryStageRequest = {
     agent: AgentPort;
     cwd: string;
     operatorCatalog?: OperatorCatalog;
+    browser?: StageBrowserSupport;
     maxActiveStagesPerRun: number;
     executionMode: StageExecutionMode;
     stageProcessLauncher?: StageProcessLauncher;

package/dist/runtime/runRetryCoordinator.js

@@ -344,6 +344,7 @@ export class RunRetryCoordinator {
                 checkoutRoot: meta.checkout_root,
                 hitl: req.hitl,
                 operatorCatalog: req.operatorCatalog,
+                ...(req.browser !== undefined ? { browser: req.browser } : {}),
             };
             if (this.isActive(runId)) {
                 await this.addRoot(runId, stageId, execution.attempt);

package/dist/runtime/scheduleSource.d.ts

@@ -0,0 +1,38 @@
+import type { RunStore } from "../runstore/port.js";
+import type { TriggerFireEvent, TriggerSourcePort } from "./triggerPort.js";
+export declare const DEFAULT_TRIGGER_TICK_INTERVAL_MS = 30000;
+/** Parse `STAGEFLOW_TRIGGER_TICK_INTERVAL_MS`; default 30s; `0` disables. */
+export declare function triggerTickIntervalMsFromEnv(env?: NodeJS.ProcessEnv): number;
+export type ScheduleSourceOptions = {
+    store: RunStore;
+    cwd?: string;
+    intervalMs?: number;
+    /** Injectable clock so tests can drive `tick()` without real timers. */
+    now?: () => Date;
+    logError?: (message: string) => void;
+};
+/**
+ * `TriggerSourcePort` adapter for `schedule`-kind triggers. Ticks on an
+ * interval; each tick re-scans the catalog for enabled `schedule` triggers,
+ * seeds `next_run_at` for any that don't have one yet (without firing), and
+ * fires any trigger whose `next_run_at` has passed — always recomputing the
+ * next fire time from *now*, never from the stale `next_run_at`, so a tick
+ * never backfills more than one missed occurrence.
+ */
+export declare class ScheduleSource implements TriggerSourcePort {
+    private readonly store;
+    private readonly cwd;
+    private readonly intervalMs;
+    private readonly now;
+    private readonly logError;
+    private interval;
+    private inFlight;
+    constructor(options: ScheduleSourceOptions);
+    start(onFire: (event: TriggerFireEvent) => Promise<void>): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * One discovery + seed + fire pass. Public so tests can drive it directly
+     * against an injected clock instead of waiting on real interval timers.
+     */
+    tick(onFire: (event: TriggerFireEvent) => Promise<void>): Promise<void>;
+}

package/dist/runtime/scheduleSource.js

@@ -0,0 +1,119 @@
+import path from "node:path";
+import { Cron } from "croner";
+import { getCatalogScanPaths } from "../config/browseCatalog.js";
+import { loadTriggerOutcome } from "../config/loadTrigger.js";
+import { catalogContextFromStageflow } from "../config/resolveCatalogContext.js";
+import { resolveStageflowContext } from "../project/resolveStageflowContext.js";
+export const DEFAULT_TRIGGER_TICK_INTERVAL_MS = 30_000;
+/** Parse `STAGEFLOW_TRIGGER_TICK_INTERVAL_MS`; default 30s; `0` disables. */
+export function triggerTickIntervalMsFromEnv(env = process.env) {
+    const raw = env.STAGEFLOW_TRIGGER_TICK_INTERVAL_MS;
+    if (raw === undefined || raw.trim() === "")
+        return DEFAULT_TRIGGER_TICK_INTERVAL_MS;
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed) || parsed < 0)
+        return DEFAULT_TRIGGER_TICK_INTERVAL_MS;
+    return parsed;
+}
+function computeNextRunAt(schedule, from) {
+    const job = new Cron(schedule.cron, schedule.timezone !== undefined ? { timezone: schedule.timezone } : {});
+    return job.nextRun(from);
+}
+/**
+ * `TriggerSourcePort` adapter for `schedule`-kind triggers. Ticks on an
+ * interval; each tick re-scans the catalog for enabled `schedule` triggers,
+ * seeds `next_run_at` for any that don't have one yet (without firing), and
+ * fires any trigger whose `next_run_at` has passed — always recomputing the
+ * next fire time from *now*, never from the stale `next_run_at`, so a tick
+ * never backfills more than one missed occurrence.
+ */
+export class ScheduleSource {
+    store;
+    cwd;
+    intervalMs;
+    now;
+    logError;
+    interval;
+    inFlight = false;
+    constructor(options) {
+        this.store = options.store;
+        this.cwd = options.cwd ?? process.cwd();
+        this.intervalMs = options.intervalMs ?? DEFAULT_TRIGGER_TICK_INTERVAL_MS;
+        this.now = options.now ?? (() => new Date());
+        this.logError =
+            options.logError ??
+                ((message) => {
+                    console.error(message);
+                });
+    }
+    async start(onFire) {
+        if (this.intervalMs <= 0)
+            return;
+        this.interval = setInterval(() => {
+            if (this.inFlight)
+                return;
+            this.inFlight = true;
+            void this.tick(onFire)
+                .catch((err) => {
+                this.logError(`trigger schedule tick failed: ${err instanceof Error ? err.message : String(err)}`);
+            })
+                .finally(() => {
+                this.inFlight = false;
+            });
+        }, this.intervalMs).unref();
+    }
+    async stop() {
+        if (this.interval !== undefined)
+            clearInterval(this.interval);
+        this.interval = undefined;
+    }
+    /**
+     * One discovery + seed + fire pass. Public so tests can drive it directly
+     * against an injected clock instead of waiting on real interval timers.
+     */
+    async tick(onFire) {
+        const ctx = catalogContextFromStageflow(await resolveStageflowContext(this.cwd));
+        const scanPaths = await getCatalogScanPaths(ctx);
+        if (!scanPaths)
+            return;
+        const now = this.now();
+        const projectRoot = ctx.projectRoot ?? undefined;
+        for (const triggerPath of scanPaths.triggerPaths) {
+            const outcome = await loadTriggerOutcome(triggerPath);
+            if (!outcome.ok)
+                continue;
+            const definition = outcome.value;
+            if (definition.kind !== "schedule" || !definition.enabled || !definition.schedule) {
+                continue;
+            }
+            let next;
+            try {
+                next = computeNextRunAt(definition.schedule, now);
+            }
+            catch (err) {
+                this.logError(`invalid schedule for trigger "${definition.id}": ${err instanceof Error ? err.message : String(err)}`);
+                continue;
+            }
+            const definitionRef = projectRoot !== undefined
+                ? path.relative(projectRoot, triggerPath).replace(/\\/g, "/")
+                : triggerPath;
+            await this.store.upsertTrigger({
+                id: definition.id,
+                definitionRef,
+                enabled: true,
+            });
+            const record = await this.store.getTrigger(definition.id);
+            if (!record?.next_run_at) {
+                if (next)
+                    await this.store.setTriggerNextRun(definition.id, next.toISOString());
+                continue;
+            }
+            const dueAt = new Date(record.next_run_at);
+            if (dueAt.getTime() <= now.getTime()) {
+                await onFire({ triggerId: definition.id });
+                if (next)
+                    await this.store.setTriggerNextRun(definition.id, next.toISOString());
+            }
+        }
+    }
+}

package/dist/runtime/settingsFile.d.ts

@@ -2,17 +2,20 @@ import type { ProjectContext } from "../project/resolveProjectContext.js";
 import type { StageflowContext } from "../project/resolveStageflowContext.js";
 export declare const SETTINGS_FILE_NAME = "settings.json";
 export declare const INVALID_SLOT_COUNT_MESSAGE = "maxConcurrent must be an integer >= 1";
-export declare const INVALID_CREDENTIAL_SOURCE_MESSAGE = "credentialSource must be \"pi_home\" or \"sf_owned\"";
-export type CredentialSource = "pi_home" | "sf_owned";
+export declare const INVALID_CREDENTIAL_SOURCE_MESSAGE = "credentialSource must be \"sf_owned\"";
+export declare const INVALID_WORKSHOP_MODEL_MESSAGE = "workshopModel must be a non-empty string";
+export type CredentialSource = "sf_owned";
 export type FactorySettings = {
     maxConcurrent?: number;
     credentialSource?: CredentialSource;
+    workshopModel?: string;
 };
 export declare function globalSettingsFilePath(): string;
 export declare function settingsFilePathForContext(ctx: ProjectContext | StageflowContext): string;
 export declare function settingsFilePath(cwd: string): string;
 export declare function parseSlotCount(value: unknown): number | undefined;
 export declare function parseCredentialSource(value: unknown): CredentialSource | undefined;
+export declare function parseWorkshopModelSetting(value: unknown): string | undefined;
 export declare function readCredentialSourceFromContext(ctx: ProjectContext): CredentialSource | undefined;
 export declare function readPersistedCredentialSourceFromContext(ctx: ProjectContext): CredentialSource | undefined;
 export declare function readFactorySettingsForContext(ctx: ProjectContext): FactorySettings;
@@ -25,6 +28,8 @@ export declare function writeMaxConcurrentToContext(ctx: ProjectContext, maxConc
 export declare function writeMaxConcurrentToFile(cwd: string, maxConcurrent: number): void;
 export declare function readMaxConcurrentFromGlobal(): number | undefined;
 export declare function writeMaxConcurrentToGlobal(maxConcurrent: number): void;
+export declare function readWorkshopModelFromFile(cwd: string): string | undefined;
+export declare function writeWorkshopModelToFile(cwd: string, workshopModel: string): void;
 export declare function readCredentialSourceFromFile(cwd: string): CredentialSource | undefined;
 export declare function writeCredentialSourceToContext(ctx: ProjectContext, credentialSource: CredentialSource): void;
 export declare function writeCredentialSourceToFile(cwd: string, credentialSource: CredentialSource): void;