stageflow 0.25.0 → 0.27.0

package/dist/runstore/port.js

@@ -6,6 +6,8 @@ export function stageStatusFromEvents(events) {
             status = "running";
         if (ev.event === "waiting_for_input")
             status = "waiting_for_input";
+        if (ev.event === "interrupted")
+            status = "interrupted";
         if (ev.event === "succeeded")
             status = "succeeded";
         if (ev.event === "failed")
@@ -54,7 +56,11 @@ function stageResolvedForSuccess(stage, dag, byId) {
     }
     return false;
 }
-export function deriveStatusFromStages(stages, dag) {
+const OPERATOR_TERMINAL_STATUSES = new Set(["cancelled", "queued"]);
+export function deriveStatusFromStages(stages, dag, currentStatus) {
+    if (currentStatus !== undefined && OPERATOR_TERMINAL_STATUSES.has(currentStatus)) {
+        return currentStatus;
+    }
     if (stages.length === 0)
         return "created";
     if (findUnhandledFailedStage(stages, dag))
@@ -65,6 +71,7 @@ export function deriveStatusFromStages(stages, dag) {
     }
     if (stages.some((s) => s.status === "running" ||
         s.status === "waiting_for_input" ||
+        s.status === "interrupted" ||
         s.status === "succeeded")) {
         return "running";
     }

package/dist/runstore/restore.d.ts

@@ -0,0 +1,81 @@
+import { type ServiceProbeResult } from "../server/ensureGlobalService.js";
+export type RestoreErrorCode = "restore_host_live" | "restore_store_busy" | "restore_verify_failed" | "restore_schema_too_new" | "restore_version_mismatch" | "restore_unsafe_member" | "restore_failed" | "restore_not_found";
+export declare class RestoreError extends Error {
+    readonly code: RestoreErrorCode;
+    readonly detail?: Record<string, unknown>;
+    constructor(message: string, code: RestoreErrorCode, detail?: Record<string, unknown>);
+}
+export type RestoreManifest = {
+    created_at?: string;
+    stageflow_version?: string;
+    schema_version?: number;
+    contents?: string[];
+    credentials_included?: boolean;
+    a2a_artifacts_included?: boolean;
+    fingerprint_skew?: boolean;
+};
+export type VerifiedRestoreArchive = {
+    stagingDir: string;
+    dbPath: string;
+    manifest: RestoreManifest | null;
+    schemaVersion: number;
+    missingA2aArtifactRows: number;
+};
+export declare function restorePendingDir(homeDir?: string): string;
+export declare function restoreMarkerPath(homeDir?: string): string;
+export declare function restoreAppliedPath(homeDir?: string): string;
+export declare function restoreFailedPath(homeDir?: string): string;
+export type RestoreMarker = {
+    archive: string;
+    attempts: number;
+    staged_at: string;
+    drain_deadline?: string;
+};
+export declare function extractVerifiedArchive(archivePath: string): Promise<VerifiedRestoreArchive>;
+export declare function assertHostDownForRestore(options?: {
+    probe?: () => Promise<ServiceProbeResult>;
+}): Promise<void>;
+/**
+ * Open an exclusive lock on state.db with busy_timeout=0 and hold it through
+ * the provided callback (used to relocate the DB triplet safely).
+ */
+export declare function withExclusiveStoreLock<T>(homeDir: string, fn: () => T): T;
+export type ApplyRestoreResult = {
+    schema_version: number;
+    missing_a2a_artifact_rows: number;
+    pre_restore_stamp: string;
+};
+export declare function applyRestoreArchive(options: {
+    archivePath: string;
+    homeDir?: string;
+    force?: boolean;
+    restoreCredentials?: boolean;
+    probe?: () => Promise<ServiceProbeResult>;
+}): Promise<ApplyRestoreResult>;
+export declare function stageRestoreForBoot(options: {
+    archivePath: string;
+    homeDir?: string;
+    graceMs?: number;
+}): Promise<{
+    marker: RestoreMarker;
+    stagedPath: string;
+}>;
+export declare function resolveBackupNameForRestore(name: string, homeDir?: string): Promise<string>;
+export type BootRestoreOutcome = {
+    status: "none";
+} | {
+    status: "applied";
+    result: ApplyRestoreResult;
+} | {
+    status: "failed";
+    reason: string;
+    attempts: number;
+} | {
+    status: "blocked";
+    reason: string;
+};
+/**
+ * Apply a staged restore marker before opening the store for serving.
+ * Never re-applies restore.applied. Two failures → restore.failed (hard non-serve).
+ */
+export declare function applyPendingRestoreAtBoot(homeDir?: string): Promise<BootRestoreOutcome>;

package/dist/runstore/restore.js

@@ -0,0 +1,355 @@
+import { chmodSync, copyFileSync, createReadStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync, } from "node:fs";
+import { mkdtemp, rm } from "node:fs/promises";
+import { createGunzip } from "node:zlib";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { pipeline } from "node:stream/promises";
+import { createWriteStream } from "node:fs";
+import Database from "better-sqlite3";
+import { PACKAGE_VERSION } from "../package-meta.js";
+import { globalStageflowHome } from "../project/globalHome.js";
+import { CURRENT_SCHEMA_VERSION } from "./sqlite/migrations/index.js";
+import { StoreSchemaError } from "./sqlite/storeSchemaError.js";
+import { assertStoreQuickCheck } from "./sqlite/applyStorePragmas.js";
+import { verifyDbSnapshot } from "./backup.js";
+import { probeGlobalServiceDetailed, } from "../server/ensureGlobalService.js";
+export class RestoreError extends Error {
+    code;
+    detail;
+    constructor(message, code, detail) {
+        super(message);
+        this.name = "RestoreError";
+        this.code = code;
+        if (detail !== undefined)
+            this.detail = detail;
+    }
+}
+const ALLOWED_MEMBERS = new Set([
+    "state.db",
+    "settings.json",
+    "agent/auth.json",
+    "manifest.json",
+]);
+export function restorePendingDir(homeDir = globalStageflowHome()) {
+    return path.join(homeDir, "restore-pending");
+}
+export function restoreMarkerPath(homeDir = globalStageflowHome()) {
+    return path.join(restorePendingDir(homeDir), "marker.json");
+}
+export function restoreAppliedPath(homeDir = globalStageflowHome()) {
+    return path.join(homeDir, "restore.applied");
+}
+export function restoreFailedPath(homeDir = globalStageflowHome()) {
+    return path.join(homeDir, "restore.failed");
+}
+function majorVersion(version) {
+    return version.split(".")[0] ?? version;
+}
+async function gunzipToFile(src, dest) {
+    await pipeline(createReadStream(src), createGunzip(), createWriteStream(dest));
+}
+function parseUstarMembers(tarPath, destDir) {
+    const buf = readFileSync(tarPath);
+    const written = [];
+    let offset = 0;
+    while (offset + 512 <= buf.length) {
+        const header = buf.subarray(offset, offset + 512);
+        offset += 512;
+        if (header.every((b) => b === 0))
+            break;
+        const name = header.subarray(0, 100).toString("utf8").replace(/\0.*$/, "");
+        const sizeOctal = header.subarray(124, 135).toString("utf8").replace(/\0| /g, "");
+        const size = Number.parseInt(sizeOctal, 8) || 0;
+        const type = header[156];
+        if (name.includes("..") || path.isAbsolute(name) || name.startsWith("/")) {
+            throw new RestoreError(`restore_unsafe_member: ${name}`, "restore_unsafe_member");
+        }
+        if (type === 0x32 /* symlink */ || type === 0x31 /* hard link */) {
+            throw new RestoreError(`restore_unsafe_member: link ${name}`, "restore_unsafe_member");
+        }
+        if (!ALLOWED_MEMBERS.has(name) && name !== "") {
+            throw new RestoreError(`restore_unsafe_member: ${name} not in allowlist`, "restore_unsafe_member");
+        }
+        const data = buf.subarray(offset, offset + size);
+        offset += size + ((512 - (size % 512)) % 512);
+        if (name === "")
+            continue;
+        const outPath = path.join(destDir, name);
+        mkdirSync(path.dirname(outPath), { recursive: true });
+        writeFileSync(outPath, data);
+        written.push(name);
+    }
+    return written;
+}
+export async function extractVerifiedArchive(archivePath) {
+    const stagingDir = await mkdtemp(path.join(tmpdir(), "sf-restore-"));
+    try {
+        let dbPath = archivePath;
+        let manifest = null;
+        if (archivePath.endsWith(".tar.gz") || archivePath.endsWith(".tgz")) {
+            const tarPath = path.join(stagingDir, "archive.tar");
+            await gunzipToFile(archivePath, tarPath);
+            parseUstarMembers(tarPath, stagingDir);
+            dbPath = path.join(stagingDir, "state.db");
+            const manifestFile = path.join(stagingDir, "manifest.json");
+            if (existsSync(manifestFile)) {
+                manifest = JSON.parse(readFileSync(manifestFile, "utf8"));
+            }
+        }
+        else {
+            const copied = path.join(stagingDir, "state.db");
+            copyFileSync(archivePath, copied);
+            dbPath = copied;
+        }
+        if (!existsSync(dbPath)) {
+            throw new RestoreError("restore_verify_failed: archive missing state.db", "restore_verify_failed");
+        }
+        const db = new Database(dbPath, { readonly: true, fileMustExist: true });
+        let schemaVersion;
+        try {
+            assertStoreQuickCheck(db);
+            schemaVersion = db.pragma("user_version", { simple: true });
+        }
+        finally {
+            db.close();
+        }
+        if (schemaVersion > CURRENT_SCHEMA_VERSION) {
+            throw new StoreSchemaError(`store_schema_too_new: on_disk=${schemaVersion} binary_max=${CURRENT_SCHEMA_VERSION}`, "store_schema_too_new");
+        }
+        verifyDbSnapshot(dbPath, schemaVersion);
+        let missingA2aArtifactRows = 0;
+        if (manifest && manifest.a2a_artifacts_included !== true) {
+            const probe = new Database(dbPath, { readonly: true, fileMustExist: true });
+            try {
+                const row = probe
+                    .prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type='table' AND name='a2a_artifacts'")
+                    .get();
+                if (row.c > 0) {
+                    const count = probe
+                        .prepare("SELECT COUNT(*) AS c FROM a2a_artifacts")
+                        .get();
+                    missingA2aArtifactRows = count.c;
+                }
+            }
+            catch {
+                missingA2aArtifactRows = 0;
+            }
+            finally {
+                probe.close();
+            }
+        }
+        return {
+            stagingDir,
+            dbPath,
+            manifest,
+            schemaVersion,
+            missingA2aArtifactRows,
+        };
+    }
+    catch (err) {
+        await rm(stagingDir, { recursive: true, force: true });
+        throw err;
+    }
+}
+export async function assertHostDownForRestore(options) {
+    const probe = options?.probe ?? probeGlobalServiceDetailed;
+    const result = await probe();
+    if (result === "up") {
+        throw new RestoreError("restore_host_live: Host is up (GET /livez). Stop it before restore.", "restore_host_live");
+    }
+}
+/**
+ * Open an exclusive lock on state.db with busy_timeout=0 and hold it through
+ * the provided callback (used to relocate the DB triplet safely).
+ */
+export function withExclusiveStoreLock(homeDir, fn) {
+    const dbPath = path.join(homeDir, "state.db");
+    if (!existsSync(dbPath)) {
+        return fn();
+    }
+    const lockDb = new Database(dbPath);
+    try {
+        lockDb.pragma("busy_timeout = 0");
+        try {
+            lockDb.prepare("BEGIN EXCLUSIVE").run();
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            if (/SQLITE_BUSY|database is locked/i.test(message)) {
+                throw new RestoreError("restore_store_busy: another process holds state.db", "restore_store_busy");
+            }
+            throw err;
+        }
+        return fn();
+    }
+    finally {
+        try {
+            lockDb.exec("ROLLBACK");
+        }
+        catch {
+            // ignore
+        }
+        lockDb.close();
+    }
+}
+function moveAsideTriplet(homeDir, stamp) {
+    for (const name of ["state.db", "state.db-wal", "state.db-shm"]) {
+        const from = path.join(homeDir, name);
+        if (!existsSync(from))
+            continue;
+        renameSync(from, `${from}.pre-restore-${stamp}`);
+    }
+}
+function assertNoSidecars(dbPath) {
+    for (const side of [`${dbPath}-wal`, `${dbPath}-shm`]) {
+        if (existsSync(side)) {
+            unlinkSync(side);
+        }
+    }
+    if (existsSync(`${dbPath}-wal`) || existsSync(`${dbPath}-shm`)) {
+        throw new RestoreError("restore_verify_failed: stale WAL/SHM sidecars remain", "restore_verify_failed");
+    }
+}
+export async function applyRestoreArchive(options) {
+    const homeDir = options.homeDir ?? globalStageflowHome();
+    await assertHostDownForRestore({ probe: options.probe });
+    const verified = await extractVerifiedArchive(options.archivePath);
+    try {
+        if (verified.manifest?.stageflow_version &&
+            majorVersion(verified.manifest.stageflow_version) !==
+                majorVersion(PACKAGE_VERSION) &&
+            options.force !== true) {
+            throw new RestoreError(`restore_version_mismatch: archive ${verified.manifest.stageflow_version} vs binary ${PACKAGE_VERSION} (pass --force)`, "restore_version_mismatch");
+        }
+        const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+        const restoreCredentials = options.restoreCredentials !== false;
+        withExclusiveStoreLock(homeDir, () => {
+            moveAsideTriplet(homeDir, stamp);
+            const destDb = path.join(homeDir, "state.db");
+            copyFileSync(verified.dbPath, destDb);
+            assertNoSidecars(destDb);
+            if (restoreCredentials) {
+                const settingsSrc = path.join(verified.stagingDir, "settings.json");
+                const authSrc = path.join(verified.stagingDir, "agent", "auth.json");
+                if (existsSync(settingsSrc)) {
+                    copyFileSync(settingsSrc, path.join(homeDir, "settings.json"));
+                }
+                if (existsSync(authSrc)) {
+                    const agentDir = path.join(homeDir, "agent");
+                    mkdirSync(agentDir, { recursive: true });
+                    try {
+                        chmodSync(agentDir, 0o700);
+                    }
+                    catch {
+                        // best-effort
+                    }
+                    const authDest = path.join(agentDir, "auth.json");
+                    copyFileSync(authSrc, authDest);
+                    try {
+                        chmodSync(authDest, 0o600);
+                    }
+                    catch {
+                        // best-effort
+                    }
+                }
+            }
+        });
+        const check = new Database(path.join(homeDir, "state.db"));
+        try {
+            assertStoreQuickCheck(check);
+        }
+        finally {
+            check.close();
+        }
+        return {
+            schema_version: verified.schemaVersion,
+            missing_a2a_artifact_rows: verified.missingA2aArtifactRows,
+            pre_restore_stamp: stamp,
+        };
+    }
+    finally {
+        await rm(verified.stagingDir, { recursive: true, force: true });
+    }
+}
+export async function stageRestoreForBoot(options) {
+    const homeDir = options.homeDir ?? globalStageflowHome();
+    if (existsSync(restoreFailedPath(homeDir))) {
+        throw new RestoreError("restore_failed: clear restore.failed before staging another restore", "restore_failed");
+    }
+    const verified = await extractVerifiedArchive(options.archivePath);
+    await rm(verified.stagingDir, { recursive: true, force: true });
+    if (verified.manifest?.stageflow_version &&
+        majorVersion(verified.manifest.stageflow_version) !==
+            majorVersion(PACKAGE_VERSION)) {
+        throw new RestoreError(`restore_version_mismatch: archive ${verified.manifest.stageflow_version} vs binary ${PACKAGE_VERSION}`, "restore_version_mismatch");
+    }
+    const pending = restorePendingDir(homeDir);
+    mkdirSync(pending, { recursive: true });
+    const basename = `staged-${Date.now()}${path.extname(options.archivePath) || ".tar.gz"}`;
+    const stagedPath = path.join(pending, basename);
+    copyFileSync(options.archivePath, stagedPath);
+    const graceMs = options.graceMs ?? 8000;
+    const marker = {
+        archive: basename,
+        attempts: 0,
+        staged_at: new Date().toISOString(),
+        drain_deadline: new Date(Date.now() + graceMs).toISOString(),
+    };
+    writeFileSync(restoreMarkerPath(homeDir), `${JSON.stringify(marker, null, 2)}\n`);
+    return { marker, stagedPath };
+}
+export async function resolveBackupNameForRestore(name, homeDir = globalStageflowHome()) {
+    const { resolveBackupDownloadPath } = await import("./backup.js");
+    return resolveBackupDownloadPath(name, homeDir);
+}
+/**
+ * Apply a staged restore marker before opening the store for serving.
+ * Never re-applies restore.applied. Two failures → restore.failed (hard non-serve).
+ */
+export async function applyPendingRestoreAtBoot(homeDir = globalStageflowHome()) {
+    if (existsSync(restoreFailedPath(homeDir))) {
+        const raw = readFileSync(restoreFailedPath(homeDir), "utf8");
+        let reason = "restore.failed present";
+        try {
+            reason = JSON.parse(raw).reason ?? reason;
+        }
+        catch {
+            // keep default
+        }
+        return { status: "blocked", reason };
+    }
+    if (existsSync(restoreAppliedPath(homeDir))) {
+        return { status: "none" };
+    }
+    const markerFile = restoreMarkerPath(homeDir);
+    if (!existsSync(markerFile)) {
+        return { status: "none" };
+    }
+    const marker = JSON.parse(readFileSync(markerFile, "utf8"));
+    const archivePath = path.join(restorePendingDir(homeDir), marker.archive);
+    marker.attempts += 1;
+    writeFileSync(markerFile, `${JSON.stringify(marker, null, 2)}\n`);
+    try {
+        if (!existsSync(archivePath)) {
+            throw new RestoreError(`restore_not_found: staged archive missing (${marker.archive})`, "restore_not_found");
+        }
+        const result = await applyRestoreArchive({
+            archivePath,
+            homeDir,
+            force: true,
+            probe: async () => "unreachable",
+        });
+        writeFileSync(restoreAppliedPath(homeDir), `${JSON.stringify({ ...result, applied_at: new Date().toISOString() }, null, 2)}\n`);
+        rmSync(markerFile, { force: true });
+        return { status: "applied", result };
+    }
+    catch (err) {
+        const reason = err instanceof Error ? err.message : String(err);
+        if (marker.attempts >= 2) {
+            writeFileSync(restoreFailedPath(homeDir), `${JSON.stringify({ reason, attempts: marker.attempts, at: new Date().toISOString() }, null, 2)}\n`);
+            rmSync(markerFile, { force: true });
+            return { status: "failed", reason, attempts: marker.attempts };
+        }
+        return { status: "failed", reason, attempts: marker.attempts };
+    }
+}

package/dist/runstore/retention.d.ts

@@ -0,0 +1,27 @@
+import type { RunMeta, RunStatus } from "./port.js";
+export type RetentionDecision = "none" | "slim" | "purge";
+export type StatusRetentionWindow = {
+    slimMs: number;
+    purgeMs: number;
+};
+export type RetentionWindows = {
+    succeeded: StatusRetentionWindow;
+    failed: StatusRetentionWindow;
+    cancelled: StatusRetentionWindow;
+};
+export declare const DAY_MS: number;
+/** Defaults: succeeded 3d/30d; failed and cancelled 30d/90d (R30 / AE6). */
+export declare const DEFAULT_RETENTION_WINDOWS: RetentionWindows;
+export declare const DEFAULT_BARE_CACHE_TTL_MS: number;
+export declare const DEFAULT_SLIM_ARTIFACT_MAX_BYTES: number;
+export type RetentionRow = Pick<RunMeta, "status" | "finished_at" | "slimmed_at">;
+/** Resolve retention windows from KD1 defaults plus per-status env overrides. */
+export declare function retentionWindowsFromEnv(env?: NodeJS.ProcessEnv): RetentionWindows;
+export declare function bareCacheTtlFromEnv(env?: NodeJS.ProcessEnv): number;
+export declare function slimArtifactMaxBytesFromEnv(env?: NodeJS.ProcessEnv): number;
+/**
+ * Clock-injected eligibility. Reads `finished_at` only (KTD1), never `updated_at`.
+ * Already-slimmed rows past SLIM but not PURGE return `"none"` (R18).
+ */
+export declare function retentionDecision(row: RetentionRow, now: Date, windows?: RetentionWindows): RetentionDecision;
+export declare function isTerminalRunStatus(status: RunStatus | undefined): status is "succeeded" | "failed" | "cancelled";

package/dist/runstore/retention.js

@@ -0,0 +1,74 @@
+export const DAY_MS = 24 * 60 * 60 * 1000;
+/** Defaults: succeeded 3d/30d; failed and cancelled 30d/90d (R30 / AE6). */
+export const DEFAULT_RETENTION_WINDOWS = {
+    succeeded: { slimMs: 3 * DAY_MS, purgeMs: 30 * DAY_MS },
+    cancelled: { slimMs: 30 * DAY_MS, purgeMs: 90 * DAY_MS },
+    failed: { slimMs: 30 * DAY_MS, purgeMs: 90 * DAY_MS },
+};
+export const DEFAULT_BARE_CACHE_TTL_MS = 30 * DAY_MS;
+export const DEFAULT_SLIM_ARTIFACT_MAX_BYTES = 1024 * 1024;
+const TERMINAL = new Set([
+    "succeeded",
+    "failed",
+    "cancelled",
+]);
+function parsePositiveMs(raw) {
+    if (raw === undefined)
+        return undefined;
+    const trimmed = raw.trim();
+    if (trimmed === "")
+        return undefined;
+    const n = Number.parseInt(trimmed, 10);
+    if (!Number.isFinite(n) || n < 0)
+        return undefined;
+    return n;
+}
+function windowFromEnv(env, status, defaults) {
+    const statusKey = status.toUpperCase();
+    return {
+        slimMs: parsePositiveMs(env[`STAGEFLOW_SLIM_${statusKey}_MS`]) ?? defaults.slimMs,
+        purgeMs: parsePositiveMs(env[`STAGEFLOW_PURGE_${statusKey}_MS`]) ??
+            defaults.purgeMs,
+    };
+}
+/** Resolve retention windows from KD1 defaults plus per-status env overrides. */
+export function retentionWindowsFromEnv(env = process.env) {
+    return {
+        succeeded: windowFromEnv(env, "succeeded", DEFAULT_RETENTION_WINDOWS.succeeded),
+        failed: windowFromEnv(env, "failed", DEFAULT_RETENTION_WINDOWS.failed),
+        cancelled: windowFromEnv(env, "cancelled", DEFAULT_RETENTION_WINDOWS.cancelled),
+    };
+}
+export function bareCacheTtlFromEnv(env = process.env) {
+    return (parsePositiveMs(env.STAGEFLOW_BARE_CACHE_TTL_MS) ?? DEFAULT_BARE_CACHE_TTL_MS);
+}
+export function slimArtifactMaxBytesFromEnv(env = process.env) {
+    return (parsePositiveMs(env.STAGEFLOW_SLIM_ARTIFACT_MAX_BYTES) ??
+        DEFAULT_SLIM_ARTIFACT_MAX_BYTES);
+}
+/**
+ * Clock-injected eligibility. Reads `finished_at` only (KTD1), never `updated_at`.
+ * Already-slimmed rows past SLIM but not PURGE return `"none"` (R18).
+ */
+export function retentionDecision(row, now, windows = DEFAULT_RETENTION_WINDOWS) {
+    const status = row.status;
+    if (status === undefined || !TERMINAL.has(status))
+        return "none";
+    if (row.finished_at === undefined || row.finished_at === "")
+        return "none";
+    const finishedMs = Date.parse(row.finished_at);
+    if (!Number.isFinite(finishedMs))
+        return "none";
+    const ageMs = now.getTime() - finishedMs;
+    if (ageMs < 0)
+        return "none";
+    const window = windows[status];
+    if (ageMs >= window.purgeMs)
+        return "purge";
+    if (ageMs >= window.slimMs && row.slimmed_at === undefined)
+        return "slim";
+    return "none";
+}
+export function isTerminalRunStatus(status) {
+    return status !== undefined && TERMINAL.has(status);
+}