stageflow 0.1.0

package/dist/cli.js

@@ -0,0 +1,273 @@
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { PiAgentAdapter } from "./agent/piAdapter.js";
+import { PROVIDERS_USAGE, runProvidersCommand } from "./cli/providersCommand.js";
+import { VALIDATE_USAGE, runValidateCommand } from "./cli/validateCommand.js";
+import { exitCodeForValidation, formatValidationHuman, } from "./cli/validateOutput.js";
+import { createRunStore } from "./runstore/createStore.js";
+import { PipelineValidationError } from "./runtime/pipelineRunner.js";
+import { RunManager } from "./runtime/runManager.js";
+import { readStageExecutionMode } from "./runtime/stageConcurrency.js";
+import { exitForOutcome, runStageWorker } from "./runtime/stageWorker.js";
+import { SF_STAGE_WORKER } from "./runtime/stageWorkerProtocol.js";
+import { DEFAULT_PORT, startUiServer } from "./server/http.js";
+const USAGE = `Usage:
+  sf run --task <path> --pipeline <name-or-path> [--checkout <path>]
+  sf validate [--pipeline <name-or-path>] [--strict] [--json]
+  sf ui [--port ${DEFAULT_PORT}]
+  sf providers list
+  sf providers status [--provider <id>]
+  sf providers detect
+  sf providers source [get | set <pi_home|sf_owned>]
+  sf providers login <providerId> [--type api_key|oauth] [--api-key-env <VAR>]
+  sf providers logout <providerId>
+  sf --help
+
+Stageflow (sf) runs YAML-defined automatic stage pipelines.
+
+Store backend: SF_STORE=sqlite only. SF_STORE=disk is rejected; disk-era .stageflow/runs trees import when the SQLite store is empty. Data under .stageflow/.
+
+${VALIDATE_USAGE}
+
+${PROVIDERS_USAGE}`;
+function parseArgs(argv) {
+    const args = argv.slice(2);
+    if (args.length === 0) {
+        return { help: true };
+    }
+    if ((args[0] === "--help" || args[0] === "-h") &&
+        args.length === 1) {
+        return { help: true };
+    }
+    const command = args[0];
+    if (command === "providers" || command === "validate") {
+        return { help: false, command };
+    }
+    if (args.includes("--help") || args.includes("-h")) {
+        return { help: true };
+    }
+    let task;
+    let pipeline;
+    let port;
+    let checkout;
+    for (let i = 1; i < args.length; i++) {
+        if (args[i] === "--task") {
+            task = args[++i];
+        }
+        else if (args[i] === "--pipeline") {
+            pipeline = args[++i];
+        }
+        else if (args[i] === "--checkout") {
+            const raw = args[++i];
+            if (raw === undefined || raw.length === 0) {
+                throw new Error("Missing value for --checkout");
+            }
+            checkout = raw;
+        }
+        else if (args[i] === "--port") {
+            const raw = args[++i];
+            port = Number(raw);
+            if (!Number.isFinite(port) || port <= 0) {
+                throw new Error(`Invalid --port: ${raw}`);
+            }
+        }
+    }
+    return { help: false, command, task, pipeline, port, checkout };
+}
+function parseRunStageArgs(argv) {
+    let runId;
+    let stageId;
+    let mode;
+    let resumeAnswer;
+    let attempt;
+    let sessionFilePath;
+    let operatorAgentDir;
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === "--run-id") {
+            runId = argv[++i];
+        }
+        else if (argv[i] === "--stage-id") {
+            stageId = argv[++i];
+        }
+        else if (argv[i] === "--mode") {
+            const raw = argv[++i];
+            if (raw !== "run" && raw !== "resume") {
+                throw new Error("--mode must be run or resume");
+            }
+            mode = raw;
+        }
+        else if (argv[i] === "--resume-answer") {
+            const raw = argv[++i];
+            if (raw === undefined) {
+                throw new Error("Missing value for --resume-answer");
+            }
+            try {
+                resumeAnswer = JSON.parse(raw);
+            }
+            catch {
+                throw new Error("--resume-answer must be valid JSON");
+            }
+        }
+        else if (argv[i] === "--attempt") {
+            const raw = argv[++i];
+            if (raw === undefined) {
+                throw new Error("Missing value for --attempt");
+            }
+            const parsed = Number.parseInt(raw, 10);
+            if (!Number.isFinite(parsed) || parsed < 1) {
+                throw new Error("--attempt must be a positive integer");
+            }
+            attempt = parsed;
+        }
+        else if (argv[i] === "--session-file") {
+            sessionFilePath = argv[++i];
+            if (sessionFilePath === undefined) {
+                throw new Error("Missing value for --session-file");
+            }
+        }
+        else if (argv[i] === "--operator-agent-dir") {
+            operatorAgentDir = argv[++i];
+            if (operatorAgentDir === undefined) {
+                throw new Error("Missing value for --operator-agent-dir");
+            }
+        }
+    }
+    if (!runId || !stageId) {
+        throw new Error("Missing --run-id and/or --stage-id");
+    }
+    return {
+        runId,
+        stageId,
+        mode,
+        resumeAnswer,
+        attempt,
+        sessionFilePath,
+        ...(operatorAgentDir !== undefined
+            ? { operatorCatalog: { cwd: process.cwd(), agentDir: operatorAgentDir } }
+            : {}),
+    };
+}
+async function handleInternalRunStage(argv) {
+    if (process.env[SF_STAGE_WORKER] !== "1") {
+        process.env[SF_STAGE_WORKER] = "1";
+    }
+    const parsed = parseRunStageArgs(argv);
+    const outcome = await runStageWorker({
+        runId: parsed.runId,
+        stageId: parsed.stageId,
+        rootDir: process.cwd(),
+        mode: parsed.mode ??
+            (parsed.resumeAnswer !== undefined ? "resume" : "run"),
+        resumeAnswer: parsed.resumeAnswer,
+        attempt: parsed.attempt,
+        sessionFilePath: parsed.sessionFilePath,
+        operatorCatalog: parsed.operatorCatalog,
+    });
+    exitForOutcome(outcome);
+}
+function openBrowser(url) {
+    const platform = process.platform;
+    if (platform === "darwin") {
+        spawn("open", [url], { detached: true, stdio: "ignore" }).unref();
+    }
+    else if (platform === "win32") {
+        spawn("cmd", ["/c", "start", "", url], { detached: true, stdio: "ignore" }).unref();
+    }
+    else {
+        spawn("xdg-open", [url], { detached: true, stdio: "ignore" }).unref();
+    }
+}
+async function main(argv) {
+    try {
+        const parsed = parseArgs(argv);
+        if (parsed.help) {
+            console.log(USAGE);
+            return argv.slice(2).length === 0 ? 1 : 0;
+        }
+        const rootDir = process.cwd();
+        if (parsed.command === "validate") {
+            return runValidateCommand(argv.slice(3), { cwd: rootDir });
+        }
+        if (parsed.command === "providers") {
+            return runProvidersCommand(argv.slice(3), rootDir);
+        }
+        const store = createRunStore({ rootDir });
+        if (parsed.command === "ui") {
+            const { url, mcpUrl } = await startUiServer({
+                agent: new PiAgentAdapter(),
+                store,
+                port: parsed.port,
+            });
+            console.log(`Operator console: ${url}`);
+            console.log(`MCP endpoint: ${mcpUrl}`);
+            openBrowser(url);
+            await new Promise(() => undefined);
+            return 0;
+        }
+        if (parsed.command === "internal") {
+            const subArgs = argv.slice(3);
+            const subcommand = subArgs[0];
+            if (subcommand !== "run-stage") {
+                console.error(`Unknown internal command: ${subcommand ?? "(none)"}`);
+                return 1;
+            }
+            return handleInternalRunStage(subArgs.slice(1));
+        }
+        if (parsed.command !== "run") {
+            console.error(`Unknown command: ${parsed.command}`);
+            console.error(USAGE);
+            return 1;
+        }
+        if (!parsed.task || !parsed.pipeline) {
+            console.error("Missing --task and/or --pipeline");
+            console.error(USAGE);
+            return 1;
+        }
+        const manager = new RunManager({
+            agent: new PiAgentAdapter(),
+            store,
+            cwd: rootDir,
+            operatorCatalog: { cwd: rootDir, agentDir: getAgentDir() },
+            executionMode: readStageExecutionMode(process.env, "process"),
+        });
+        const started = await manager.startRun({
+            task: parsed.task,
+            pipeline: parsed.pipeline,
+            checkoutOverride: parsed.checkout,
+        });
+        if (!started.ok) {
+            const code = started.code ?? "error";
+            console.error(`${code}: ${started.reason}`);
+            if (started.conflictingRunId !== undefined) {
+                console.error(`conflictingRunId: ${started.conflictingRunId}`);
+            }
+            if (started.conflictingCheckout !== undefined) {
+                console.error(`conflictingCheckout: ${started.conflictingCheckout}`);
+            }
+            return started.status ?? 1;
+        }
+        const result = await started.done;
+        if (result.ok) {
+            console.log(`Pipeline succeeded. Run folder: ${result.runDir}`);
+            return 0;
+        }
+        console.error(`Pipeline failed: ${result.reason}`);
+        console.error(`Run folder: ${result.runDir}`);
+        return 1;
+    }
+    catch (err) {
+        if (err instanceof PipelineValidationError) {
+            console.error(formatValidationHuman(err.result));
+            return exitCodeForValidation(err.result);
+        }
+        console.error(err instanceof Error ? err.message : String(err));
+        return 1;
+    }
+}
+main(process.argv)
+    .then((code) => process.exit(code))
+    .catch((err) => {
+    console.error(err instanceof Error ? err.message : String(err));
+    process.exit(1);
+});

package/dist/config/createPipeline.d.ts

@@ -0,0 +1,31 @@
+import type { PipelineListing } from "./listConfig.js";
+export type CreatePipelineStageRef = {
+    id: string;
+    needs?: string;
+};
+export type CreatePipelineInput = {
+    id: string;
+    stages: CreatePipelineStageRef[] | string[];
+};
+export type CreatePipelineParseError = {
+    ok: false;
+    status: 400;
+    error: string;
+};
+export type CreatePipelineResult = {
+    ok: true;
+    pipeline: PipelineListing;
+} | {
+    ok: false;
+    status: 400 | 409 | 422 | 500;
+    error: string;
+};
+export declare function normalizeCreatePipelineStages(stages: CreatePipelineStageRef[] | string[]): CreatePipelineStageRef[];
+export declare function parseCreatePipelineBody(body: unknown): CreatePipelineInput | CreatePipelineParseError;
+export declare function pipelineConfigToYaml(pipeline: {
+    id: string;
+    stages: CreatePipelineStageRef[];
+}, options: {
+    format: "linear" | "dag";
+}): string;
+export declare function createPipeline(cwd: string, input: CreatePipelineInput): Promise<CreatePipelineResult>;

package/dist/config/createPipeline.js

@@ -0,0 +1,255 @@
+import { access, mkdir, readdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { STAGE_ID_PATTERN } from "./createStage.js";
+import { loadPipeline } from "./loadPipeline.js";
+import { loadStage } from "./loadStage.js";
+import { readYamlObject } from "./readYamlObject.js";
+import { resolvePipelineDag } from "./resolvePipelineDag.js";
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function validatePipelineId(id) {
+    if (id.length === 0 || id.length > 64) {
+        return "id must be 1-64 characters";
+    }
+    if (!STAGE_ID_PATTERN.test(id)) {
+        return "id must be lowercase kebab-case";
+    }
+    return null;
+}
+export function normalizeCreatePipelineStages(stages) {
+    if (stages.length > 0 && typeof stages[0] === "string") {
+        return stages.map((id) => ({ id }));
+    }
+    return stages;
+}
+function parseStageEntry(entry, index) {
+    if (!isPlainObject(entry)) {
+        return {
+            ok: false,
+            status: 400,
+            error: `stages[${index}] must be a string or object with id`,
+        };
+    }
+    if (typeof entry.id !== "string") {
+        return { ok: false, status: 400, error: `stages[${index}].id is required` };
+    }
+    const ref = { id: entry.id };
+    if (entry.needs !== undefined) {
+        if (typeof entry.needs !== "string") {
+            return {
+                ok: false,
+                status: 400,
+                error: `stages[${index}].needs must be a string`,
+            };
+        }
+        ref.needs = entry.needs;
+    }
+    return ref;
+}
+export function parseCreatePipelineBody(body) {
+    if (!isPlainObject(body)) {
+        return { ok: false, status: 400, error: "Request body must be an object" };
+    }
+    if (typeof body.id !== "string") {
+        return { ok: false, status: 400, error: "id is required" };
+    }
+    const idError = validatePipelineId(body.id);
+    if (idError) {
+        return { ok: false, status: 400, error: idError };
+    }
+    if (!Array.isArray(body.stages)) {
+        return { ok: false, status: 400, error: "stages must be an array" };
+    }
+    if (body.stages.length === 0) {
+        return { ok: false, status: 400, error: "stages must be a non-empty array" };
+    }
+    let hasString = false;
+    let hasObject = false;
+    for (const entry of body.stages) {
+        if (typeof entry === "string") {
+            hasString = true;
+        }
+        else if (isPlainObject(entry)) {
+            hasObject = true;
+        }
+        else {
+            return { ok: false, status: 400, error: "stages must be an array of strings" };
+        }
+    }
+    if (hasString && hasObject) {
+        return {
+            ok: false,
+            status: 400,
+            error: "stages must be either all strings or all objects",
+        };
+    }
+    if (hasString) {
+        return {
+            id: body.id,
+            stages: body.stages,
+        };
+    }
+    const refs = [];
+    for (let index = 0; index < body.stages.length; index++) {
+        const parsed = parseStageEntry(body.stages[index], index);
+        if ("ok" in parsed) {
+            return parsed;
+        }
+        refs.push(parsed);
+    }
+    return {
+        id: body.id,
+        stages: refs,
+    };
+}
+export function pipelineConfigToYaml(pipeline, options) {
+    const lines = [`id: ${pipeline.id}`, "stages:"];
+    if (options.format === "linear") {
+        for (const stage of pipeline.stages) {
+            lines.push(`  - ${stage.id}`);
+        }
+    }
+    else {
+        for (const stage of pipeline.stages) {
+            lines.push(`  - id: ${stage.id}`);
+            if (stage.needs) {
+                lines.push(`    needs: ${stage.needs}`);
+            }
+        }
+    }
+    lines.push("");
+    return lines.join("\n");
+}
+async function fileExists(filePath) {
+    try {
+        await access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function findPipelineIdCollision(cwd, id, targetPath) {
+    if (await fileExists(targetPath)) {
+        return path.relative(cwd, targetPath);
+    }
+    const dir = path.join(cwd, "pipelines");
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return null;
+    }
+    for (const entry of entries) {
+        if (!entry.isFile() || !entry.name.endsWith(".yaml"))
+            continue;
+        const filePath = path.join(dir, entry.name);
+        if (filePath === targetPath)
+            continue;
+        try {
+            const raw = await readYamlObject(filePath);
+            if (raw?.id === id) {
+                return path.relative(cwd, filePath);
+            }
+        }
+        catch {
+            // ignore unreadable files for collision scan
+        }
+    }
+    return null;
+}
+async function validateStageReferences(cwd, stages) {
+    const seen = new Set();
+    for (const stage of stages) {
+        if (seen.has(stage.id)) {
+            return {
+                ok: false,
+                status: 422,
+                error: "Pipeline stages must not contain duplicates",
+            };
+        }
+        seen.add(stage.id);
+    }
+    const stagesDir = path.join(cwd, "stages");
+    for (const stage of stages) {
+        const stagePath = path.join(stagesDir, `${stage.id}.yaml`);
+        try {
+            await loadStage(stagePath);
+        }
+        catch {
+            return {
+                ok: false,
+                status: 422,
+                error: "one or more selected Stages no longer exist",
+            };
+        }
+    }
+    return { ok: true };
+}
+function buildResolverInput(input, normalized) {
+    if (input.stages.length > 0 && typeof input.stages[0] === "string") {
+        return input.stages;
+    }
+    return normalized.map((ref) => ref.needs ? { id: ref.id, needs: ref.needs } : { id: ref.id });
+}
+function usesDagYaml(input, normalized) {
+    if (input.stages.length > 0 && typeof input.stages[0] !== "string") {
+        return true;
+    }
+    return normalized.some((ref) => ref.needs !== undefined);
+}
+export async function createPipeline(cwd, input) {
+    const normalized = normalizeCreatePipelineStages(input.stages);
+    const stageValidation = await validateStageReferences(cwd, normalized);
+    if (!stageValidation.ok) {
+        return stageValidation;
+    }
+    const pipelinesDir = path.join(cwd, "pipelines");
+    const filePath = path.join(pipelinesDir, `${input.id}.yaml`);
+    const relPath = path.relative(cwd, filePath);
+    const collision = await findPipelineIdCollision(cwd, input.id, filePath);
+    if (collision) {
+        return {
+            ok: false,
+            status: 409,
+            error: `Pipeline id already exists (${collision})`,
+        };
+    }
+    const resolverInput = buildResolverInput(input, normalized);
+    try {
+        resolvePipelineDag(resolverInput, { pipelineId: input.id, path: relPath });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 422, error: message };
+    }
+    const yamlFormat = usesDagYaml(input, normalized) ? "dag" : "linear";
+    try {
+        await mkdir(pipelinesDir, { recursive: true });
+        await writeFile(filePath, pipelineConfigToYaml({ id: input.id, stages: normalized }, { format: yamlFormat }), "utf8");
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 500, error: message };
+    }
+    try {
+        const loaded = await loadPipeline(filePath, { cwd });
+        return {
+            ok: true,
+            pipeline: {
+                path: relPath,
+                id: loaded.pipeline.id,
+                stages: loaded.stages.map((stage) => ({
+                    id: stage.id,
+                    ...(stage.gate_kinds ? { gate_kinds: stage.gate_kinds } : {}),
+                })),
+            },
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return { ok: false, status: 500, error: message };
+    }
+}

package/dist/config/createStage.d.ts

@@ -0,0 +1,25 @@
+import { type StageGateKind } from "../types/stage.js";
+import type { ValidStageListing } from "./listConfig.js";
+export declare const STAGE_ID_PATTERN: RegExp;
+export type CreateStageInput = {
+    id: string;
+    system_prompt: string;
+    model: string;
+    gate_kinds?: StageGateKind[];
+};
+export type CreateStageParseError = {
+    ok: false;
+    status: 400;
+    error: string;
+};
+export type CreateStageResult = {
+    ok: true;
+    stage: ValidStageListing;
+} | {
+    ok: false;
+    status: 400 | 409 | 500;
+    error: string;
+};
+export declare function parseCreateStageBody(body: unknown): CreateStageInput | CreateStageParseError;
+export declare function stageConfigToYaml(stage: CreateStageInput): string;
+export declare function createStage(cwd: string, input: CreateStageInput): Promise<CreateStageResult>;