stageflow 0.1.0 → 0.3.0

package/dist/config/loadPipeline.js

@@ -1,68 +1,38 @@
 import { access } from "node:fs/promises";
 import path from "node:path";
 import { loadFailure, loadSuccess } from "./loadOutcome.js";
-import { loadStageOutcome } from "./loadStage.js";
-import { readYamlObject } from "./readYamlObject.js";
-import { extractPipelineStageIds, resolvePipelineDag } from "./resolvePipelineDag.js";
-import { findingStageIdFilenameMismatch, findingsFromLoadIssues, } from "./validateCatalog.js";
-async function resolvePipelinePath(nameOrPath, cwd) {
-    const direct = path.resolve(cwd, nameOrPath);
+import { mergePipelineStages } from "./mergePipelineIncludes.js";
+import { normalizePipelineStageEntries, toWiringRefs, } from "./normalizePipelineStageEntry.js";
+import { loadStageFromObjectOutcome, loadStageOutcome } from "./loadStage.js";
+import { resolvePipelineDagFromRefs } from "./resolvePipelineDag.js";
+export async function resolvePipelinePath(pipelinePath, cwd) {
+    const resolved = path.normalize(path.resolve(cwd, pipelinePath));
     try {
-        await access(direct);
-        return direct;
+        await access(resolved);
+        return resolved;
     }
     catch {
-        const candidate = path.resolve(cwd, "pipelines", `${nameOrPath}.yaml`);
-        await access(candidate);
-        return candidate;
+        throw new Error(`Pipeline file not found: ${resolved}. Pass a filesystem path to --pipeline (e.g. pipelines/hello.pipeline.yaml), not a bare pipeline id.`);
     }
 }
-export async function loadPipelineOutcome(nameOrPath, options = {}) {
-    const cwd = options.cwd ?? process.cwd();
-    const stagesDir = options.stagesDir ?? path.join(cwd, "stages");
-    let pipelinePath;
-    try {
-        pipelinePath = await resolvePipelinePath(nameOrPath, cwd);
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        return loadFailure([
-            {
-                code: "pipeline.load_error",
-                message,
-                category: "pipeline",
-            },
-        ]);
-    }
-    let raw;
-    try {
-        raw = await readYamlObject(pipelinePath);
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        return loadFailure([
-            {
-                code: "pipeline.load_error",
-                message,
-                category: "pipeline",
-            },
-        ]);
-    }
-    if (typeof raw?.id !== "string" || !Array.isArray(raw.stages)) {
-        return loadFailure([
-            {
-                code: "pipeline.invalid_shape",
-                message: `Invalid pipeline ${pipelinePath}: id and stages[] are required`,
-                category: "pipeline",
-            },
-        ]);
-    }
-    const pipelineId = raw.id;
-    const ctx = { pipelineId: raw.id, path: pipelinePath };
+async function loadPipelineFromPath(pipelinePath) {
+    const normalizedPipelinePath = path.normalize(path.resolve(pipelinePath));
+    const mergeOutcome = await mergePipelineStages(normalizedPipelinePath);
+    if (!mergeOutcome.ok) {
+        return loadFailure(mergeOutcome.issues);
+    }
+    const { entries: rawEntries, pipelineId } = mergeOutcome.value;
+    const ctx = { pipelineId, path: normalizedPipelinePath };
+    const normalizeOutcome = normalizePipelineStageEntries(rawEntries, ctx);
+    if (!normalizeOutcome.ok) {
+        return loadFailure(normalizeOutcome.issues);
+    }
+    const normalizedEntries = normalizeOutcome.value;
+    const wiringRefs = toWiringRefs(normalizedEntries);
     let stageIds;
     let dag;
     try {
-        const resolved = resolvePipelineDag(raw.stages, ctx);
+        const resolved = resolvePipelineDagFromRefs(wiringRefs, ctx);
         stageIds = resolved.stages;
         dag = resolved.dag;
     }
@@ -77,115 +47,87 @@ export async function loadPipelineOutcome(nameOrPath, options = {}) {
             },
         ]);
     }
-    const pipeline = {
-        id: raw.id,
-        stages: stageIds,
-    };
+    const entryById = new Map(normalizedEntries.map((entry) => [entry.id, entry]));
+    const stageSources = {};
     const stages = [];
-    for (const stageId of pipeline.stages) {
-        const stagePath = path.join(stagesDir, `${stageId}.yaml`);
-        const stageOutcome = await loadStageOutcome(stagePath);
+    for (const stageId of stageIds) {
+        const entry = entryById.get(stageId);
+        if (!entry) {
+            return loadFailure([
+                {
+                    code: "pipeline.invalid_shape",
+                    message: `Pipeline ${pipelineId}: missing normalized entry for stage "${stageId}"`,
+                    category: "pipeline",
+                    pipelineId,
+                },
+            ]);
+        }
+        if (entry.body.kind === "inline") {
+            const inlineOutcome = loadStageFromObjectOutcome(entry.body.raw, {
+                entryId: entry.id,
+                declaringPath: entry.declaringPath,
+            });
+            if (!inlineOutcome.ok) {
+                return loadFailure(inlineOutcome.issues);
+            }
+            stages.push(inlineOutcome.value);
+            stageSources[stageId] = { kind: "inline" };
+            continue;
+        }
+        const stageOutcome = await loadStageOutcome(entry.body.absolutePath);
         if (!stageOutcome.ok) {
-            const message = `Pipeline ${pipeline.id} references missing stage "${stageId}" at ${stagePath}`;
             return loadFailure([
                 {
                     code: "pipeline.missing_stage",
-                    message,
+                    message: `Pipeline ${pipelineId} references missing stage "${stageId}" at ${entry.body.absolutePath}`,
                     category: "pipeline",
                     pipelineId,
                 },
+                ...stageOutcome.issues,
             ]);
         }
-        stages.push(stageOutcome.value);
-    }
-    return loadSuccess({ pipeline, stages, dag });
-}
-async function checkStageIdFilename(cwd, stagePath) {
-    const fileStem = path.basename(stagePath, ".yaml");
-    try {
-        const raw = await readYamlObject(stagePath);
-        if (typeof raw?.id === "string" && raw.id !== fileStem) {
-            return findingStageIdFilenameMismatch(cwd, stagePath, fileStem, raw.id);
+        if (stageOutcome.value.id !== entry.id) {
+            return loadFailure([
+                {
+                    code: "pipeline.stage_id_mismatch",
+                    message: `Pipeline ${pipelineId}: stage entry "${entry.id}" in ${entry.declaringPath} uses ${entry.body.path} which declares id "${stageOutcome.value.id}"`,
+                    category: "pipeline",
+                    pipelineId,
+                },
+            ]);
         }
+        stages.push(stageOutcome.value);
+        stageSources[stageId] = { kind: "file", path: entry.body.absolutePath };
     }
-    catch {
-        // loadStageOutcome will report read errors
-    }
-    return null;
-}
-async function validateStageFile(cwd, stagePath) {
-    const findings = [];
-    const mismatch = await checkStageIdFilename(cwd, stagePath);
-    if (mismatch)
-        findings.push(mismatch);
-    const outcome = await loadStageOutcome(stagePath);
-    if (!outcome.ok) {
-        findings.push(...findingsFromLoadIssues(cwd, stagePath, outcome.issues));
-        return findings;
-    }
-    if (!mismatch && path.basename(stagePath, ".yaml") !== outcome.value.id) {
-        findings.push(findingStageIdFilenameMismatch(cwd, stagePath, path.basename(stagePath, ".yaml"), outcome.value.id));
-    }
-    return findings;
+    const pipeline = {
+        id: pipelineId,
+        stages: stageIds,
+    };
+    return loadSuccess({
+        pipeline,
+        stages,
+        dag,
+        pipelinePath: normalizedPipelinePath,
+        stageSources,
+    });
 }
-export async function loadPipelineValidated(nameOrPath, options = {}) {
+export async function loadPipelineOutcome(nameOrPath, options = {}) {
     const cwd = options.cwd ?? process.cwd();
-    const stagesDir = options.stagesDir ?? path.join(cwd, "stages");
-    const validateStages = options.validateStages ?? true;
     let pipelinePath;
     try {
         pipelinePath = await resolvePipelinePath(nameOrPath, cwd);
     }
     catch (err) {
         const message = err instanceof Error ? err.message : String(err);
-        return {
-            ok: false,
-            findings: findingsFromLoadIssues(cwd, path.resolve(cwd, nameOrPath), [
-                {
-                    code: "pipeline.load_error",
-                    message,
-                    category: "pipeline",
-                },
-            ]),
-        };
-    }
-    const referencedStageIds = new Set();
-    try {
-        const raw = await readYamlObject(pipelinePath);
-        if (typeof raw?.id === "string" && Array.isArray(raw.stages)) {
-            const stageIds = extractPipelineStageIds(raw.stages);
-            if (stageIds) {
-                for (const stageId of stageIds) {
-                    referencedStageIds.add(stageId);
-                }
-            }
-        }
-    }
-    catch {
-        // loadPipelineOutcome will report read errors
-    }
-    const outcome = await loadPipelineOutcome(nameOrPath, { cwd, stagesDir });
-    const findings = [];
-    if (!outcome.ok) {
-        findings.push(...findingsFromLoadIssues(cwd, pipelinePath, outcome.issues));
-        if (validateStages) {
-            for (const stageId of referencedStageIds) {
-                const stagePath = path.join(stagesDir, `${stageId}.yaml`);
-                findings.push(...(await validateStageFile(cwd, stagePath)));
-            }
-        }
-        return { ok: false, findings };
-    }
-    if (validateStages) {
-        for (const stageId of outcome.value.pipeline.stages) {
-            const stagePath = path.join(stagesDir, `${stageId}.yaml`);
-            findings.push(...(await validateStageFile(cwd, stagePath)));
-        }
-    }
-    if (findings.some((finding) => finding.severity === "error")) {
-        return { ok: false, findings };
+        return loadFailure([
+            {
+                code: "pipeline.load_error",
+                message,
+                category: "pipeline",
+            },
+        ]);
     }
-    return { ok: true, loaded: outcome.value };
+    return loadPipelineFromPath(pipelinePath);
 }
 export async function loadPipeline(nameOrPath, options = {}) {
     const outcome = await loadPipelineOutcome(nameOrPath, options);

package/dist/config/loadStage.d.ts

@@ -1,4 +1,8 @@
 import { type StageConfig } from "../types/stage.js";
 import { type LoadOutcome } from "./loadOutcome.js";
+export declare function loadStageFromObjectOutcome(raw: Record<string, unknown>, ctx: {
+    entryId: string;
+    declaringPath: string;
+}): LoadOutcome<StageConfig>;
 export declare function loadStageOutcome(filePath: string): Promise<LoadOutcome<StageConfig>>;
 export declare function loadStage(filePath: string): Promise<StageConfig>;

package/dist/config/loadStage.js

@@ -5,14 +5,14 @@ import { readYamlObject } from "./readYamlObject.js";
 function isGateKind(value) {
     return STAGE_GATE_KINDS.includes(value);
 }
-function parseGateKinds(raw, filePath) {
+function parseGateKinds(raw, label) {
     if (raw === undefined)
         return loadSuccess(undefined);
     if (!Array.isArray(raw) || !raw.every((item) => typeof item === "string")) {
         return loadFailure([
             {
                 code: "stage.invalid_gate_kinds",
-                message: `Invalid stage file ${filePath}: gate_kinds must be an array of strings`,
+                message: `Invalid stage ${label}: gate_kinds must be an array of strings`,
                 category: "stage",
             },
         ]);
@@ -23,7 +23,7 @@ function parseGateKinds(raw, filePath) {
             return loadFailure([
                 {
                     code: "stage.invalid_gate_kinds",
-                    message: `Invalid stage file ${filePath}: unsupported gate kind "${item}" (allowed: ${STAGE_GATE_KINDS.join(", ")})`,
+                    message: `Invalid stage ${label}: unsupported gate kind "${item}" (allowed: ${STAGE_GATE_KINDS.join(", ")})`,
                     category: "stage",
                 },
             ]);
@@ -32,35 +32,20 @@ function parseGateKinds(raw, filePath) {
     }
     return loadSuccess(kinds.length > 0 ? kinds : undefined);
 }
-export async function loadStageOutcome(filePath) {
-    let raw;
-    try {
-        raw = await readYamlObject(filePath);
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
-        return loadFailure([
-            {
-                code: "stage.load_error",
-                message,
-                category: "stage",
-            },
-        ]);
-    }
-    if (typeof raw?.id !== "string" ||
-        typeof raw?.system_prompt !== "string" ||
-        typeof raw?.model !== "string") {
+function parseStageFields(raw, label, entryId) {
+    if (typeof raw.system_prompt !== "string" ||
+        typeof raw.model !== "string") {
         return loadFailure([
             {
                 code: "stage.invalid_shape",
-                message: `Invalid stage file ${filePath}: id, system_prompt, and model are required strings`,
+                message: `Invalid stage ${label}: system_prompt and model are required strings`,
                 category: "stage",
-                stageId: typeof raw?.id === "string" ? raw.id : undefined,
+                stageId: entryId,
             },
         ]);
     }
     const stage = {
-        id: raw.id,
+        id: entryId,
         system_prompt: raw.system_prompt,
         model: raw.model,
     };
@@ -71,9 +56,9 @@ export async function loadStageOutcome(filePath) {
             return loadFailure([
                 {
                     code: "stage.invalid_payload_schema",
-                    message: `Invalid stage file ${filePath}: payload_schema must be an object`,
+                    message: `Invalid stage ${label}: payload_schema must be an object`,
                     category: "stage",
-                    stageId: raw.id,
+                    stageId: entryId,
                 },
             ]);
         }
@@ -85,19 +70,19 @@ export async function loadStageOutcome(filePath) {
             return loadFailure([
                 {
                     code: "stage.invalid_payload_schema",
-                    message: `Invalid stage file ${filePath}: invalid payload_schema: ${message}`,
+                    message: `Invalid stage ${label}: invalid payload_schema: ${message}`,
                     category: "stage",
-                    stageId: raw.id,
+                    stageId: entryId,
                 },
             ]);
         }
         stage.payload_schema = raw.payload_schema;
     }
-    const gateKindsOutcome = parseGateKinds(raw.gate_kinds, filePath);
+    const gateKindsOutcome = parseGateKinds(raw.gate_kinds, label);
     if (!gateKindsOutcome.ok) {
         const issues = gateKindsOutcome.issues.map((issue) => ({
             ...issue,
-            stageId: stage.id,
+            stageId: entryId,
         }));
         return loadFailure(issues);
     }
@@ -108,9 +93,9 @@ export async function loadStageOutcome(filePath) {
             return loadFailure([
                 {
                     code: "stage.invalid_skill",
-                    message: `Invalid stage file ${filePath}: skill must be a non-empty string`,
+                    message: `Invalid stage ${label}: skill must be a non-empty string`,
                     category: "stage",
-                    stageId: stage.id,
+                    stageId: entryId,
                 },
             ]);
         }
@@ -118,6 +103,49 @@ export async function loadStageOutcome(filePath) {
     }
     return loadSuccess(stage);
 }
+export function loadStageFromObjectOutcome(raw, ctx) {
+    const label = `${ctx.entryId} (${ctx.declaringPath})`;
+    return parseStageFields(raw, label, ctx.entryId);
+}
+export async function loadStageOutcome(filePath) {
+    let raw;
+    try {
+        raw = await readYamlObject(filePath);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return loadFailure([
+            {
+                code: "stage.load_error",
+                message,
+                category: "stage",
+            },
+        ]);
+    }
+    if (typeof raw?.id !== "string") {
+        return loadFailure([
+            {
+                code: "stage.invalid_shape",
+                message: `Invalid stage file ${filePath}: id, system_prompt, and model are required strings`,
+                category: "stage",
+            },
+        ]);
+    }
+    const outcome = parseStageFields(raw, `file ${filePath}`, raw.id);
+    if (!outcome.ok)
+        return outcome;
+    if (outcome.value.id !== raw.id) {
+        return loadFailure([
+            {
+                code: "stage.invalid_shape",
+                message: `Invalid stage file ${filePath}: id mismatch`,
+                category: "stage",
+                stageId: raw.id,
+            },
+        ]);
+    }
+    return outcome;
+}
 export async function loadStage(filePath) {
     const outcome = await loadStageOutcome(filePath);
     if (!outcome.ok) {

package/dist/config/loadStageflowManifest.d.ts

@@ -0,0 +1,5 @@
+import type { LoadedManifest } from "../types/stageflowManifest.js";
+import { type LoadOutcome } from "./loadOutcome.js";
+export declare function manifestPathForProject(projectRoot: string): string;
+export declare function parseStageflowManifestOutcome(yamlText: string, manifestPath: string, projectRoot: string): LoadOutcome<LoadedManifest>;
+export declare function loadStageflowManifestOutcome(projectRoot: string): Promise<LoadOutcome<LoadedManifest>>;

package/dist/config/loadStageflowManifest.js

@@ -0,0 +1,157 @@
+import { access, readFile } from "node:fs/promises";
+import path from "node:path";
+import { parse as parseYaml } from "yaml";
+import { loadFailure, loadSuccess } from "./loadOutcome.js";
+const DEFAULT_PIPELINE_PATTERN = "*.pipeline.yaml";
+const DEFAULT_TASK_PATTERN = "*.task.yaml";
+export function manifestPathForProject(projectRoot) {
+    return path.join(projectRoot, "stageflow.yaml");
+}
+function catalogIssue(code, message) {
+    return { code, message, category: "catalog" };
+}
+function normalizeEntryPath(entry) {
+    return entry.replace(/\\/g, "/").replace(/^\.\/+/, "");
+}
+function parseStringArray(raw, field, issues) {
+    if (!Array.isArray(raw)) {
+        issues.push(catalogIssue("catalog.manifest_invalid", `catalog.${field} must be an array`));
+        return null;
+    }
+    const entries = [];
+    for (const item of raw) {
+        if (typeof item !== "string" || item.trim().length === 0) {
+            issues.push(catalogIssue("catalog.manifest_invalid", `catalog.${field} entries must be non-empty strings`));
+            return null;
+        }
+        const normalized = normalizeEntryPath(item.trim());
+        if (path.isAbsolute(normalized)) {
+            issues.push(catalogIssue("catalog.manifest_invalid", `catalog.${field} entries must be repo-relative paths`));
+            return null;
+        }
+        entries.push(normalized);
+    }
+    return entries;
+}
+function parsePatterns(raw, issues) {
+    const defaults = {
+        pipeline: DEFAULT_PIPELINE_PATTERN,
+        task: DEFAULT_TASK_PATTERN,
+    };
+    if (raw === undefined || raw === null) {
+        return defaults;
+    }
+    if (typeof raw !== "object" || Array.isArray(raw)) {
+        issues.push(catalogIssue("catalog.manifest_invalid", "catalog.patterns must be an object"));
+        return defaults;
+    }
+    const record = raw;
+    const pipeline = record.pipeline === undefined
+        ? defaults.pipeline
+        : typeof record.pipeline === "string" && record.pipeline.trim().length > 0
+            ? record.pipeline.trim()
+            : null;
+    const task = record.task === undefined
+        ? defaults.task
+        : typeof record.task === "string" && record.task.trim().length > 0
+            ? record.task.trim()
+            : null;
+    if (pipeline === null) {
+        issues.push(catalogIssue("catalog.manifest_invalid", "catalog.patterns.pipeline must be a string"));
+    }
+    if (task === null) {
+        issues.push(catalogIssue("catalog.manifest_invalid", "catalog.patterns.task must be a string"));
+    }
+    return {
+        pipeline: pipeline ?? defaults.pipeline,
+        task: task ?? defaults.task,
+    };
+}
+export function parseStageflowManifestOutcome(yamlText, manifestPath, projectRoot) {
+    let raw;
+    try {
+        raw = parseYaml(yamlText);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return loadFailure([catalogIssue("catalog.manifest_load_error", message)]);
+    }
+    const issues = [];
+    const record = raw;
+    if (!record || typeof record !== "object" || Array.isArray(record)) {
+        return loadFailure([
+            catalogIssue("catalog.manifest_invalid", "stageflow.yaml must be a mapping"),
+        ]);
+    }
+    if (record.version !== 1) {
+        issues.push(catalogIssue("catalog.manifest_invalid", `Unsupported manifest version: ${record.version}`));
+    }
+    const catalogRaw = record.catalog;
+    if (catalogRaw === undefined ||
+        typeof catalogRaw !== "object" ||
+        catalogRaw === null ||
+        Array.isArray(catalogRaw)) {
+        issues.push(catalogIssue("catalog.manifest_invalid", "catalog object is required"));
+        if (issues.length > 0) {
+            return loadFailure(issues);
+        }
+    }
+    const catalogRecord = catalogRaw;
+    const pipelines = parseStringArray(catalogRecord.pipelines, "pipelines", issues);
+    const tasks = parseStringArray(catalogRecord.tasks, "tasks", issues);
+    const excludeRaw = catalogRecord.exclude;
+    let exclude;
+    if (excludeRaw !== undefined) {
+        const parsed = parseStringArray(excludeRaw, "exclude", issues);
+        if (parsed !== null) {
+            exclude = parsed;
+        }
+    }
+    const patterns = parsePatterns(catalogRecord.patterns, issues);
+    if (issues.length > 0) {
+        return loadFailure(issues);
+    }
+    if (pipelines === null || tasks === null) {
+        return loadFailure(issues);
+    }
+    const catalog = { pipelines, tasks };
+    if (exclude !== undefined) {
+        catalog.exclude = exclude;
+    }
+    if (catalogRecord.patterns !== undefined) {
+        catalog.patterns = {};
+        if (typeof catalogRecord.patterns.pipeline === "string") {
+            catalog.patterns.pipeline = patterns.pipeline;
+        }
+        if (typeof catalogRecord.patterns.task === "string") {
+            catalog.patterns.task = patterns.task;
+        }
+    }
+    const manifest = { version: 1, catalog };
+    return loadSuccess({
+        path: manifestPath,
+        projectRoot,
+        manifest,
+        patterns,
+    });
+}
+export async function loadStageflowManifestOutcome(projectRoot) {
+    const manifestPath = manifestPathForProject(projectRoot);
+    try {
+        await access(manifestPath);
+    }
+    catch {
+        return loadFailure([
+            catalogIssue("catalog.manifest_load_error", `Manifest not found: ${manifestPath}`),
+        ]);
+    }
+    let yamlText;
+    try {
+        yamlText = await readFile(manifestPath, "utf8");
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return loadFailure([catalogIssue("catalog.manifest_load_error", message)]);
+    }
+    return parseStageflowManifestOutcome(yamlText, manifestPath, projectRoot);
+}

package/dist/config/mergePipelineIncludes.d.ts

@@ -0,0 +1,9 @@
+import { type LoadOutcome } from "./loadOutcome.js";
+export type RawMergedEntry = {
+    raw: unknown;
+    declaringPath: string;
+};
+export declare function mergePipelineStages(rootPath: string): Promise<LoadOutcome<{
+    entries: RawMergedEntry[];
+    pipelineId: string;
+}>>;