poe-code 3.0.182 → 3.0.184

package/dist/cli/commands/pipeline.js

@@ -3,15 +3,16 @@ import { readFile, stat } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import { acp, cancel, createDashboard, isCancel, multiselect, promptText, select } from "@poe-code/design-system";
 import { resolveAgentId, parseAgentSpecifier, formatAgentSpecifier, allAgents } from "@poe-code/agent-defs";
-import { isActivityTimeoutError, renderAcpEvent } from "@poe-code/agent-spawn";
+import { renderAcpEvent } from "@poe-code/agent-spawn";
 import { installSkill, resolveAgentSupport, supportedAgents } from "@poe-code/agent-skill-config";
 import { readMergedDocument, resolveScope } from "@poe-code/poe-code-config";
 import { pipelineConfigScope, planConfigScope } from "../../services/config.js";
 import { ValidationError } from "../errors.js";
+import { discoverPipelineInitSources } from "./pipeline-init.js";
 import { createExecutionResources, resolveCommandFlags, resolveDefaultAgent } from "./shared.js";
-import { runPipeline as sdkRunPipeline } from "../../sdk/pipeline.js";
+import { runPipelineInit as sdkRunPipelineInit, runPipeline as sdkRunPipeline } from "../../sdk/pipeline.js";
 import { spawn as sdkSpawn } from "../../sdk/spawn.js";
-import { buildExecutionPrompt, interpolate, loadResolvedSteps, parsePlan, resolveAbsolutePlanPath, resolveFileIncludes, resolvePlanDirectory, resolvePlanPaths } from "@poe-code/pipeline";
+import { buildExecutionPrompt, interpolatePipelineVars, loadResolvedSteps, parsePlan, resolveAbsolutePlanPath, resolveFileIncludes, resolvePipelineVars, resolvePlanDirectory, resolvePlanPaths, validateResolvedPromptVars } from "@poe-code/pipeline";
 import { createDashboardLineBuffer, formatDashboardDuration, formatDashboardTimestamp, registerDashboardQuitCommands, shouldUseInteractiveDashboard } from "./dashboard-loop-shared.js";
 async function resolvePipelineCommandConfig(container) {
     const configDoc = await readMergedDocument(container.fs, container.env.configPath, container.env.projectConfigPath);
@@ -47,11 +48,24 @@ function resolveMaxRuns(value) {
     }
     return parsed;
 }
+function resolvePipelineInitSourcePath(container, sourcePath) {
+    const absolutePath = sourcePath.startsWith("~/")
+        ? path.join(container.env.homeDir, sourcePath.slice(2))
+        : path.isAbsolute(sourcePath)
+            ? sourcePath
+            : path.resolve(container.env.cwd, sourcePath);
+    return {
+        absolutePath,
+        relativePath: sourcePath,
+        title: path.basename(sourcePath, path.extname(sourcePath))
+    };
+}
 function formatRunSummary(result) {
     const metrics = result.metrics;
     return [
         `Runs: ${result.runsCompleted}`,
-        `tasksCompleted: ${metrics.tasksCompleted}, tasksFailed: ${metrics.tasksFailed}, stepsCompleted: ${metrics.stepsCompleted}`,
+        `Tasks: ${metrics.tasksCompleted} completed, ${metrics.tasksFailed} failed`,
+        `Steps: ${metrics.stepsCompleted} completed`,
         `Total tokens: ${metrics.totalInputTokens} input, ${metrics.totalOutputTokens} output, ${metrics.totalCachedTokens} cached`,
         `Duration: ${formatDashboardDuration(result.totalDurationMs)}`
     ].join("\n   ");
@@ -96,6 +110,14 @@ function formatTaskCompleteMessage(progress) {
     if (progress.phase) {
         return `${progress.taskTitle} ${status} in ${duration}${usage}`;
     }
+    if (progress.stepName) {
+        if (progress.success && !progress.taskCompleted) {
+            return `Task ${progress.taskId} (${progress.stepName}) step done in ${duration}${usage}`;
+        }
+        if (!progress.success) {
+            return `Task ${progress.taskId} (${progress.stepName}) failed in ${duration}${usage}`;
+        }
+    }
     return `Task ${progress.taskId} ${status} in ${duration}${usage}`;
 }
 function formatDashboardCurrentAction(progress) {
@@ -174,69 +196,64 @@ async function streamAcpEventsToDashboard(options) {
 }
 function createPipelineDashboardRunAgent(options) {
     return async (input) => {
-        for (let attempt = 1; attempt <= 3; attempt += 1) {
-            const toolBuffer = createDashboardLineBuffer((line) => {
-                options.appendOutput("tool", `[${options.activeStage()}] ${line}`);
-            });
-            const errorBuffer = createDashboardLineBuffer((line) => {
-                options.appendOutput("error", `[${options.activeStage()}] ${line}`);
-            });
-            let sawStdout = false;
-            let sawStderr = false;
-            try {
-                const { events, result } = sdkSpawn(input.agent, {
-                    prompt: input.prompt,
-                    cwd: input.cwd,
-                    logDir: input.logDir,
-                    model: input.model,
-                    mode: input.mode,
-                    ...(input.mcpServers ? { mcpServers: input.mcpServers } : {}),
-                    ...(input.signal ? { signal: input.signal } : {}),
-                    tee: {
-                        stdout: {
-                            write(chunk) {
-                                sawStdout = true;
-                                toolBuffer.push(chunk);
-                            }
-                        },
-                        stderr: {
-                            write(chunk) {
-                                sawStderr = true;
-                                errorBuffer.push(chunk);
-                            }
+        const toolBuffer = createDashboardLineBuffer((line) => {
+            options.appendOutput("tool", `[${options.activeStage()}] ${line}`);
+        });
+        const errorBuffer = createDashboardLineBuffer((line) => {
+            options.appendOutput("error", `[${options.activeStage()}] ${line}`);
+        });
+        let sawStdout = false;
+        let sawStderr = false;
+        try {
+            const { events, result } = sdkSpawn(input.agent, {
+                prompt: input.prompt,
+                cwd: input.cwd,
+                logDir: input.logDir,
+                model: input.model,
+                mode: input.mode,
+                ...(input.mcpServers ? { mcpServers: input.mcpServers } : {}),
+                ...(input.signal ? { signal: input.signal } : {}),
+                tee: {
+                    stdout: {
+                        write(chunk) {
+                            sawStdout = true;
+                            toolBuffer.push(chunk);
                         }
                     },
-                    activityTimeoutMs: 10 * 60 * 1000
-                });
-                const eventStream = streamAcpEventsToDashboard({
-                    events,
-                    onToolOutput(chunk) {
-                        toolBuffer.push(chunk);
-                    },
-                    onErrorOutput(chunk) {
-                        errorBuffer.push(chunk);
+                    stderr: {
+                        write(chunk) {
+                            sawStderr = true;
+                            errorBuffer.push(chunk);
+                        }
                     }
-                });
-                const [spawnResult, sawEvents] = await Promise.all([result, eventStream]);
-                if (!sawEvents && !sawStdout && spawnResult.stdout.length > 0) {
-                    toolBuffer.push(spawnResult.stdout);
-                }
-                if (!sawStderr && spawnResult.stderr.length > 0) {
-                    errorBuffer.push(spawnResult.stderr);
+                },
+                activityTimeoutMs: 10 * 60 * 1000
+            });
+            const eventStream = streamAcpEventsToDashboard({
+                events,
+                onToolOutput(chunk) {
+                    toolBuffer.push(chunk);
+                },
+                onErrorOutput(chunk) {
+                    errorBuffer.push(chunk);
                 }
-                toolBuffer.flush();
-                errorBuffer.flush();
-                return spawnResult;
+            });
+            const [spawnResult, sawEvents] = await Promise.all([result, eventStream]);
+            if (!sawEvents && !sawStdout && spawnResult.stdout.length > 0) {
+                toolBuffer.push(spawnResult.stdout);
             }
-            catch (error) {
-                toolBuffer.flush();
-                errorBuffer.flush();
-                if (!isActivityTimeoutError(error) || attempt === 3) {
-                    throw error;
-                }
+            if (!sawStderr && spawnResult.stderr.length > 0) {
+                errorBuffer.push(spawnResult.stderr);
             }
+            toolBuffer.flush();
+            errorBuffer.flush();
+            return spawnResult;
+        }
+        catch (error) {
+            toolBuffer.flush();
+            errorBuffer.flush();
+            throw error;
         }
-        throw new Error("Unreachable");
     };
 }
 function dashboardStatusForResult(result) {
@@ -261,15 +278,18 @@ async function runPipelineWithDashboard(options) {
     let currentAction;
     let currentStage = "pipeline";
     let status = "running";
+    let waitingForLock = false;
     const syncStats = () => {
-        dashboard.updateStats({
+        const stats = {
             status,
             iterations,
+            iterationsLabel: "Tasks",
             tokensIn,
             tokensOut,
             elapsedMs: Math.max(0, Date.now() - startedAt),
             ...(currentAction ? { currentAction } : {})
-        });
+        };
+        dashboard.updateStats(stats);
     };
     const appendOutput = (kind, message) => {
         dashboard.appendOutput({
@@ -312,6 +332,18 @@ async function runPipelineWithDashboard(options) {
             onPlanReloadError(error) {
                 appendOutput("error", `Plan reload failed, using last good state: ${error.message}`);
             },
+            onLockStatusChange(lockStatus) {
+                appendOutput("status", lockStatus.message);
+                if (lockStatus.state === "waiting") {
+                    waitingForLock = true;
+                    currentAction = lockStatus.message;
+                }
+                else if (waitingForLock) {
+                    waitingForLock = false;
+                    currentAction = undefined;
+                }
+                syncStats();
+            },
             onPlanResolved(summary) {
                 appendOutput("info", `Config · ${formatPipelineConfigSummary({
                     agent: options.agent,
@@ -330,7 +362,9 @@ async function runPipelineWithDashboard(options) {
                 syncStats();
             },
             onTaskComplete(progress) {
-                iterations += 1;
+                if (progress.taskCompleted) {
+                    iterations += 1;
+                }
                 if (progress.usage) {
                     tokensIn += progress.usage.inputTokens;
                     tokensOut += progress.usage.outputTokens;
@@ -362,11 +396,16 @@ function resolvePipelinePaths(scope, cwd, homeDir) {
         ? path.join(homeDir, ".poe-code", "pipeline")
         : path.join(cwd, ".poe-code", "pipeline");
     const displayRoot = scope === "global" ? "~/.poe-code/pipeline" : ".poe-code/pipeline";
+    const stepsDirectoryPath = path.join(rootPath, "steps");
     return {
         plansPath: path.join(rootPath, "plans"),
-        stepsPath: path.join(rootPath, "steps.yaml"),
+        stepsDirectoryPath,
+        defaultStepsPath: path.join(stepsDirectoryPath, "default.yaml"),
+        legacyStepsPath: path.join(rootPath, "steps.yaml"),
         displayPlansPath: `${displayRoot}/plans`,
-        displayStepsPath: `${displayRoot}/steps.yaml`
+        displayStepsDirectoryPath: `${displayRoot}/steps`,
+        displayDefaultStepsPath: `${displayRoot}/steps/default.yaml`,
+        displayLegacyStepsPath: `${displayRoot}/steps.yaml`
     };
 }
 async function loadPipelineTemplates() {
@@ -541,6 +580,9 @@ export function registerPipelineCommand(program, container) {
                         onPlanReloadError(error) {
                             resources.logger.warn(`Plan reload failed, using last good state: ${error.message}`);
                         },
+                        onLockStatusChange(lockStatus) {
+                            resources.logger.info(lockStatus.message);
+                        },
                         onPlanResolved(summary) {
                             resources.logger.resolved("Config", formatPipelineConfigSummary({
                                 agent,
@@ -589,6 +631,122 @@ export function registerPipelineCommand(program, container) {
             resources.context.finalize();
         }
     });
+    pipeline
+        .command("init")
+        .description("Initialize pipeline plans from source Markdown docs.")
+        .argument("[question]", "Optional user question to forward to the plan generator")
+        .option("--agent <name>", "Agent to generate the plan with")
+        .option("--model <model>", "Model override passed to the agent")
+        .option("--source <path>", "Single source Markdown doc to convert")
+        .option("--sources <paths...>", "Multiple source Markdown docs to convert")
+        .action(async function (question) {
+        const flags = resolveCommandFlags(program);
+        const resources = createExecutionResources(container, flags, "pipeline:init");
+        const options = this.opts();
+        resources.logger.intro("pipeline init");
+        try {
+            let sourcePaths = options.sources;
+            let resolvedQuestion = question;
+            if (!resolvedQuestion && sourcePaths && sourcePaths.length > 0) {
+                const trailingArgument = sourcePaths[sourcePaths.length - 1];
+                if (trailingArgument && !trailingArgument.toLowerCase().endsWith(".md")) {
+                    resolvedQuestion = trailingArgument;
+                    sourcePaths = sourcePaths.slice(0, -1);
+                }
+            }
+            let agent;
+            if (options.agent) {
+                agent = resolvePipelineAgent(options.agent);
+            }
+            else if (flags.assumeYes) {
+                agent = DEFAULT_PIPELINE_AGENT;
+            }
+            else {
+                const selected = await select({
+                    message: "Select agent to generate pipeline plans with:",
+                    options: supportedAgents.map((value) => ({
+                        value,
+                        label: value
+                    }))
+                });
+                if (isCancel(selected)) {
+                    cancel("Pipeline init cancelled.");
+                    return;
+                }
+                agent = resolvePipelineAgent(selected);
+            }
+            if (options.source && sourcePaths && sourcePaths.length > 0) {
+                throw new ValidationError("Use either --source or --sources, not both.");
+            }
+            let sources;
+            if (options.source) {
+                sources = [resolvePipelineInitSourcePath(container, options.source)];
+            }
+            else if (sourcePaths && sourcePaths.length > 0) {
+                sources = sourcePaths.map((sourcePath) => resolvePipelineInitSourcePath(container, sourcePath));
+            }
+            else {
+                if (flags.assumeYes) {
+                    throw new ValidationError("Provide --source or --sources when using --yes.");
+                }
+                const discoveredSources = await discoverPipelineInitSources({ container });
+                if (discoveredSources.length === 0) {
+                    resources.logger.info("No source documents available to initialize.");
+                    return;
+                }
+                const selected = await multiselect({
+                    message: "Select source Markdown docs to convert:",
+                    options: discoveredSources.map((source) => ({
+                        value: source.relativePath,
+                        label: `${source.title} (${source.relativePath})`
+                    })),
+                    required: true
+                });
+                if (isCancel(selected)) {
+                    cancel("Pipeline init cancelled.");
+                    return;
+                }
+                const selectedSourcePaths = Array.isArray(selected)
+                    ? new Set(selected)
+                    : new Set();
+                sources = discoveredSources.filter((source) => selectedSourcePaths.has(source.relativePath));
+                if (sources.length === 0) {
+                    return;
+                }
+            }
+            const result = await sdkRunPipelineInit({
+                agent,
+                cwd: container.env.cwd,
+                homeDir: container.env.homeDir,
+                ...(options.model ? { model: options.model } : {}),
+                ...(resolvedQuestion ? { question: resolvedQuestion } : {}),
+                sources,
+                assumeYes: flags.assumeYes,
+                onSourceStart(source, index, total) {
+                    resources.logger.info(`Source ${index}/${total}: ${source.relativePath}`);
+                },
+                onSourceComplete(source, index, total) {
+                    resources.logger.success(`Completed ${index}/${total}: ${source.relativePath}`);
+                }
+            });
+            if (result.stopReason === "failed") {
+                process.exitCode = 1;
+                resources.logger.error(result.failedSource
+                    ? `Pipeline init failed at ${result.failedSource}.`
+                    : "Pipeline init failed.");
+                return;
+            }
+            if (result.stopReason === "cancelled") {
+                process.exitCode = 130;
+                resources.logger.warn("Pipeline init cancelled.");
+                return;
+            }
+            resources.logger.success("Pipeline init finished.");
+        }
+        finally {
+            resources.context.finalize();
+        }
+    });
     pipeline
         .command("validate")
         .description("Validate a pipeline plan markdown file without running it.")
@@ -601,38 +759,43 @@ export function registerPipelineCommand(program, container) {
             resources.logger.intro("pipeline validate");
             const absolutePath = resolveAbsolutePlanPath(file, container.env.cwd, container.env.homeDir);
             const content = await container.fs.readFile(absolutePath, "utf8");
+            const draftPlan = parsePlan(content);
             const steps = await loadResolvedSteps({
                 cwd: container.env.cwd,
                 homeDir: container.env.homeDir,
-                fs: container.fs
+                fs: container.fs,
+                name: draftPlan.extends,
+                stepOverrides: draftPlan.stepOverrides
             });
-            const hasSteps = Object.keys(steps.steps).length > 0;
-            const plan = parsePlan(content, hasSteps ? { availableSteps: steps.steps } : {});
+            const plan = parsePlan(content, { availableSteps: steps.steps });
             const total = plan.tasks.length;
             const done = plan.tasks.filter((t) => {
                 if (typeof t.status === "string")
                     return t.status === "done";
                 return Object.values(t.status).every((s) => s === "done");
             }).length;
+            const readFile = container.fs.readFile.bind(container.fs);
+            const resolvedVars = await resolvePipelineVars(plan.vars ?? {}, container.env.cwd, readFile);
+            const resolvedSetup = plan.setup === null ? undefined : (plan.setup ?? steps.setup);
+            const resolvedTeardown = plan.teardown === null ? undefined : (plan.teardown ?? steps.teardown);
+            validateResolvedPromptVars({
+                plan,
+                steps: steps.steps,
+                planPath: file,
+                vars: resolvedVars,
+                setup: resolvedSetup,
+                teardown: resolvedTeardown
+            });
             resources.logger.resolved("Plan", file);
             resources.logger.resolved("Tasks", `${total} tasks (${done} done)`);
-            if (hasSteps) {
+            if (Object.keys(steps.steps).length > 0) {
                 resources.logger.resolved("Steps", Object.keys(steps.steps).join(", "));
             }
             resources.logger.success("Plan is valid.");
             const opts = this.opts();
             if (opts.preview) {
-                const readFile = container.fs.readFile.bind(container.fs);
-                const resolvedVars = {};
-                for (const [key, value] of Object.entries(plan.vars ?? {})) {
-                    resolvedVars[key] = await resolveFileIncludes(value, container.env.cwd, readFile);
-                }
-                const resolvedSetup = plan.setup === null ? undefined : (plan.setup ?? steps.setup);
-                const resolvedTeardown = plan.teardown === null ? undefined : (plan.teardown ?? steps.teardown);
                 if (resolvedSetup) {
-                    const raw = Object.keys(resolvedVars).length > 0
-                        ? interpolate(resolvedSetup.prompt, resolvedVars)
-                        : resolvedSetup.prompt;
+                    const raw = interpolatePipelineVars(resolvedSetup.prompt, resolvedVars, "setup");
                     const expanded = await resolveFileIncludes(raw, container.env.cwd, readFile);
                     resources.logger.resolved("setup", expanded);
                 }
@@ -659,9 +822,7 @@ export function registerPipelineCommand(program, container) {
                     }
                 }
                 if (resolvedTeardown) {
-                    const raw = Object.keys(resolvedVars).length > 0
-                        ? interpolate(resolvedTeardown.prompt, resolvedVars)
-                        : resolvedTeardown.prompt;
+                    const raw = interpolatePipelineVars(resolvedTeardown.prompt, resolvedVars, "teardown");
                     const expanded = await resolveFileIncludes(raw, container.env.cwd, readFile);
                     resources.logger.resolved("teardown", expanded);
                 }
@@ -689,7 +850,7 @@ export function registerPipelineCommand(program, container) {
         .option("--agent <name>", "Agent to install the Pipeline skill for")
         .option("--local", "Install project-local skill and pipeline files")
         .option("--global", "Install user-global skill and pipeline files")
-        .option("--force", "Overwrite an existing steps.yaml scaffold")
+        .option("--force", "Overwrite an existing default step config scaffold")
         .action(async function () {
         const flags = resolveCommandFlags(program);
         const resources = createExecutionResources(container, flags, "pipeline:install");
@@ -778,21 +939,35 @@ export function registerPipelineCommand(program, container) {
                     resources.logger.info(`Create: ${pipelinePaths.displayPlansPath}`);
                 }
             }
-            const stepsExists = await pathExists(container.fs, pipelinePaths.stepsPath);
+            const legacyStepsExists = await pathExists(container.fs, pipelinePaths.legacyStepsPath);
+            let stepsExists = await pathExists(container.fs, pipelinePaths.defaultStepsPath);
+            if (legacyStepsExists && !stepsExists) {
+                if (flags.dryRun) {
+                    resources.logger.dryRun(`Would rename: ${pipelinePaths.displayLegacyStepsPath} -> ${pipelinePaths.displayDefaultStepsPath}`);
+                }
+                else {
+                    await container.fs.mkdir(pipelinePaths.stepsDirectoryPath, {
+                        recursive: true
+                    });
+                    await container.fs.rename(pipelinePaths.legacyStepsPath, pipelinePaths.defaultStepsPath);
+                    resources.logger.info(`Rename: ${pipelinePaths.displayLegacyStepsPath} -> ${pipelinePaths.displayDefaultStepsPath}`);
+                }
+                stepsExists = true;
+            }
             if (stepsExists && !options.force) {
-                resources.logger.info(`Skip: ${pipelinePaths.displayStepsPath} (already exists)`);
+                resources.logger.info(`Skip: ${pipelinePaths.displayDefaultStepsPath} (already exists)`);
             }
             else if (flags.dryRun) {
-                resources.logger.dryRun(`Would ${stepsExists ? "overwrite" : "create"}: ${pipelinePaths.displayStepsPath}`);
+                resources.logger.dryRun(`Would ${stepsExists ? "overwrite" : "create"}: ${pipelinePaths.displayDefaultStepsPath}`);
             }
             else {
-                await container.fs.mkdir(path.dirname(pipelinePaths.stepsPath), {
+                await container.fs.mkdir(path.dirname(pipelinePaths.defaultStepsPath), {
                     recursive: true
                 });
-                await container.fs.writeFile(pipelinePaths.stepsPath, templates.steps, {
+                await container.fs.writeFile(pipelinePaths.defaultStepsPath, templates.steps, {
                     encoding: "utf8"
                 });
-                resources.logger.info(`${stepsExists ? "Overwrite" : "Create"}: ${pipelinePaths.displayStepsPath}`);
+                resources.logger.info(`${stepsExists ? "Overwrite" : "Create"}: ${pipelinePaths.displayDefaultStepsPath}`);
             }
             resources.context.complete({
                 success: `Installed Pipeline skill for ${support.id} and scaffolded ${scope} pipeline files`,