substrate-ai 0.1.20 → 0.1.21

package/dist/cli/index.js

@@ -3,6 +3,7 @@ import { AdapterRegistry, ConfigError, ConfigIncompatibleFormatError, DatabaseWr
 import { CURRENT_CONFIG_FORMAT_VERSION, CURRENT_TASK_GRAPH_VERSION, PartialSubstrateConfigSchema, SUPPORTED_CONFIG_FORMAT_VERSIONS, SubstrateConfigSchema } from "../config-schema-C9tTMcm1.js";
 import { defaultConfigMigrator } from "../version-manager-impl-O25ieEjS.js";
 import { registerUpgradeCommand } from "../upgrade-CHhsJc_q.js";
+import { createRequire } from "module";
 import { Command } from "commander";
 import { fileURLToPath } from "url";
 import { dirname, extname, isAbsolute, join, relative, resolve } from "path";
@@ -23,8 +24,12 @@ import * as readline$1 from "readline";
 import * as readline from "readline";
 import { createInterface as createInterface$1 } from "readline";
 import { randomUUID as randomUUID$1 } from "crypto";
-import { createRequire } from "node:module";
+import { createRequire as createRequire$1 } from "node:module";
 
+//#region rolldown:runtime
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+
+//#endregion
 //#region src/cli/utils/formatting.ts
 /**
 * Build adapter list rows from discovery results.
@@ -6492,6 +6497,70 @@ const PIPELINE_EVENT_METADATA = [
 				description: "Log message."
 			}
 		]
+	},
+	{
+		type: "pipeline:heartbeat",
+		description: "Periodic heartbeat emitted every 30s when no other progress events have fired.",
+		when: "Every 30 seconds during pipeline execution. Allows detection of stalled pipelines.",
+		fields: [
+			{
+				name: "ts",
+				type: "string",
+				description: "ISO-8601 timestamp generated at emit time."
+			},
+			{
+				name: "run_id",
+				type: "string",
+				description: "Pipeline run ID."
+			},
+			{
+				name: "active_dispatches",
+				type: "number",
+				description: "Number of sub-agents currently running."
+			},
+			{
+				name: "completed_dispatches",
+				type: "number",
+				description: "Number of dispatches completed."
+			},
+			{
+				name: "queued_dispatches",
+				type: "number",
+				description: "Number of dispatches waiting to start."
+			}
+		]
+	},
+	{
+		type: "story:stall",
+		description: "Emitted when the watchdog detects no progress for an extended period (default: 10 minutes).",
+		when: "When a story has shown no progress for longer than the watchdog timeout. Indicates likely stall.",
+		fields: [
+			{
+				name: "ts",
+				type: "string",
+				description: "ISO-8601 timestamp generated at emit time."
+			},
+			{
+				name: "run_id",
+				type: "string",
+				description: "Pipeline run ID."
+			},
+			{
+				name: "story_key",
+				type: "string",
+				description: "Story key that appears stalled."
+			},
+			{
+				name: "phase",
+				type: "string",
+				description: "Phase the story was in when stall was detected."
+			},
+			{
+				name: "elapsed_ms",
+				type: "number",
+				description: "Milliseconds since last progress event."
+			}
+		]
 	}
 ];
 /**
@@ -10066,6 +10135,10 @@ function createImplementationOrchestrator(deps) {
 	const _stories = new Map();
 	let _paused = false;
 	let _pauseGate = null;
+	let _lastProgressTs = Date.now();
+	let _heartbeatTimer = null;
+	const HEARTBEAT_INTERVAL_MS = 3e4;
+	const WATCHDOG_TIMEOUT_MS = 6e5;
 	function getStatus() {
 		const stories = {};
 		for (const [key, s] of _stories) stories[key] = { ...s };
@@ -10087,6 +10160,7 @@ function createImplementationOrchestrator(deps) {
 	}
 	function persistState() {
 		if (config.pipelineRunId === void 0) return;
+		recordProgress();
 		try {
 			const serialized = JSON.stringify(getStatus());
 			updatePipelineRun(db, config.pipelineRunId, {
@@ -10097,6 +10171,50 @@ function createImplementationOrchestrator(deps) {
 			logger$31.warn("Failed to persist orchestrator state", { err });
 		}
 	}
+	function recordProgress() {
+		_lastProgressTs = Date.now();
+	}
+	function startHeartbeat() {
+		if (_heartbeatTimer !== null) return;
+		_heartbeatTimer = setInterval(() => {
+			if (_state !== "RUNNING") return;
+			let active = 0;
+			let completed = 0;
+			let queued = 0;
+			for (const s of _stories.values()) if (s.phase === "COMPLETE" || s.phase === "ESCALATED") completed++;
+			else if (s.phase === "PENDING") queued++;
+			else active++;
+			eventBus.emit("orchestrator:heartbeat", {
+				runId: config.pipelineRunId ?? "",
+				activeDispatches: active,
+				completedDispatches: completed,
+				queuedDispatches: queued
+			});
+			const elapsed = Date.now() - _lastProgressTs;
+			if (elapsed >= WATCHDOG_TIMEOUT_MS) {
+				for (const [key, s] of _stories) if (s.phase !== "PENDING" && s.phase !== "COMPLETE" && s.phase !== "ESCALATED") {
+					logger$31.warn({
+						storyKey: key,
+						phase: s.phase,
+						elapsedMs: elapsed
+					}, "Watchdog: possible stall detected");
+					eventBus.emit("orchestrator:stall", {
+						runId: config.pipelineRunId ?? "",
+						storyKey: key,
+						phase: s.phase,
+						elapsedMs: elapsed
+					});
+				}
+			}
+		}, HEARTBEAT_INTERVAL_MS);
+		if (_heartbeatTimer && typeof _heartbeatTimer === "object" && "unref" in _heartbeatTimer) _heartbeatTimer.unref();
+	}
+	function stopHeartbeat() {
+		if (_heartbeatTimer !== null) {
+			clearInterval(_heartbeatTimer);
+			_heartbeatTimer = null;
+		}
+	}
 	/**
 	* Wait until the orchestrator is un-paused (if currently paused).
 	*/
@@ -10814,6 +10932,8 @@ function createImplementationOrchestrator(deps) {
 			pipelineRunId: config.pipelineRunId
 		});
 		persistState();
+		recordProgress();
+		startHeartbeat();
 		if (projectRoot !== void 0) {
 			const seedResult = seedMethodologyContext(db, projectRoot);
 			if (seedResult.decisionsCreated > 0) logger$31.info({
@@ -10830,12 +10950,14 @@ function createImplementationOrchestrator(deps) {
 		try {
 			await runWithConcurrency(groups, config.maxConcurrency);
 		} catch (err) {
+			stopHeartbeat();
 			_state = "FAILED";
 			_completedAt = new Date().toISOString();
 			persistState();
 			logger$31.error("Orchestrator failed with unhandled error", { err });
 			return getStatus();
 		}
+		stopHeartbeat();
 		_state = "COMPLETE";
 		_completedAt = new Date().toISOString();
 		let completed = 0;
@@ -13027,8 +13149,8 @@ const PACKAGE_ROOT = join(__dirname, "..", "..", "..");
 */
 function resolveBmadMethodSrcPath(fromDir = __dirname) {
 	try {
-		const require = createRequire(join(fromDir, "synthetic.js"));
-		const pkgJsonPath = require.resolve("bmad-method/package.json");
+		const require$1 = createRequire$1(join(fromDir, "synthetic.js"));
+		const pkgJsonPath = require$1.resolve("bmad-method/package.json");
 		return join(dirname(pkgJsonPath), "src");
 	} catch {
 		return null;
@@ -13040,9 +13162,9 @@ function resolveBmadMethodSrcPath(fromDir = __dirname) {
 */
 function resolveBmadMethodVersion(fromDir = __dirname) {
 	try {
-		const require = createRequire(join(fromDir, "synthetic.js"));
-		const pkgJsonPath = require.resolve("bmad-method/package.json");
-		const pkg = require(pkgJsonPath);
+		const require$1 = createRequire$1(join(fromDir, "synthetic.js"));
+		const pkgJsonPath = require$1.resolve("bmad-method/package.json");
+		const pkg = require$1(pkgJsonPath);
 		return pkg.version ?? "unknown";
 	} catch {
 		return "unknown";
@@ -13163,7 +13285,9 @@ function buildPipelineStatusOutput(run, tokenSummary, decisionsCount, storiesCou
 			cost_usd: totalCost
 		},
 		decisions_count: decisionsCount,
-		stories_count: storiesCount
+		stories_count: storiesCount,
+		last_activity: run.updated_at,
+		staleness_seconds: Math.round((Date.now() - new Date(run.updated_at).getTime()) / 1e3)
 	};
 }
 /**
@@ -13799,6 +13923,26 @@ async function runAutoRun(options) {
 					msg: payload.msg
 				});
 			});
+			eventBus.on("orchestrator:heartbeat", (payload) => {
+				ndjsonEmitter.emit({
+					type: "pipeline:heartbeat",
+					ts: new Date().toISOString(),
+					run_id: payload.runId,
+					active_dispatches: payload.activeDispatches,
+					completed_dispatches: payload.completedDispatches,
+					queued_dispatches: payload.queuedDispatches
+				});
+			});
+			eventBus.on("orchestrator:stall", (payload) => {
+				ndjsonEmitter.emit({
+					type: "story:stall",
+					ts: new Date().toISOString(),
+					run_id: payload.runId,
+					story_key: payload.storyKey,
+					phase: payload.phase,
+					elapsed_ms: payload.elapsedMs
+				});
+			});
 		}
 		const orchestrator = createImplementationOrchestrator({
 			db,
@@ -14501,6 +14645,175 @@ async function runAutoStatus(options) {
 		} catch {}
 	}
 }
+function inspectProcessTree() {
+	const result = {
+		orchestrator_pid: null,
+		child_pids: [],
+		zombies: []
+	};
+	try {
+		const { execFileSync } = __require("node:child_process");
+		const psOutput = execFileSync("ps", ["-eo", "pid,ppid,stat,command"], {
+			encoding: "utf-8",
+			timeout: 5e3
+		});
+		const lines = psOutput.split("\n");
+		for (const line of lines) if (line.includes("substrate auto run") && !line.includes("grep")) {
+			const match = line.trim().match(/^(\d+)/);
+			if (match) {
+				result.orchestrator_pid = parseInt(match[1], 10);
+				break;
+			}
+		}
+		if (result.orchestrator_pid !== null) for (const line of lines) {
+			const parts = line.trim().split(/\s+/);
+			if (parts.length >= 3) {
+				const pid = parseInt(parts[0], 10);
+				const ppid = parseInt(parts[1], 10);
+				const stat$2 = parts[2];
+				if (ppid === result.orchestrator_pid && pid !== result.orchestrator_pid) {
+					result.child_pids.push(pid);
+					if (stat$2.includes("Z")) result.zombies.push(pid);
+				}
+			}
+		}
+	} catch {}
+	return result;
+}
+async function runAutoHealth(options) {
+	const { outputFormat, runId, projectRoot } = options;
+	const dbRoot = await resolveMainRepoRoot(projectRoot);
+	const dbPath = join(dbRoot, ".substrate", "substrate.db");
+	if (!existsSync(dbPath)) {
+		const output = {
+			verdict: "NO_PIPELINE_RUNNING",
+			run_id: null,
+			status: null,
+			current_phase: null,
+			staleness_seconds: 0,
+			last_activity: "",
+			process: {
+				orchestrator_pid: null,
+				child_pids: [],
+				zombies: []
+			},
+			stories: {
+				active: 0,
+				completed: 0,
+				escalated: 0,
+				details: {}
+			}
+		};
+		if (outputFormat === "json") process.stdout.write(formatOutput(output, "json", true) + "\n");
+		else process.stdout.write("NO_PIPELINE_RUNNING — no substrate database found\n");
+		return 0;
+	}
+	const dbWrapper = new DatabaseWrapper(dbPath);
+	try {
+		dbWrapper.open();
+		const db = dbWrapper.db;
+		let run;
+		if (runId !== void 0) run = getPipelineRunById(db, runId);
+		else run = getLatestRun(db);
+		if (run === void 0) {
+			const output$1 = {
+				verdict: "NO_PIPELINE_RUNNING",
+				run_id: null,
+				status: null,
+				current_phase: null,
+				staleness_seconds: 0,
+				last_activity: "",
+				process: {
+					orchestrator_pid: null,
+					child_pids: [],
+					zombies: []
+				},
+				stories: {
+					active: 0,
+					completed: 0,
+					escalated: 0,
+					details: {}
+				}
+			};
+			if (outputFormat === "json") process.stdout.write(formatOutput(output$1, "json", true) + "\n");
+			else process.stdout.write("NO_PIPELINE_RUNNING — no pipeline runs found\n");
+			return 0;
+		}
+		const updatedAt = new Date(run.updated_at);
+		const stalenessSeconds = Math.round((Date.now() - updatedAt.getTime()) / 1e3);
+		let storyDetails = {};
+		let active = 0;
+		let completed = 0;
+		let escalated = 0;
+		try {
+			if (run.token_usage_json) {
+				const state = JSON.parse(run.token_usage_json);
+				if (state.stories) for (const [key, s] of Object.entries(state.stories)) {
+					storyDetails[key] = {
+						phase: s.phase,
+						review_cycles: s.reviewCycles
+					};
+					if (s.phase === "COMPLETE") completed++;
+					else if (s.phase === "ESCALATED") escalated++;
+					else if (s.phase !== "PENDING") active++;
+				}
+			}
+		} catch {}
+		const processInfo = inspectProcessTree();
+		let verdict = "NO_PIPELINE_RUNNING";
+		if (run.status === "running") if (processInfo.zombies.length > 0) verdict = "STALLED";
+		else if (stalenessSeconds > 600) verdict = "STALLED";
+		else if (processInfo.orchestrator_pid !== null && processInfo.child_pids.length === 0 && active > 0) verdict = "STALLED";
+		else verdict = "HEALTHY";
+		else if (run.status === "completed" || run.status === "failed" || run.status === "stopped") verdict = "NO_PIPELINE_RUNNING";
+		const output = {
+			verdict,
+			run_id: run.id,
+			status: run.status,
+			current_phase: run.current_phase,
+			staleness_seconds: stalenessSeconds,
+			last_activity: run.updated_at,
+			process: processInfo,
+			stories: {
+				active,
+				completed,
+				escalated,
+				details: storyDetails
+			}
+		};
+		if (outputFormat === "json") process.stdout.write(formatOutput(output, "json", true) + "\n");
+		else {
+			const verdictLabel = verdict === "HEALTHY" ? "HEALTHY" : verdict === "STALLED" ? "STALLED" : "NO PIPELINE RUNNING";
+			process.stdout.write(`\nPipeline Health: ${verdictLabel}\n`);
+			process.stdout.write(`  Run:          ${run.id}\n`);
+			process.stdout.write(`  Status:       ${run.status}\n`);
+			process.stdout.write(`  Phase:        ${run.current_phase ?? "N/A"}\n`);
+			process.stdout.write(`  Last Active:  ${run.updated_at} (${stalenessSeconds}s ago)\n`);
+			if (processInfo.orchestrator_pid !== null) {
+				process.stdout.write(`  Orchestrator: PID ${processInfo.orchestrator_pid}\n`);
+				process.stdout.write(`  Children:     ${processInfo.child_pids.length} active`);
+				if (processInfo.zombies.length > 0) process.stdout.write(` (${processInfo.zombies.length} ZOMBIE)`);
+				process.stdout.write("\n");
+			} else process.stdout.write("  Orchestrator: not running\n");
+			if (Object.keys(storyDetails).length > 0) {
+				process.stdout.write("\n  Stories:\n");
+				for (const [key, s] of Object.entries(storyDetails)) process.stdout.write(`    ${key}: ${s.phase} (${s.review_cycles} review cycles)\n`);
+				process.stdout.write(`\n  Summary: ${active} active, ${completed} completed, ${escalated} escalated\n`);
+			}
+		}
+		return 0;
+	} catch (err) {
+		const msg = err instanceof Error ? err.message : String(err);
+		if (outputFormat === "json") process.stdout.write(formatOutput(null, "json", false, msg) + "\n");
+		else process.stderr.write(`Error: ${msg}\n`);
+		logger$3.error({ err }, "auto health failed");
+		return 1;
+	} finally {
+		try {
+			dbWrapper.close();
+		} catch {}
+	}
+}
 /**
 * Detect and apply supersessions after a phase completes in an amendment run.
 *
@@ -14869,6 +15182,15 @@ function registerAutoCommand(program, _version = "0.0.0", projectRoot = process.
 		});
 		process.exitCode = exitCode;
 	});
+	auto.command("health").description("Check pipeline health: process status, stall detection, and verdict").option("--run-id <id>", "Pipeline run ID to query (defaults to latest)").option("--project-root <path>", "Project root directory", projectRoot).option("--output-format <format>", "Output format: human (default) or json", "human").action(async (opts) => {
+		const outputFormat = opts.outputFormat === "json" ? "json" : "human";
+		const exitCode = await runAutoHealth({
+			outputFormat,
+			runId: opts.runId,
+			projectRoot: opts.projectRoot
+		});
+		process.exitCode = exitCode;
+	});
 	auto.command("amend").description("Run an amendment pipeline against a completed run and an existing run").option("--concept <text>", "Amendment concept description (inline)").option("--concept-file <path>", "Path to concept file").option("--run-id <id>", "Parent run ID (defaults to latest completed run)").option("--stop-after <phase>", "Stop pipeline after this phase completes").option("--from <phase>", "Start pipeline from this phase").option("--pack <name>", "Methodology pack name", "bmad").option("--project-root <path>", "Project root directory", projectRoot).option("--output-format <format>", "Output format: human (default) or json", "human").action(async (opts) => {
 		const exitCode = await runAmendCommand({
 			concept: opts.concept,

package/dist/index.d.ts

@@ -197,6 +197,40 @@ interface StoryLogEvent {
   /** Log message */
   msg: string;
 }
+/**
+ * Emitted periodically (every 30s) when no other progress events have fired.
+ * Allows parent agents to distinguish "working silently" from "stuck".
+ */
+interface PipelineHeartbeatEvent {
+  type: 'pipeline:heartbeat';
+  /** ISO-8601 timestamp generated at emit time */
+  ts: string;
+  /** Unique identifier for the current pipeline run */
+  run_id: string;
+  /** Number of sub-agent dispatches currently running */
+  active_dispatches: number;
+  /** Number of dispatches that have completed */
+  completed_dispatches: number;
+  /** Number of dispatches waiting to start */
+  queued_dispatches: number;
+}
+/**
+ * Emitted when the watchdog timer detects no progress for an extended period.
+ * Indicates a likely stall that may require operator intervention.
+ */
+interface StoryStallEvent {
+  type: 'story:stall';
+  /** ISO-8601 timestamp generated at emit time */
+  ts: string;
+  /** Unique identifier for the current pipeline run */
+  run_id: string;
+  /** Story key that appears stalled */
+  story_key: string;
+  /** Phase the story was in when the stall was detected */
+  phase: string;
+  /** Milliseconds since the last progress event */
+  elapsed_ms: number;
+}
 /**
  * Discriminated union of all pipeline event types.
  *
@@ -209,7 +243,7 @@ interface StoryLogEvent {
  * }
  * ```
  */
-type PipelineEvent = PipelineStartEvent | PipelineCompleteEvent | StoryPhaseEvent | StoryDoneEvent | StoryEscalationEvent | StoryWarnEvent | StoryLogEvent; //#endregion
+type PipelineEvent = PipelineStartEvent | PipelineCompleteEvent | StoryPhaseEvent | StoryDoneEvent | StoryEscalationEvent | StoryWarnEvent | StoryLogEvent | PipelineHeartbeatEvent | StoryStallEvent; //#endregion
 //#region src/core/errors.d.ts
 
 /**
@@ -793,6 +827,20 @@ interface OrchestratorEvents {
   'orchestrator:paused': Record<string, never>;
   /** Implementation orchestrator has been resumed */
   'orchestrator:resumed': Record<string, never>;
+  /** Periodic heartbeat emitted every 30s during pipeline execution */
+  'orchestrator:heartbeat': {
+    runId: string;
+    activeDispatches: number;
+    completedDispatches: number;
+    queuedDispatches: number;
+  };
+  /** Watchdog detected no progress for an extended period */
+  'orchestrator:stall': {
+    runId: string;
+    storyKey: string;
+    phase: string;
+    elapsedMs: number;
+  };
 }
 
 //#endregion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "substrate-ai",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "Substrate — multi-agent orchestration daemon for AI coding agents",
   "type": "module",
   "license": "MIT",