substrate-ai 0.2.11 → 0.2.13

package/dist/cli/index.js

@@ -2,7 +2,7 @@
 import { createLogger, deepMask } from "../logger-C6n1g8uP.js";
 import { AdapterRegistry, createEventBus } from "../event-bus-J-bw-pkp.js";
 import { CURRENT_CONFIG_FORMAT_VERSION, CURRENT_TASK_GRAPH_VERSION, PartialSubstrateConfigSchema, SUPPORTED_CONFIG_FORMAT_VERSIONS, SubstrateConfigSchema, defaultConfigMigrator } from "../version-manager-impl-BpVx2DkY.js";
-import { DatabaseWrapper, SUBSTRATE_OWNED_SETTINGS_KEYS, VALID_PHASES, buildPipelineStatusOutput, createContextCompiler, createDispatcher, createImplementationOrchestrator, createPackLoader, createPhaseOrchestrator, createStopAfterGate, findPackageRoot, formatOutput, formatPhaseCompletionSummary, formatPipelineStatusHuman, formatPipelineSummary, formatTokenTelemetry, getSubstrateDefaultSettings, parseDbTimestampAsUtc, registerRunCommand, resolveBmadMethodSrcPath, resolveBmadMethodVersion, resolveMainRepoRoot, runAnalysisPhase, runMigrations, runPlanningPhase, runSolutioningPhase, validateStopAfterFromConflict } from "../run-L-R_XYNT.js";
+import { DatabaseWrapper, SUBSTRATE_OWNED_SETTINGS_KEYS, VALID_PHASES, buildPipelineStatusOutput, createContextCompiler, createDispatcher, createImplementationOrchestrator, createPackLoader, createPhaseOrchestrator, createStopAfterGate, findPackageRoot, formatOutput, formatPhaseCompletionSummary, formatPipelineStatusHuman, formatPipelineSummary, formatTokenTelemetry, getSubstrateDefaultSettings, parseDbTimestampAsUtc, registerRunCommand, resolveBmadMethodSrcPath, resolveBmadMethodVersion, resolveMainRepoRoot, runAnalysisPhase, runMigrations, runPlanningPhase, runSolutioningPhase, validateStopAfterFromConflict } from "../run-CoP8UQU3.js";
 import { ConfigError, ConfigIncompatibleFormatError } from "../errors-BPqtzQ4U.js";
 import { addTokenUsage, createDecision, getDecisionsByPhaseForRun, getLatestRun, getPipelineRunById, getTokenUsageSummary, listRequirements, updatePipelineRun } from "../decisions-DNYByk0U.js";
 import { aggregateTokenUsageForRun, compareRunMetrics, getBaselineRunMetrics, getRunMetrics, getStoryMetricsForRun, incrementRunRestarts, listRunMetrics, tagRunAsBaseline } from "../metrics-BSg8VIHd.js";
@@ -2414,16 +2414,24 @@ const DEFAULT_STALL_THRESHOLD_SECONDS = 600;
 *   - `node dist/cli/index.js run` (npm run substrate:dev)
 *   - `npx substrate run`
 *   - any node process whose command contains `run` with `--events` or `--stories`
+*
+* When `projectRoot` is provided, additionally checks that the command line
+* contains that path (via `--project-root` flag or as part of the binary/CWD path).
+* This ensures multi-project environments match the correct orchestrator.
 */
-function isOrchestratorProcessLine(line) {
+function isOrchestratorProcessLine(line, projectRoot) {
 	if (line.includes("grep")) return false;
-	if (line.includes("substrate run")) return true;
-	if (line.includes("substrate-ai run")) return true;
-	if (line.includes("index.js run")) return true;
-	if (line.includes("node") && /\srun(\s|$)/.test(line) && (line.includes("--events") || line.includes("--stories"))) return true;
-	return false;
-}
-function inspectProcessTree(execFileSyncOverride) {
+	let isOrchestrator = false;
+	if (line.includes("substrate run")) isOrchestrator = true;
+	else if (line.includes("substrate-ai run")) isOrchestrator = true;
+	else if (line.includes("index.js run")) isOrchestrator = true;
+	else if (line.includes("node") && /\srun(\s|$)/.test(line) && (line.includes("--events") || line.includes("--stories"))) isOrchestrator = true;
+	if (!isOrchestrator) return false;
+	if (projectRoot !== void 0) return line.includes(projectRoot);
+	return true;
+}
+function inspectProcessTree(opts) {
+	const { projectRoot, execFileSync: execFileSyncOverride } = opts ?? {};
 	const result = {
 		orchestrator_pid: null,
 		child_pids: [],
@@ -2443,7 +2451,7 @@ function inspectProcessTree(execFileSyncOverride) {
 			});
 		}
 		const lines = psOutput.split("\n");
-		for (const line of lines) if (isOrchestratorProcessLine(line)) {
+		for (const line of lines) if (isOrchestratorProcessLine(line, projectRoot)) {
 			const match = line.trim().match(/^(\d+)/);
 			if (match) {
 				result.orchestrator_pid = parseInt(match[1], 10);
@@ -2466,6 +2474,58 @@ function inspectProcessTree(execFileSyncOverride) {
 	return result;
 }
 /**
+* Collect all descendant PIDs of the given root PIDs by walking the process
+* tree recursively. This ensures that grandchildren of the orchestrator
+* (e.g. node subprocesses spawned by `claude -p`) are also killed during
+* stall recovery, leaving no orphan processes.
+*
+* Returns only the descendants — the root PIDs themselves are NOT included.
+*/
+function getAllDescendantPids(rootPids, execFileSyncOverride) {
+	if (rootPids.length === 0) return [];
+	try {
+		let psOutput;
+		if (execFileSyncOverride !== void 0) psOutput = execFileSyncOverride("ps", ["-eo", "pid,ppid"], {
+			encoding: "utf-8",
+			timeout: 5e3
+		});
+		else {
+			const { execFileSync } = __require("node:child_process");
+			psOutput = execFileSync("ps", ["-eo", "pid,ppid"], {
+				encoding: "utf-8",
+				timeout: 5e3
+			});
+		}
+		const childrenOf = new Map();
+		for (const line of psOutput.split("\n")) {
+			const parts = line.trim().split(/\s+/);
+			if (parts.length >= 2) {
+				const pid = parseInt(parts[0], 10);
+				const ppid = parseInt(parts[1], 10);
+				if (!isNaN(pid) && !isNaN(ppid) && pid > 0) {
+					if (!childrenOf.has(ppid)) childrenOf.set(ppid, []);
+					childrenOf.get(ppid).push(pid);
+				}
+			}
+		}
+		const descendants = [];
+		const seen = new Set(rootPids);
+		const queue = [...rootPids];
+		while (queue.length > 0) {
+			const current = queue.shift();
+			const children = childrenOf.get(current) ?? [];
+			for (const child of children) if (!seen.has(child)) {
+				seen.add(child);
+				descendants.push(child);
+				queue.push(child);
+			}
+		}
+		return descendants;
+	} catch {
+		return [];
+	}
+}
+/**
 * Fetch pipeline health data as a structured object without any stdout side-effects.
 * Used by runSupervisorAction to poll health without formatting overhead.
 *
@@ -2524,10 +2584,11 @@ async function getAutoHealthData(options) {
 				}
 			}
 		} catch {}
-		const processInfo = inspectProcessTree();
+		const processInfo = inspectProcessTree({ projectRoot });
 		let verdict = "NO_PIPELINE_RUNNING";
 		if (run.status === "running") if (processInfo.orchestrator_pid === null && active === 0 && completed > 0) verdict = "NO_PIPELINE_RUNNING";
 		else if (processInfo.zombies.length > 0) verdict = "STALLED";
+		else if (processInfo.orchestrator_pid !== null && processInfo.child_pids.length > 0 && stalenessSeconds > DEFAULT_STALL_THRESHOLD_SECONDS) verdict = "HEALTHY";
 		else if (stalenessSeconds > DEFAULT_STALL_THRESHOLD_SECONDS) verdict = "STALLED";
 		else if (processInfo.orchestrator_pid !== null && processInfo.child_pids.length === 0 && active > 0) verdict = "STALLED";
 		else verdict = "HEALTHY";
@@ -2660,6 +2721,7 @@ function defaultSupervisorDeps() {
 				};
 			}
 		},
+		getAllDescendants: (rootPids) => getAllDescendantPids(rootPids),
 		runAnalysis: async (runId, projectRoot) => {
 			const dbPath = join(projectRoot, ".substrate", "substrate.db");
 			if (!existsSync(dbPath)) return;
@@ -2701,7 +2763,7 @@ function defaultSupervisorDeps() {
 */
 async function runSupervisorAction(options, deps = {}) {
 	const { pollInterval, stallThreshold, maxRestarts, outputFormat, projectRoot, runId, pack, experiment, maxExperiments } = options;
-	const { getHealth, killPid, resumePipeline, sleep, incrementRestarts, runAnalysis, getTokenSnapshot } = {
+	const { getHealth, killPid, resumePipeline, sleep, incrementRestarts, runAnalysis, getTokenSnapshot, getAllDescendants } = {
 		...defaultSupervisorDeps(),
 		...deps
 	};
@@ -2833,7 +2895,7 @@ async function runSupervisorAction(options, deps = {}) {
 								const expDb = expDbWrapper.db;
 								const { runRunAction: runPipeline } = await import(
 									/* @vite-ignore */
-									"../run-C8aOWnKG.js"
+									"../run-B9IglY4m.js"
 );
 								const runStoryFn = async (opts) => {
 									const exitCode = await runPipeline({
@@ -2899,7 +2961,10 @@ async function runSupervisorAction(options, deps = {}) {
 			return failed.length > 0 || escalated.length > 0 ? 1 : 0;
 		}
 		if (health.staleness_seconds >= stallThreshold) {
-			const pids = [...health.process.orchestrator_pid !== null ? [health.process.orchestrator_pid] : [], ...health.process.child_pids];
+			const directPids = [...health.process.orchestrator_pid !== null ? [health.process.orchestrator_pid] : [], ...health.process.child_pids];
+			const descendantPids = getAllDescendants(directPids);
+			const directPidSet = new Set(directPids);
+			const pids = [...directPids, ...descendantPids.filter((p) => !directPidSet.has(p))];
 			emitEvent({
 				type: "supervisor:kill",
 				run_id: health.run_id,

package/dist/index.d.ts

@@ -1025,6 +1025,8 @@ interface OrchestratorEvents {
     storyKey: string;
     phase: string;
     elapsedMs: number;
+    /** PID of the stalled child process, or null if not tracked */
+    childPid: number | null;
   };
   /** Readiness check has completed — emitted for all verdicts (READY, NEEDS_WORK, NOT_READY) */
   'solutioning:readiness-check': {

package/dist/{run-C8aOWnKG.js → run-B9IglY4m.js}

@@ -1,6 +1,6 @@
 import "./logger-C6n1g8uP.js";
 import "./event-bus-J-bw-pkp.js";
-import { registerRunCommand, runRunAction } from "./run-L-R_XYNT.js";
+import { registerRunCommand, runRunAction } from "./run-CoP8UQU3.js";
 import "./decisions-DNYByk0U.js";
 import "./metrics-BSg8VIHd.js";
 

package/dist/{run-L-R_XYNT.js → run-CoP8UQU3.js}

@@ -1202,6 +1202,15 @@ function buildPipelineStatusOutput(run, tokenSummary, decisionsCount, storiesCou
 		totalOutput += row.total_output_tokens;
 		totalCost += row.total_cost_usd;
 	}
+	let activeDispatches = 0;
+	try {
+		if (run.token_usage_json) {
+			const state = JSON.parse(run.token_usage_json);
+			if (state.stories) {
+				for (const s of Object.values(state.stories)) if (s.phase !== "PENDING" && s.phase !== "COMPLETE" && s.phase !== "ESCALATED") activeDispatches++;
+			}
+		}
+	} catch {}
 	return {
 		run_id: run.id,
 		current_phase: currentPhase,
@@ -1214,7 +1223,9 @@ function buildPipelineStatusOutput(run, tokenSummary, decisionsCount, storiesCou
 		decisions_count: decisionsCount,
 		stories_count: storiesCount,
 		last_activity: run.updated_at,
-		staleness_seconds: Math.round((Date.now() - parseDbTimestampAsUtc(run.updated_at).getTime()) / 1e3)
+		staleness_seconds: Math.round((Date.now() - parseDbTimestampAsUtc(run.updated_at).getTime()) / 1e3),
+		last_event_ts: run.updated_at,
+		active_dispatches: activeDispatches
 	};
 }
 /**
@@ -5359,6 +5370,7 @@ function createImplementationOrchestrator(deps) {
 	let _heartbeatTimer = null;
 	const HEARTBEAT_INTERVAL_MS = 3e4;
 	const WATCHDOG_TIMEOUT_MS = 6e5;
+	const _stalledStories = new Set();
 	const _phaseStartMs = new Map();
 	const _phaseEndMs = new Map();
 	const _storyDispatches = new Map();
@@ -5454,6 +5466,7 @@ function createImplementationOrchestrator(deps) {
 	}
 	function recordProgress() {
 		_lastProgressTs = Date.now();
+		_stalledStories.clear();
 	}
 	function startHeartbeat() {
 		if (_heartbeatTimer !== null) return;
@@ -5465,7 +5478,8 @@ function createImplementationOrchestrator(deps) {
 			for (const s of _stories.values()) if (s.phase === "COMPLETE" || s.phase === "ESCALATED") completed++;
 			else if (s.phase === "PENDING") queued++;
 			else active++;
-			eventBus.emit("orchestrator:heartbeat", {
+			const timeSinceProgress = Date.now() - _lastProgressTs;
+			if (timeSinceProgress >= HEARTBEAT_INTERVAL_MS) eventBus.emit("orchestrator:heartbeat", {
 				runId: config.pipelineRunId ?? "",
 				activeDispatches: active,
 				completedDispatches: completed,
@@ -5474,6 +5488,8 @@ function createImplementationOrchestrator(deps) {
 			const elapsed = Date.now() - _lastProgressTs;
 			if (elapsed >= WATCHDOG_TIMEOUT_MS) {
 				for (const [key, s] of _stories) if (s.phase !== "PENDING" && s.phase !== "COMPLETE" && s.phase !== "ESCALATED") {
+					if (_stalledStories.has(key)) continue;
+					_stalledStories.add(key);
 					logger$16.warn({
 						storyKey: key,
 						phase: s.phase,
@@ -5483,7 +5499,8 @@ function createImplementationOrchestrator(deps) {
 						runId: config.pipelineRunId ?? "",
 						storyKey: key,
 						phase: s.phase,
-						elapsedMs: elapsed
+						elapsedMs: elapsed,
+						childPid: null
 					});
 				}
 			}
@@ -6244,7 +6261,7 @@ function createImplementationOrchestrator(deps) {
 		});
 		persistState();
 		recordProgress();
-		startHeartbeat();
+		if (config.enableHeartbeat) startHeartbeat();
 		if (projectRoot !== void 0) {
 			const seedResult = seedMethodologyContext(db, projectRoot);
 			if (seedResult.decisionsCreated > 0) logger$16.info({
@@ -10697,7 +10714,8 @@ async function runRunAction(options) {
 					run_id: payload.runId,
 					story_key: payload.storyKey,
 					phase: payload.phase,
-					elapsed_ms: payload.elapsedMs
+					elapsed_ms: payload.elapsedMs,
+					child_pid: payload.childPid
 				});
 			});
 		}
@@ -10710,7 +10728,8 @@ async function runRunAction(options) {
 			config: {
 				maxConcurrency: concurrency,
 				maxReviewCycles: 2,
-				pipelineRunId: pipelineRun.id
+				pipelineRunId: pipelineRun.id,
+				enableHeartbeat: eventsFlag === true
 			},
 			projectRoot
 		});
@@ -11153,4 +11172,4 @@ function registerRunCommand(program, _version = "0.0.0", projectRoot = process.c
 
 //#endregion
 export { DatabaseWrapper, SUBSTRATE_OWNED_SETTINGS_KEYS, VALID_PHASES, buildPipelineStatusOutput, createContextCompiler, createDispatcher, createImplementationOrchestrator, createPackLoader, createPhaseOrchestrator, createStopAfterGate, findPackageRoot, formatOutput, formatPhaseCompletionSummary, formatPipelineStatusHuman, formatPipelineSummary, formatTokenTelemetry, getSubstrateDefaultSettings, parseDbTimestampAsUtc, registerRunCommand, resolveBmadMethodSrcPath, resolveBmadMethodVersion, resolveMainRepoRoot, runAnalysisPhase, runMigrations, runPlanningPhase, runRunAction, runSolutioningPhase, validateStopAfterFromConflict };
-//# sourceMappingURL=run-L-R_XYNT.js.map
+//# sourceMappingURL=run-CoP8UQU3.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "substrate-ai",
-  "version": "0.2.11",
+  "version": "0.2.13",
   "description": "Substrate — multi-agent orchestration daemon for AI coding agents",
   "type": "module",
   "license": "MIT",