pi-crew 0.9.53 → 0.9.55

package/src/runtime/task-runner/context-retrieval.ts

@@ -0,0 +1,163 @@
+/**
+ * Iterative retrieval loop — workers progressively discover needed context.
+ *
+ * Pattern origin: ECC/skills/iterative-retrieval/SKILL.md — 4-phase loop:
+ * Dispatch → Evaluate → Refine → Loop. Max 3 cycles. Convergence when
+ * ≥3 high-relevance files found AND no critical gaps.
+ *
+ * This module provides the scoring and convergence logic.
+ * The actual file discovery is delegated to the caller (prompt-builder or task-runner).
+ */
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export interface RetrievalQuery {
+	patterns: string[];
+	keywords: string[];
+	excludes: string[];
+	focusAreas?: string[];
+}
+
+export interface RelevanceEvaluation {
+	path: string;
+	relevance: number; // 0.0–1.0
+	reason: string;
+	missingContext: string[];
+}
+
+export interface RetrievalResult {
+	query: RetrievalQuery;
+	evaluations: RelevanceEvaluation[];
+	cycle: number;
+	converged: boolean;
+}
+
+// ── Scoring ──────────────────────────────────────────────────────────────
+
+/**
+ * Score relevance of a file to a task description.
+ *
+ * Uses keyword matching as a heuristic. In production, this would be
+ * replaced by embedding-based similarity or BM25 scoring.
+ *
+ * @param filePath - Path to the file
+ * @param fileContent - Content of the file (or excerpt)
+ * @param keywords - Task-relevant keywords
+ * @returns Relevance score 0.0–1.0
+ */
+export function scoreRelevance(filePath: string, fileContent: string, keywords: string[]): number {
+	if (keywords.length === 0) return 0;
+
+	const pathLower = filePath.toLowerCase();
+	const contentLower = fileContent.toLowerCase();
+	let matchCount = 0;
+	let weightedScore = 0;
+
+	for (const keyword of keywords) {
+		const kw = keyword.toLowerCase();
+		// Path match is worth more (file naming is intentional)
+		if (pathLower.includes(kw)) {
+			matchCount++;
+			weightedScore += 0.3;
+		}
+		// Content match
+		const contentMatches = contentLower.split(kw).length - 1;
+		if (contentMatches > 0) {
+			matchCount++;
+			// Diminishing returns for repeated matches
+			weightedScore += Math.min(0.2, 0.05 * Math.log2(contentMatches + 1));
+		}
+	}
+
+	// Normalize: if all keywords matched, score is high
+	const keywordCoverage = matchCount / keywords.length;
+	const rawScore = keywordCoverage * 0.6 + Math.min(weightedScore, 0.4);
+
+	return Math.min(1.0, Math.max(0.0, rawScore));
+}
+
+// ── Convergence ──────────────────────────────────────────────────────────
+
+const CONVERGENCE_MIN_HIGH_RELEVANCE = 3;
+const HIGH_RELEVANCE_THRESHOLD = 0.7;
+
+/**
+ * Check if retrieval has converged — enough high-relevance files found.
+ *
+ * @param evaluations - Current relevance evaluations
+ * @returns true if converged
+ */
+export function hasConverged(evaluations: RelevanceEvaluation[]): boolean {
+	const highRelevance = evaluations.filter((e) => e.relevance >= HIGH_RELEVANCE_THRESHOLD);
+	if (highRelevance.length < CONVERGENCE_MIN_HIGH_RELEVANCE) return false;
+
+	// Check for critical gaps — any evaluation with empty missingContext
+	const criticalGaps = evaluations.some((e) => e.relevance < 0.3 && e.missingContext.length > 0);
+
+	return !criticalGaps;
+}
+
+// ── Refinement ───────────────────────────────────────────────────────────
+
+/**
+ * Refine a retrieval query based on evaluation results.
+ *
+ * Extracts new keywords from high-relevance files, adds discovered
+ * terminology, and excludes confirmed-irrelevant paths.
+ *
+ * @param query - Original query
+ * @param evaluations - Results from the current cycle
+ * @returns Refined query for the next cycle
+ */
+export function refineQuery(query: RetrievalQuery, evaluations: RelevanceEvaluation[]): RetrievalQuery {
+	const newKeywords = new Set(query.keywords);
+	const newExcludes = new Set(query.excludes);
+	const newFocusAreas = new Set(query.focusAreas ?? []);
+
+	for (const eval_ of evaluations) {
+		if (eval_.relevance >= HIGH_RELEVANCE_THRESHOLD) {
+			// Extract potential keywords from the file path
+			const parts = eval_.path.replace(/[\\/]/g, "/").split("/");
+			for (const part of parts) {
+				// Skip common non-informative segments
+				if (part.length > 2 && !["src", "lib", "test", "dist", "node_modules"].includes(part)) {
+					// Use the filename stem as a keyword hint
+					const stem = part.replace(/\.[^.]+$/, "").replace(/[.-]/g, " ");
+					for (const word of stem.split(/\s+/)) {
+						if (word.length > 3) newKeywords.add(word);
+					}
+				}
+			}
+		}
+
+		if (eval_.relevance < 0.2) {
+			// Exclude confirmed-irrelevant paths
+			newExcludes.add(eval_.path);
+		}
+
+		// Track missing context as focus areas
+		for (const gap of eval_.missingContext) {
+			newFocusAreas.add(gap);
+		}
+	}
+
+	return {
+		patterns: query.patterns, // patterns don't change
+		keywords: [...newKeywords],
+		excludes: [...newExcludes],
+		focusAreas: newFocusAreas.size > 0 ? [...newFocusAreas] : undefined,
+	};
+}
+
+// ── Loop Control ─────────────────────────────────────────────────────────
+
+const MAX_CYCLES = 3;
+
+/**
+ * Determine if another retrieval cycle should run.
+ */
+export function shouldContinue(evaluations: RelevanceEvaluation[], cycle: number): boolean {
+	if (cycle >= MAX_CYCLES) return false;
+	if (hasConverged(evaluations)) return false;
+	return true;
+}

package/src/runtime/task-runner/live-executor.ts

@@ -0,0 +1,227 @@
+import * as fs from "node:fs";
+import type { AgentConfig } from "../../agents/agent-config.ts";
+import type { CrewRuntimeConfig } from "../../config/config.ts";
+import { writeArtifact } from "../../state/artifact-store.ts";
+import { appendEventFireAndForget } from "../../state/event-log.ts";
+import { loadRunManifestById } from "../../state/state-store.ts";
+import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../../state/types.ts";
+import type { WorkflowStep } from "../../workflows/workflow-config.ts";
+import {
+	appendCrewAgentEvent,
+	appendCrewAgentOutput,
+	emptyCrewAgentProgress,
+	recordFromTask,
+	upsertCrewAgent,
+} from "../crew-agent-records.ts";
+import { runLiveSessionTask } from "../live-session-runtime.ts";
+import type { ParsedPiJsonOutput } from "../pi-json-output.ts";
+import { type ProgressEventSummary, shouldAppendProgressEventUpdate } from "../progress-event-coalescer.ts";
+import { createWorkerHeartbeat, touchWorkerHeartbeat } from "../worker-heartbeat.ts";
+import { createStartupEvidence, type WorkerStartupEvidence } from "../worker-startup.ts";
+import { applyAgentProgressEvent, applyUsageToProgress, progressEventSummary, shouldFlushProgressEvent } from "./progress.ts";
+import { persistSingleTaskUpdate } from "./state-helpers.ts";
+
+export interface RunLiveTaskInput {
+	manifest: TeamRunManifest;
+	tasks: TeamTaskState[];
+	task: TeamTaskState;
+	step: WorkflowStep;
+	agent: AgentConfig;
+	prompt: string;
+	signal?: AbortSignal;
+	runtimeConfig?: CrewRuntimeConfig;
+	parentContext?: string;
+	parentModel?: unknown;
+	modelRegistry?: unknown;
+	modelOverride?: string;
+	teamRoleModel?: string;
+	isCurrent?: () => boolean;
+	/** Workspace where this task run was initiated — used for session-scoped live-agent visibility. */
+	workspaceId: string;
+	/** Phase 2: Output schema for validating yield data. */
+	outputSchema?: unknown;
+}
+
+export interface RunLiveTaskOutput {
+	task: TeamTaskState;
+	tasks: TeamTaskState[];
+	startupEvidence: WorkerStartupEvidence;
+	exitCode: number | null;
+	error?: string;
+	parsedOutput?: ParsedPiJsonOutput;
+	resultArtifact: ArtifactDescriptor;
+	logArtifact?: ArtifactDescriptor;
+	transcriptArtifact?: ArtifactDescriptor;
+}
+
+function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): TeamTaskState[] {
+	return tasks.map((task) => (task.id === updated.id ? updated : task));
+}
+
+export async function runLiveTask(input: RunLiveTaskInput): Promise<RunLiveTaskOutput> {
+	const { manifest, step, agent, prompt } = input;
+	let task = input.task;
+	let tasks = input.tasks;
+	const transcriptPath = `${manifest.artifactsRoot}/transcripts/${task.id}.jsonl`;
+	const isCurrent =
+		input.isCurrent ??
+		(() => {
+			if (input.signal?.aborted) return false;
+			const loaded = loadRunManifestById(manifest.cwd, manifest.runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency;
+			const currentTask = loaded?.tasks.find((item) => item.id === task.id);
+			if (loaded?.manifest.status && loaded.manifest.status !== "running") return false;
+			return currentTask ? currentTask.status === "running" : true;
+		});
+	let lastAgentRecordPersistedAt = 0;
+	let lastRunProgressPersistedAt = 0;
+	let lastHeartbeatPersistedAt = 0;
+	let lastRunProgressSummary: ProgressEventSummary | undefined;
+	const persistLiveProgress = (event: unknown, force = false): void => {
+		if (!isCurrent()) return;
+		const now = Date.now();
+		if (force || shouldFlushProgressEvent(event) || now - lastAgentRecordPersistedAt >= 500) {
+			upsertCrewAgent(manifest, recordFromTask(manifest, task, "live-session"));
+			lastAgentRecordPersistedAt = now;
+		}
+		// Bug #5 fix: also persist heartbeat to tasks.json so heartbeat-watcher can detect live-session activity
+		if (force || now - lastHeartbeatPersistedAt >= 1000) {
+			task = {
+				...task,
+				heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id)),
+			};
+			tasks = persistSingleTaskUpdate(manifest, tasks, task);
+			lastHeartbeatPersistedAt = now;
+		}
+		const summary = progressEventSummary(task, event);
+		const decision = shouldAppendProgressEventUpdate({
+			previous: lastRunProgressSummary,
+			next: summary,
+			nowMs: now,
+			lastAppendMs: lastRunProgressPersistedAt || undefined,
+			minIntervalMs: 1000,
+			force,
+		});
+		if (decision.shouldAppend) {
+			// 2.2 caller migration: see task-runner.ts.
+			appendEventFireAndForget(manifest.eventsPath, {
+				type: "task.progress",
+				runId: manifest.runId,
+				taskId: task.id,
+				data: { ...summary, coalesceReason: decision.reason },
+			});
+			lastRunProgressSummary = summary;
+			lastRunProgressPersistedAt = now;
+		}
+	};
+	const attemptStartedAt = new Date();
+	// Apply agent-level maxTurns override if specified (only positive integers)
+	const agentMax = agent.maxTurns;
+	const effectiveRuntimeConfig =
+		agentMax != null && agentMax > 0 && (!input.runtimeConfig?.maxTurns || agentMax < input.runtimeConfig.maxTurns)
+			? { ...input.runtimeConfig, maxTurns: agentMax }
+			: input.runtimeConfig;
+	const liveResult = await runLiveSessionTask({
+		manifest,
+		task,
+		step,
+		agent,
+		prompt,
+		signal: input.signal,
+		transcriptPath,
+		runtimeConfig: effectiveRuntimeConfig,
+		parentContext: input.parentContext,
+		parentModel: input.parentModel,
+		modelRegistry: input.modelRegistry,
+		modelOverride: input.modelOverride,
+		teamRoleModel: input.teamRoleModel,
+		isCurrent,
+		workspaceId: input.workspaceId,
+		// Phase 2: Pass output schema for yield validation
+		outputSchema: undefined,
+		onOutput: (text) => {
+			if (!isCurrent()) return;
+			appendCrewAgentOutput(manifest, task.id, text);
+		},
+		onEvent: (event) => {
+			if (!isCurrent()) return;
+			appendCrewAgentEvent(manifest, task.id, event);
+			task = {
+				...task,
+				agentProgress: applyAgentProgressEvent(task.agentProgress ?? emptyCrewAgentProgress(), event, task.startedAt),
+			};
+			tasks = updateTask(tasks, task);
+			persistLiveProgress(event);
+		},
+	});
+	const startupEvidence = createStartupEvidence({
+		command: "live-session",
+		startedAt: attemptStartedAt,
+		finishedAt: new Date(),
+		promptSentAt: attemptStartedAt,
+		promptAccepted: liveResult.exitCode === 0 && !liveResult.error,
+		stderr: liveResult.stderr,
+		error: liveResult.error,
+		exitCode: liveResult.exitCode,
+	});
+	const exitCode = liveResult.exitCode;
+	const error =
+		liveResult.error ||
+		(liveResult.exitCode && liveResult.exitCode !== 0
+			? liveResult.stderr || `Live session exited with ${liveResult.exitCode}`
+			: undefined);
+	const parsedOutput = {
+		finalText: liveResult.stdout,
+		textEvents: liveResult.stdout ? [liveResult.stdout] : [],
+		jsonEvents: liveResult.jsonEvents,
+		usage: liveResult.usage,
+	};
+	if (liveResult.usage && isCurrent())
+		task = {
+			...task,
+			usage: liveResult.usage,
+			agentProgress: applyUsageToProgress(task.agentProgress, liveResult.usage),
+		};
+	persistLiveProgress({ type: "attempt_finished" }, true);
+	const resultArtifact = writeArtifact(manifest.artifactsRoot, {
+		kind: "result",
+		relativePath: `results/${task.id}.txt`,
+		content: liveResult.stdout || liveResult.stderr || "(no output)",
+		producer: task.id,
+	});
+	const logArtifact = writeArtifact(manifest.artifactsRoot, {
+		kind: "log",
+		relativePath: `logs/${task.id}.log`,
+		content: [
+			`runtime=live-session`,
+			`finalExitCode=${exitCode ?? "null"}`,
+			`jsonEvents=${liveResult.jsonEvents}`,
+			liveResult.usage ? `usage=${JSON.stringify(liveResult.usage)}` : "",
+			"",
+			"STDOUT:",
+			liveResult.stdout,
+			"",
+			"STDERR:",
+			liveResult.stderr,
+		].join("\n"),
+		producer: task.id,
+	});
+	const transcriptArtifact = fs.existsSync(transcriptPath)
+		? writeArtifact(manifest.artifactsRoot, {
+				kind: "log",
+				relativePath: `transcripts/${task.id}.jsonl`,
+				content: fs.readFileSync(transcriptPath, "utf-8"),
+				producer: task.id,
+			})
+		: undefined;
+	return {
+		task,
+		tasks,
+		startupEvidence,
+		exitCode,
+		error: error || undefined,
+		parsedOutput,
+		resultArtifact,
+		logArtifact,
+		transcriptArtifact,
+	};
+}

package/src/runtime/task-runner/output-splitter.ts

@@ -0,0 +1,152 @@
+/**
+ * output-splitter.ts — partition one worker's combined output into N
+ * per-task results (M6 real-dispatch, MVP).
+ *
+ * The combined worker is told to wrap each task's result in
+ * `<<<TASK_RESULT:{taskId}>>>` ... `<<<END_TASK_RESULT>>>` delimiters.
+ * This module parses the raw output and returns a Map<taskId, resultText>.
+ *
+ * Strategy (in priority order):
+ *   1. Delimiter parse — exact match of `<<<TASK_RESULT:id>>>` markers
+ *   2. Section heading parse — `### Task N of M` / `## Task N` headers
+ *   3. Whole-output broadcast — assign full raw output to ALL taskIds
+ *      (safe fallback for read-only roles where identical context is OK)
+ *
+ * Returns an array of { taskId, text } records in input-order. When the
+ * delimiter strategy succeeds, each entry contains that task's delimited
+ * text. When broadcast fallback is used, all entries contain the full
+ * raw output.
+ *
+ * Note: pure function — no I/O, no LLM calls, deterministic. This is
+ * deliberately the simplest possible splitter; we don't try fuzzy
+ * matching, NL extraction, or retry-the-LLM. The MVP trusts workers
+ * to follow delimiters 99% of the time and broadcasts the rest.
+ */
+
+export interface SplitResult {
+	taskId: string;
+	text: string;
+	/** Which strategy produced this entry. */
+	strategy: "delimiter" | "section" | "broadcast";
+}
+
+/**
+ * Split a raw worker output into N per-task result texts.
+ *
+ * @param rawOutput  — the worker's combined output text
+ * @param taskIds    — ordered list of task IDs (matches the dispatch order)
+ */
+export function splitCoalescedOutput(rawOutput: string, taskIds: string[]): SplitResult[] {
+	if (taskIds.length === 0) return [];
+	if (taskIds.length === 1) {
+		// Single-task group: return the whole output as the one task's result.
+		return [{ taskId: taskIds[0]!, text: rawOutput, strategy: "delimiter" }];
+	}
+
+	const byId = new Map<string, string>();
+	const delimiterHits = new Set<string>();
+
+	// Strategy 1: delimiter parse.
+	// Pattern: <<<TASK_RESULT:{id}>>> ... <<<END_TASK_RESULT>>>
+	// We match greedily, allowing nested markers between start and end.
+	const delimiterRegex = /<<<TASK_RESULT:([^\s>]+)>>>([\s\S]*?)<<<END_TASK_RESULT>>>/g;
+	let match: RegExpExecArray | null;
+	while ((match = delimiterRegex.exec(rawOutput)) !== null) {
+		const id = match[1]!;
+		const body = match[2]!.trim();
+		if (taskIds.includes(id) && !byId.has(id)) {
+			byId.set(id, body);
+			delimiterHits.add(id);
+		}
+	}
+
+	if (delimiterHits.size === taskIds.length) {
+		// All tasks got delimiter hits — return in input order.
+		return taskIds.map((id) => ({
+			taskId: id,
+			text: byId.get(id) ?? "",
+			strategy: "delimiter" as const,
+		}));
+	}
+
+	// Strategy 2: section heading parse.
+	// Look for `### Task N of M` or `## Task {id}` markers. We try both.
+	if (delimiterHits.size === 0) {
+		const bySection = parseBySectionHeadings(rawOutput, taskIds);
+		if (bySection.size === taskIds.length) {
+			return taskIds.map((id) => ({
+				taskId: id,
+				text: bySection.get(id) ?? "",
+				strategy: "section" as const,
+			}));
+		}
+	}
+
+	// Strategy 3: broadcast fallback. All tasks get the full raw output.
+	// This is safe for read-only roles (explorers, reviewers) where
+	// downstream consumers care about findings, not strict per-task partitioning.
+	return taskIds.map((id) => ({
+		taskId: id,
+		text: rawOutput,
+		strategy: "broadcast" as const,
+	}));
+}
+
+/**
+ * Strategy 2 implementation: parse sections by heading patterns.
+ *
+ * Matches `### Task N of M` (N=1..M) or `### Task N` headers, then
+ * captures text up to the next header or end-of-input.
+ *
+ * Falls back to "first heading wins all text" if no N-of-M numbering is
+ * present, which is common in worker output that just labels sections.
+ */
+function parseBySectionHeadings(rawOutput: string, taskIds: string[]): Map<string, string> {
+	const result = new Map<string, string>();
+	// First try numbered: "### Task 1 of N" / "## Task 1" / "### Task 1"
+	const numberedRegex = /^(#{2,3})\s+Task\s+(\d+)\s+of\s+\d+.*$/gm;
+	const sections: { num: number; start: number }[] = [];
+	let m: RegExpExecArray | null;
+	while ((m = numberedRegex.exec(rawOutput)) !== null) {
+		sections.push({ num: Number(m[2]), start: m.index + m[0].length });
+	}
+	if (sections.length >= taskIds.length) {
+		sections.sort((a, b) => a.start - b.start);
+		for (let i = 0; i < sections.length && i < taskIds.length; i += 1) {
+			const section = sections[i]!;
+			const end = sections[i + 1]?.start ?? rawOutput.length;
+			const text = rawOutput.slice(section.start, end).trim();
+			const taskId = taskIds[i];
+			if (taskId) result.set(taskId, text);
+		}
+		return result;
+	}
+	// Try "### Task {taskId}" headers — direct ID matching.
+	const idHeaderRegex = /^(#{2,3})\s+Task\s+([^\s].*?)$/gm;
+	while ((m = idHeaderRegex.exec(rawOutput)) !== null) {
+		const headerText = m[2]!.trim();
+		const matchedId = taskIds.find((id) => headerText.includes(id));
+		if (matchedId && !result.has(matchedId)) {
+			result.set(matchedId, ""); // placeholder; filled below
+		}
+	}
+	if (result.size === taskIds.length) {
+		// Re-scan to extract body per matched header
+		const headerPositions: { taskId: string; start: number }[] = [];
+		const idHeaderRegex2 = /^(#{2,3})\s+Task\s+([^\s].*?)$/gm;
+		while ((m = idHeaderRegex2.exec(rawOutput)) !== null) {
+			const headerText = m[2]!.trim();
+			const matchedId = taskIds.find((id) => headerText.includes(id));
+			if (matchedId) headerPositions.push({ taskId: matchedId, start: m.index + m[0].length });
+		}
+		headerPositions.sort((a, b) => a.start - b.start);
+		for (let i = 0; i < headerPositions.length; i += 1) {
+			const cur = headerPositions[i]!;
+			const next = headerPositions[i + 1];
+			const end = next ? rawOutput.lastIndexOf("\n#", next.start) : rawOutput.length;
+			const text = rawOutput.slice(cur.start, end > 0 ? end : rawOutput.length).trim();
+			result.set(cur.taskId, text);
+		}
+	}
+	return result;
+}