pi-blackhole 0.3.8 → 0.4.0

package/src/extract/commits.ts

@@ -18,41 +18,84 @@ const firstLineOf = (text: string): string => {
 const cleanMessage = (msg: string): string =>
   msg.replace(/\\"/g, '"').replace(/\\'/g, "'").trim();
 
+/** Extract commit hash from git output text (tool_result or bash output). */
+const extractHashFromOutput = (text: string): string | undefined => {
+  const bracket = text.match(/\[\S+\s+([0-9a-f]{7,12})\]/);
+  if (bracket) return bracket[1];
+  const range = text.match(/\b([0-9a-f]{7,12})\.\.([0-9a-f]{7,12})\b/);
+  if (range) return range[2];
+  const plain = text.match(HASH_RE);
+  if (plain) return plain[1];
+  return undefined;
+};
+
+/** Try to extract a commit message from a git commit command string. */
+const tryExtractMessage = (cmd: string): string | undefined => {
+  if (!/\bgit\s+commit\b/.test(cmd)) return undefined;
+  const m = cmd.match(COMMIT_MSG_RE);
+  if (!m) return undefined;
+  const message = firstLineOf(cleanMessage(m[1] ?? m[2] ?? m[3] ?? ""));
+  return message || undefined;
+};
+
 /**
- * Extract git commits from bash tool calls (`git commit -m "..."`) and pair
- * with hash from the immediately following tool_result.
+ * Extract git commits from bash tool calls, bash execution messages,
+ * and user messages that wrap bash execution output (post-convertToLlm).
+ *
+ * Handles three block kinds:
+ * - tool_call (name: "bash") — agent tool call to the bash tool
+ * - bash — pi's internal bashExecution message
+ * - user — convertToLlm wraps bashExecution as "Ran `cmd`\n```\noutput\n```"
  */
 export const extractCommits = (blocks: NormalizedBlock[]): CommitInfo[] => {
   const commits: CommitInfo[] = [];
+  const addCommit = (hash: string | undefined, message: string) => {
+    const key = `${hash ?? ""}::${message}`;
+    if (!commits.some((c) => `${c.hash ?? ""}::${c.message}` === key)) {
+      commits.push({ hash, message });
+    }
+  };
 
   for (let i = 0; i < blocks.length; i++) {
     const b = blocks[i];
-    if (b.kind !== "tool_call" || b.name !== "bash") continue;
-    const cmd = typeof b.args.command === "string" ? b.args.command : "";
-    if (!/\bgit\s+commit\b/.test(cmd)) continue;
-    const m = cmd.match(COMMIT_MSG_RE);
-    if (!m) continue;
-    const message = firstLineOf(cleanMessage(m[1] ?? m[2] ?? m[3] ?? ""));
-    if (!message) continue;
 
-    let hash: string | undefined;
-    // Look at next tool_result for hash
-    for (let j = i + 1; j < Math.min(blocks.length, i + 3); j++) {
-      const r = blocks[j];
-      if (r.kind !== "tool_result") continue;
-      // Common git commit output: `[branch <hash>] message` or `<branch> <hash>..<hash>`
-      const bracket = r.text.match(/\[\S+\s+([0-9a-f]{7,12})\]/);
-      if (bracket) { hash = bracket[1]; break; }
-      const range = r.text.match(/\b([0-9a-f]{7,12})\.\.([0-9a-f]{7,12})\b/);
-      if (range) { hash = range[2]; break; }
-      const plain = r.text.match(HASH_RE);
-      if (plain) { hash = plain[1]; break; }
+    // ── Case 1: tool_call (agent calls bash tool) ──
+    if (b.kind === "tool_call" && b.name === "bash") {
+      const cmd = (b.args && typeof b.args.command === "string") ? b.args.command : "";
+      const message = tryExtractMessage(cmd);
+      if (!message) continue;
+
+      let hash: string | undefined;
+      for (let j = i + 1; j < Math.min(blocks.length, i + 3); j++) {
+        const r = blocks[j];
+        if (r.kind !== "tool_result") continue;
+        hash = extractHashFromOutput(r.text);
+        if (hash) break;
+      }
+      addCommit(hash, message);
+      continue;
     }
 
-    // Dedup by message+hash
-    const key = `${hash ?? ""}::${message}`;
-    if (!commits.some((c) => `${c.hash ?? ""}::${c.message}` === key)) {
-      commits.push({ hash, message });
+    // ── Case 2: bash execution message ──
+    if (b.kind === "bash") {
+      const message = tryExtractMessage(b.command);
+      if (!message) continue;
+      const hash = extractHashFromOutput(b.output);
+      addCommit(hash, message);
+      continue;
+    }
+
+    // ── Case 3: user message wrapping a bash execution (post-convertToLlm) ──
+    if (b.kind === "user") {
+      // Detect "Ran `git commit -m "..."`" pattern in user text
+      const ranCmd = b.text.match(/Ran\s+`((?:[^`\\]|\\.)*)`/);
+      if (!ranCmd) continue;
+      const message = tryExtractMessage(ranCmd[1]);
+      if (!message) continue;
+      // Extract output from the code block following the command
+      const codeBlock = b.text.match(/```\n([\s\S]*?)```/);
+      const hash = codeBlock ? extractHashFromOutput(codeBlock[1]) : undefined;
+      addCommit(hash, message);
     }
   }
 

package/src/extract/goals.ts

@@ -40,6 +40,11 @@ const isSubstantiveGoal = (text: string): boolean => {
   return true;
 };
 
+const FIRST_MSG_CLIP = 80;
+
+const indexSuffix = (sourceIndex?: number): string =>
+  sourceIndex != null ? ` (#${sourceIndex})` : "";
+
 // Test scope-change / task intent only on the leading portion of a user block
 // so that pasted outputs below the actual instruction do not trigger matches.
 const LEADING_CHARS = 200;
@@ -47,6 +52,7 @@ const LEADING_CHARS = 200;
 export const extractGoals = (blocks: NormalizedBlock[]): string[] => {
   const goals: string[] = [];
   let latestScopeChange: string[] | null = null;
+  let latestScopeIndex: number | undefined;
 
   for (const b of blocks) {
     if (b.kind !== "user") continue;
@@ -58,21 +64,26 @@ export const extractGoals = (blocks: NormalizedBlock[]): string[] => {
     if (lines.length === 0) continue;
 
     if (goals.length === 0) {
-      goals.push(...lines.slice(0, 6));
+      goals.push(...lines.slice(0, 6).map((l) => clip(l, FIRST_MSG_CLIP) + indexSuffix(b.sourceIndex)));
       continue;
     }
 
     const leading = b.text.slice(0, LEADING_CHARS);
     if (SCOPE_CHANGE_RE.test(leading)) {
       latestScopeChange = lines.slice(0, 3).map((l) => clip(l, MAX_GOAL_CHARS));
+      latestScopeIndex = b.sourceIndex;
     } else if (TASK_RE.test(leading) && lines[0].length > 15) {
       latestScopeChange = lines.slice(0, 2).map((l) => clip(l, MAX_GOAL_CHARS));
+      latestScopeIndex = b.sourceIndex;
     }
   }
 
   // Only emit the [Scope change] marker when we actually captured bullets.
   if (latestScopeChange && latestScopeChange.length > 0) {
-    goals.push("[Scope change]", ...latestScopeChange);
+    goals.push("[Scope change]");
+    for (const line of latestScopeChange) {
+      goals.push(line + indexSuffix(latestScopeIndex));
+    }
   }
 
   return goals.slice(0, 8);

package/src/hooks/before-compact.ts

@@ -240,7 +240,7 @@ const REASON_MESSAGES: Record<OwnCutCancelReason, string> = {
 export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime) => {
   pi.on("session_before_compact", (event, ctx) => {
     const { preparation, branchEntries, customInstructions } = event;
-    omRuntime.ensureConfig(ctx.cwd ?? process.cwd());
+    omRuntime.ensureConfig(ctx.cwd ?? process.cwd(), (msg) => ctx.ui?.notify?.(msg, "warning"));
     const trace = (ev: string, d?: Record<string, unknown>) => debugLog(ev, d, omRuntime.config.debugLog === true);
 
     trace("before_compact.enter", {
@@ -460,7 +460,10 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
       const projection = buildCompactionProjection(
       branchEntries as any[],
       firstKeptEntryId,
-      { observationsPoolMaxTokens: omRuntime.config.observationsPoolMaxTokens },
+      {
+        observationsPoolMaxTokens: omRuntime.config.observationsPoolMaxTokens,
+        fullFoldAlways: omRuntime.config.fullFoldAlways,
+      },
     );
       omContent = renderSummary(projection.reflections, projection.observations);
       omDetails = projection.details;

package/src/om/agents/dropper/agent.ts

@@ -8,7 +8,7 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { createBridgeStreamFn } from "../../provider-stream.js";
-import { streamSimple } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import { Type } from "typebox";
 import type { Static } from "typebox";
 import { debugLog } from "../../debug-log.js";

package/src/om/agents/observer/agent.ts

@@ -9,7 +9,7 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { createBridgeStreamFn } from "../../provider-stream.js";
-import { streamSimple } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import { Type } from "typebox";
 import type { Static } from "typebox";
 import { hashId } from "../../ids.js";

package/src/om/agents/reflector/agent.ts

@@ -8,7 +8,7 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { createBridgeStreamFn } from "../../provider-stream.js";
-import { streamSimple } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import { Type } from "typebox";
 import type { Static } from "typebox";
 import { hashId } from "../../ids.js";

package/src/om/cleanup.ts

@@ -0,0 +1,367 @@
+/**
+ * Cleanup utility — scans pending JSON files and cross-references against
+ * session JSONL files to find orphaned entries safe to delete.
+ *
+ * Per-session pending files under ~/.pi/agent/pi-blackhole/ accumulate when:
+ * - compaction is set to "manual" (noAutoCompact legacy) — OM outputs are
+ *   buffered rather than appended to the session
+ * - sessions are forked, abandoned, or deleted — the pending files remain
+ * - stale backup files (-pending.stale.json) persist after write-safe renames
+ *
+ * Safety invariant: a pending file is ONLY orphaned if its sessionId does NOT
+ * appear in ANY session JSONL file across all known session directories.
+ */
+import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from "node:fs";
+import { join, resolve, sep } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+export interface PendingFile {
+	sessionId: string;
+	filename: string;
+	path: string;
+	/** True for -pending.stale.json (backup of previous write). */
+	isStale: boolean;
+	sizeBytes: number;
+	/** Last modified timestamp (epoch ms). */
+	mtimeMs: number;
+}
+
+export interface CleanupReport {
+	/** All pending files found (both pending and stale). */
+	all: PendingFile[];
+	/** Files whose sessionId does not appear in any session JSONL. */
+	orphaned: PendingFile[];
+	/** Files whose sessionId was matched to an active session. */
+	active: PendingFile[];
+}
+
+// ── Constants ───────────────────────────────────────────────────────────────
+
+const PENDING_DIR = "pi-blackhole";
+const PENDING_SUFFIX = "-pending.json";
+const STALE_SUFFIX = "-pending.stale.json";
+
+// ── Helper: extract session ID from filename ────────────────────────────────
+
+function extractSessionId(filename: string): string | null {
+	if (filename.endsWith(STALE_SUFFIX)) {
+		return filename.slice(0, -STALE_SUFFIX.length) || null;
+	}
+	if (filename.endsWith(PENDING_SUFFIX)) {
+		return filename.slice(0, -PENDING_SUFFIX.length) || null;
+	}
+	return null;
+}
+
+// ── Scan pending files ──────────────────────────────────────────────────────
+
+/**
+ * Scan the pi-blackhole directory for all *-pending.json and *-pending.stale.json
+ * files. Returns file metadata sorted by mtime (newest first).
+ */
+function scanPendingFiles(agentDir: string = getAgentDir()): PendingFile[] {
+	const dir = join(agentDir, PENDING_DIR);
+	if (!existsSync(dir)) return [];
+
+	const results: PendingFile[] = [];
+	let entries: string[];
+	try {
+		entries = readdirSync(dir);
+	} catch {
+		return [];
+	}
+
+	for (const filename of entries) {
+		const sessionId = extractSessionId(filename);
+		if (!sessionId) continue;
+
+		const filePath = join(dir, filename);
+		let stat;
+		try {
+			stat = statSync(filePath);
+		} catch {
+			continue; // file disappeared between readdir and stat
+		}
+
+		results.push({
+			sessionId,
+			filename,
+			path: filePath,
+			isStale: filename.endsWith(STALE_SUFFIX),
+			sizeBytes: stat.size,
+			mtimeMs: stat.mtimeMs,
+		});
+	}
+
+	// Sort: newest first
+	results.sort((a, b) => b.mtimeMs - a.mtimeMs);
+	return results;
+}
+
+// ── Scan session IDs from JSONL files ───────────────────────────────────────
+
+/**
+ * Recursively scan a directory for .jsonl files and extract session IDs from
+ * their header line (first line: {"type":"session","id":"<uuid>",...}).
+ */
+function scanSessionDir(dir: string): Set<string> {
+	const ids = new Set<string>();
+	if (!existsSync(dir)) return ids;
+
+	const stack: string[] = [dir];
+	while (stack.length > 0) {
+		const current = stack.pop()!;
+		let names: string[];
+		try {
+			names = readdirSync(current);
+		} catch {
+			continue;
+		}
+
+		for (const name of names) {
+			const fullPath = join(current, name);
+			let st;
+			try {
+				st = statSync(fullPath);
+			} catch {
+				continue;
+			}
+
+			if (st.isDirectory()) {
+				stack.push(fullPath);
+			} else if (st.isFile() && name.endsWith(".jsonl")) {
+				try {
+					const fd = readFileSync(fullPath, "utf-8");
+					const newlineIdx = fd.indexOf("\n");
+					const firstLine = newlineIdx >= 0 ? fd.slice(0, newlineIdx) : fd;
+					const header = JSON.parse(firstLine) as Record<string, unknown>;
+					if (header.type === "session" && typeof header.id === "string" && header.id.length > 0) {
+						ids.add(header.id);
+					}
+				} catch {
+					// Corrupt or unreadable file — skip
+				}
+			}
+		}
+	}
+
+	return ids;
+}
+
+/**
+ * Read sessionDir override from settings.json, if any.
+ * Returns the resolved absolute path, or undefined if not set or unreadable.
+ */
+function readSettingsSessionDir(): string | undefined {
+	try {
+		const settingsPath = join(getAgentDir(), "settings.json");
+		if (!existsSync(settingsPath)) return undefined;
+		const raw = JSON.parse(readFileSync(settingsPath, "utf-8")) as Record<string, unknown>;
+		const dir = raw.sessionDir;
+		if (typeof dir === "string" && dir.trim().length > 0) {
+			// Expand ~ if present
+			const expanded = dir.startsWith("~") ? join(process.env.HOME ?? "/home", dir.slice(2)) : dir;
+			return resolve(expanded);
+		}
+	} catch {
+		// Unreadable settings — use default
+	}
+	return undefined;
+}
+
+/**
+ * Find all known session directories.
+ * Always includes the default sessions dir; also includes custom sessionDir
+ * from settings.json if present and different from the default.
+ */
+/** Get the default sessions directory. */
+function getDefaultSessionsDir(): string {
+	return join(getAgentDir(), "sessions");
+}
+
+function findSessionDirs(): string[] {
+	const dirs: string[] = [];
+	const default_ = getDefaultSessionsDir();
+	dirs.push(default_);
+
+	const custom = readSettingsSessionDir();
+	if (custom && custom !== default_ && existsSync(custom)) {
+		dirs.push(custom);
+	}
+
+	return dirs;
+}
+
+/**
+ * Collect all session IDs from all known session directories.
+ * Returns a Set of session UUIDs found in JSONL headers.
+ */
+function collectAllSessionIds(sessionDirs?: string[]): Set<string> {
+	const dirs = sessionDirs ?? findSessionDirs();
+	const allIds = new Set<string>();
+	for (const dir of dirs) {
+		const ids = scanSessionDir(dir);
+		for (const id of ids) allIds.add(id);
+	}
+	return allIds;
+}
+
+// ── Cross-reference: find orphaned files ────────────────────────────────────
+
+/**
+ * Determine which pending files are orphaned (no matching session JSONL).
+ *
+ * A pending file is orphaned when no session JSONL in any known session
+ * directory declares the same session ID in its header.
+ *
+ * Returns the full report with all files classified.
+ */
+function crossReference(
+	pending: PendingFile[],
+	sessionIds: Set<string>,
+): CleanupReport {
+	const orphaned: PendingFile[] = [];
+	const active: PendingFile[] = [];
+
+	for (const pf of pending) {
+		if (sessionIds.has(pf.sessionId)) {
+			active.push(pf);
+		} else {
+			orphaned.push(pf);
+		}
+	}
+
+	return { all: pending, orphaned, active };
+}
+
+/**
+ * Full pipeline: scan pending files, collect session IDs, cross-reference.
+ * Returns the cleanup report.
+ */
+export function analyzeOrphaned(agentDir?: string, sessionDirs?: string[]): CleanupReport {
+	const pending = scanPendingFiles(agentDir);
+	if (pending.length === 0) {
+		return { all: [], orphaned: [], active: [] };
+	}
+	const sessionIds = collectAllSessionIds(sessionDirs);
+	return crossReference(pending, sessionIds);
+}
+
+// ── Deletion ────────────────────────────────────────────────────────────────
+
+/** Session IDs are UUIDs or similar alphanumeric+hyphen strings. */
+const SAFE_SESSION_ID_RE = /^[a-zA-Z0-9][-a-zA-Z0-9]*$/;
+
+/**
+ * Validate that resolving and joining with `sessionId` cannot escape the
+ * pi-blackhole directory.  Must be called before any unlink.
+ */
+function validateDeletionPaths(
+	sessionId: string,
+	pendingDir: string,
+): { ok: true; pendingPath: string; stalePath: string } | { ok: false } {
+	// sessionId must be non-empty and contain only safe filesystem characters
+	if (!sessionId || typeof sessionId !== "string") return { ok: false };
+	if (!SAFE_SESSION_ID_RE.test(sessionId)) return { ok: false };
+
+	const resolvedDir = resolve(pendingDir);
+	const pendingPath = join(resolvedDir, `${sessionId}${PENDING_SUFFIX}`);
+	const stalePath = join(resolvedDir, `${sessionId}${STALE_SUFFIX}`);
+
+	// Resolve to absolute and verify containment within pi-blackhole/
+	const resolvedPending = resolve(pendingPath);
+	const resolvedStale = resolve(stalePath);
+
+	if (!resolvedPending.startsWith(resolvedDir + sep)) return { ok: false };
+	if (!resolvedStale.startsWith(resolvedDir + sep)) return { ok: false };
+
+	return { ok: true, pendingPath: resolvedPending, stalePath: resolvedStale };
+}
+
+/**
+ * Delete all pending files (pending + stale) for a given sessionId.
+ *
+ * Safety: validates that both resolved paths live under the pi-blackhole/
+ * directory before unlinking.  Returns false if validation fails.
+ *
+ * Returns true if at least one file was deleted, false if no files existed
+ * or if the paths failed containment validation.
+ */
+export function deletePendingFiles(sessionId: string, agentDir?: string): boolean {
+	const dir = join(agentDir ?? getAgentDir(), PENDING_DIR);
+	const valid = validateDeletionPaths(sessionId, dir);
+	if (!valid.ok) return false;
+
+	const { pendingPath, stalePath } = valid;
+
+	let deleted = false;
+
+	try {
+		if (existsSync(pendingPath)) {
+			unlinkSync(pendingPath);
+			deleted = true;
+		}
+	} catch {
+		// Best-effort — file may be locked or already gone
+	}
+
+	try {
+		if (existsSync(stalePath)) {
+			unlinkSync(stalePath);
+			deleted = true;
+		}
+	} catch {
+		// Best-effort
+	}
+
+	return deleted;
+}
+
+/**
+ * Delete multiple pending files by sessionId.
+ * Returns the count of successfully deleted file sets.
+ */
+export function deleteOrphanedBatch(
+	orphaned: PendingFile[],
+	agentDir?: string,
+): number {
+	let count = 0;
+	for (const pf of orphaned) {
+		if (deletePendingFiles(pf.sessionId, agentDir)) {
+			count++;
+		}
+	}
+	return count;
+}
+
+// ── Formatting helpers ──────────────────────────────────────────────────────
+
+/** Human-readable file size. */
+function formatSize(bytes: number): string {
+	if (bytes < 1024) return `${bytes} B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+	return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+/** Human-readable age from epoch ms. */
+function formatAge(mtimeMs: number, nowMs: number = Date.now()): string {
+	const diffMs = nowMs - mtimeMs;
+	const seconds = Math.floor(diffMs / 1000);
+	if (seconds < 60) return `${seconds}s ago`;
+	const minutes = Math.floor(seconds / 60);
+	if (minutes < 60) return `${minutes}m ago`;
+	const hours = Math.floor(minutes / 60);
+	if (hours < 24) return `${hours}h ago`;
+	const days = Math.floor(hours / 24);
+	if (days < 365) return `${days}d ago`;
+	const years = Math.floor(days / 365);
+	return `${years}y ago`;
+}
+
+/** Summary line for a pending file. */
+export function describeFile(pf: PendingFile, nowMs?: number): string {
+	const label = pf.isStale ? "stale" : "pending";
+	return `${pf.sessionId.slice(0, 8)}… ${label}  ${formatSize(pf.sizeBytes)}  ${formatAge(pf.mtimeMs, nowMs)}`;
+}