pi-blackhole 0.3.9 → 0.4.1

package/src/core/unified-config.ts

@@ -62,6 +62,14 @@ export interface UnifiedConfig {
 	 *  "pi-default" — Pi's built-in summarization */
 	compactionEngine: "blackhole" | "pi-default";
 
+	/** Mid-run auto-compaction (turn_end trigger, fires while the agent is still
+	 *  executing tool loops — agent_end alone never fires during long runs).
+	 *  "resume" — compact at threshold and inject a resume message so the agent
+	 *             continues the task (default)
+	 *  "pause"  — compact at threshold but stop; user continues manually
+	 *  "off"    — only evaluate the threshold when the agent finishes a run */
+	midRunCompaction: "resume" | "pause" | "off";
+
 	/** How much recent transcript to keep visible after compaction.
 	 *  "pi-default" — use Pi's firstKeptEntryId (respects Pi's keepRecentTokens)
 	 *  "minimal"    — keep only last user message (current agressive pi-vcc behavior)
@@ -76,6 +84,9 @@ export interface UnifiedConfig {
 	compactAfterTokens: number;
 	/** Observation pool token pressure for full fold. */
 	observationsPoolMaxTokens: number;
+	/** Treat every compaction as a full-fold boundary so early reflections/drops
+	 *  survive the first compaction in a fresh session. Default true. */
+	fullFoldAlways: boolean;
 	/** Target token budget for the observation pool (dropper aims here).
 	 *  Optional; defaults to half of observationsPoolMaxTokens when unset.
 	 *  Must be less than observationsPoolMaxTokens.
@@ -146,11 +157,13 @@ export const DEFAULTS: UnifiedConfig = {
 	compaction: "auto",
 	compactionEngine: "blackhole",
 	tailBehavior: "minimal",
+	midRunCompaction: "resume",
 
 	observeAfterTokens: 15_000,
 	reflectAfterTokens: 25_000,
 	compactAfterTokens: 81_000,
 	observationsPoolMaxTokens: 20_000,
+	fullFoldAlways: true,
 	observationsPoolTargetTokens: 10_000,
 	reflectorInputMaxTokens: 80_000,
 	dropperInputMaxTokens: 80_000,
@@ -171,6 +184,7 @@ const THINKING_LEVELS: readonly string[] = ["off", "minimal", "low", "medium", "
 const COMPACTION_VALUES = ["auto", "manual", "off"] as const;
 const COMPACTION_ENGINE_VALUES = ["blackhole", "pi-default"] as const;
 const TAIL_BEHAVIOR_VALUES = ["pi-default", "minimal"] as const;
+const MID_RUN_COMPACTION_VALUES = ["resume", "pause", "off"] as const;
 
 function isCompaction(v: unknown): v is "auto" | "manual" | "off" {
 	return typeof v === "string" && (COMPACTION_VALUES as readonly string[]).includes(v);
@@ -181,6 +195,9 @@ function isCompactionEngine(v: unknown): v is "blackhole" | "pi-default" {
 function isTailBehavior(v: unknown): v is "pi-default" | "minimal" {
 	return typeof v === "string" && (TAIL_BEHAVIOR_VALUES as readonly string[]).includes(v);
 }
+function isMidRunCompaction(v: unknown): v is "resume" | "pause" | "off" {
+	return typeof v === "string" && (MID_RUN_COMPACTION_VALUES as readonly string[]).includes(v);
+}
 
 function isRecord(v: unknown): v is Record<string, unknown> {
 	return typeof v === "object" && v !== null;
@@ -230,6 +247,7 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
 	if (isCompaction(raw.compaction)) c.compaction = raw.compaction;
 	if (isCompactionEngine(raw.compactionEngine)) c.compactionEngine = raw.compactionEngine;
 	if (isTailBehavior(raw.tailBehavior)) c.tailBehavior = raw.tailBehavior;
+	if (isMidRunCompaction(raw.midRunCompaction)) c.midRunCompaction = raw.midRunCompaction;
 
 	// Booleans — pi-vcc
 	if (typeof raw.overrideDefaultCompaction === "boolean") c.overrideDefaultCompaction = raw.overrideDefaultCompaction;
@@ -240,6 +258,7 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
 	if (typeof raw.noAutoCompact === "boolean") c.noAutoCompact = raw.noAutoCompact;
 	if (typeof raw.passive === "boolean") c.passive = raw.passive;
 	if (typeof raw.memory === "boolean") c.memory = raw.memory;
+	if (typeof raw.fullFoldAlways === "boolean") c.fullFoldAlways = raw.fullFoldAlways;
 	if (typeof raw.debugLog === "boolean") c.debugLog = raw.debugLog;
 
 	// Numeric fields — use nonNegativeInt for observerPreambleMaxTokens (0 = auto)
@@ -319,35 +338,54 @@ function migrateOldKnobs(parsed: Record<string, unknown>): void {
 
 // ── Load and save ────────────────────────────────────────────────────────────
 
-function readJson(path: string): Record<string, unknown> | null {
-	if (!existsSync(path)) return null;
+function readJson(path: string): { data: Record<string, unknown> | null; error: string | null } {
+	if (!existsSync(path)) return { data: null, error: null };
 	try {
-		return JSON.parse(readFileSync(path, "utf-8"));
-	} catch {
-		return null;
+		return { data: JSON.parse(readFileSync(path, "utf-8")), error: null };
+	} catch (e) {
+		const msg = `blackhole: config file at ${path} has invalid JSON: ${(e as Error).message}. Using defaults.`;
+		console.warn(msg);
+		return { data: null, error: msg };
 	}
 }
 
+/** Optional warning callback invoked when the primary config file has invalid JSON.
+ * Receives the warning message string. Used by callers with UI access to surface
+ * the error via ctx.ui.notify(message, "warning").
+ */
+type WarnFn = (message: string) => void;
+
 /**
  * Load unified configuration from ~/.pi/agent/pi-blackhole/pi-blackhole-config.json.
  * Falls back to legacy sources if the unified file doesn't exist.
  */
-export function loadUnifiedConfig(cwd: string): UnifiedConfig {
+export function loadUnifiedConfig(cwd: string, onWarn?: WarnFn): UnifiedConfig {
 	const path = configPath();
-	let raw = readJson(path);
+	let raw: Record<string, unknown> | null;
+	let primaryError: string | null = null;
+	const result = readJson(path);
+	raw = result.data;
+	primaryError = result.error;
+	if (primaryError && onWarn) onWarn(primaryError);
 
 	// Fallback to legacy sources if unified file doesn't exist
 	if (!raw) {
 		// Try legacy pi-vcc config
 		const piVccPath = join(getAgentDir(), "pi-vcc-config.json");
-		const piVccRaw = readJson(piVccPath);
+		const piVccResult = readJson(piVccPath);
+		const piVccRaw = piVccResult.data;
+		if (piVccResult.error && onWarn) onWarn(piVccResult.error);
 
 		// Try legacy om config from settings.json
 		const settingsPath = join(getAgentDir(), "settings.json");
-		const settingsRaw = readJson(settingsPath);
+		const settingsResult = readJson(settingsPath);
+		const settingsRaw = settingsResult.data;
+		if (settingsResult.error && onWarn) onWarn(settingsResult.error);
 		const omRaw = settingsRaw?.["pi-blackhole"] ?? settingsRaw?.["observational-memory"];
 		const projectSettingsPath = join(cwd, ".pi", "settings.json");
-		const projectRaw = readJson(projectSettingsPath);
+		const projectResult = readJson(projectSettingsPath);
+		const projectRaw = projectResult.data;
+		if (projectResult.error && onWarn) onWarn(projectResult.error);
 		const projectOmRaw = projectRaw?.["pi-blackhole"] ?? projectRaw?.["observational-memory"];
 
 		// Merge legacy sources
@@ -449,7 +487,11 @@ export function saveUnifiedConfig(settings: Partial<UnifiedConfig>): boolean {
 		const path = configPath();
 		const dir = dirname(path);
 		if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-		const existing = readJson(path) ?? {};
+		const existingResult = readJson(path);
+		const existing = existingResult.data ?? {};
+		if (existingResult.error) {
+			console.warn("blackhole: overwriting corrupt config file at " + path);
+		}
 		const next = { ...existing, ...settings };
 		writeFileSync(path, `${JSON.stringify(next, null, 2)}\n`);
 		return true;

package/src/extract/commits.ts

@@ -18,41 +18,84 @@ const firstLineOf = (text: string): string => {
 const cleanMessage = (msg: string): string =>
   msg.replace(/\\"/g, '"').replace(/\\'/g, "'").trim();
 
+/** Extract commit hash from git output text (tool_result or bash output). */
+const extractHashFromOutput = (text: string): string | undefined => {
+  const bracket = text.match(/\[\S+\s+([0-9a-f]{7,12})\]/);
+  if (bracket) return bracket[1];
+  const range = text.match(/\b([0-9a-f]{7,12})\.\.([0-9a-f]{7,12})\b/);
+  if (range) return range[2];
+  const plain = text.match(HASH_RE);
+  if (plain) return plain[1];
+  return undefined;
+};
+
+/** Try to extract a commit message from a git commit command string. */
+const tryExtractMessage = (cmd: string): string | undefined => {
+  if (!/\bgit\s+commit\b/.test(cmd)) return undefined;
+  const m = cmd.match(COMMIT_MSG_RE);
+  if (!m) return undefined;
+  const message = firstLineOf(cleanMessage(m[1] ?? m[2] ?? m[3] ?? ""));
+  return message || undefined;
+};
+
 /**
- * Extract git commits from bash tool calls (`git commit -m "..."`) and pair
- * with hash from the immediately following tool_result.
+ * Extract git commits from bash tool calls, bash execution messages,
+ * and user messages that wrap bash execution output (post-convertToLlm).
+ *
+ * Handles three block kinds:
+ * - tool_call (name: "bash") — agent tool call to the bash tool
+ * - bash — pi's internal bashExecution message
+ * - user — convertToLlm wraps bashExecution as "Ran `cmd`\n```\noutput\n```"
  */
 export const extractCommits = (blocks: NormalizedBlock[]): CommitInfo[] => {
   const commits: CommitInfo[] = [];
+  const addCommit = (hash: string | undefined, message: string) => {
+    const key = `${hash ?? ""}::${message}`;
+    if (!commits.some((c) => `${c.hash ?? ""}::${c.message}` === key)) {
+      commits.push({ hash, message });
+    }
+  };
 
   for (let i = 0; i < blocks.length; i++) {
     const b = blocks[i];
-    if (b.kind !== "tool_call" || b.name !== "bash") continue;
-    const cmd = typeof b.args.command === "string" ? b.args.command : "";
-    if (!/\bgit\s+commit\b/.test(cmd)) continue;
-    const m = cmd.match(COMMIT_MSG_RE);
-    if (!m) continue;
-    const message = firstLineOf(cleanMessage(m[1] ?? m[2] ?? m[3] ?? ""));
-    if (!message) continue;
 
-    let hash: string | undefined;
-    // Look at next tool_result for hash
-    for (let j = i + 1; j < Math.min(blocks.length, i + 3); j++) {
-      const r = blocks[j];
-      if (r.kind !== "tool_result") continue;
-      // Common git commit output: `[branch <hash>] message` or `<branch> <hash>..<hash>`
-      const bracket = r.text.match(/\[\S+\s+([0-9a-f]{7,12})\]/);
-      if (bracket) { hash = bracket[1]; break; }
-      const range = r.text.match(/\b([0-9a-f]{7,12})\.\.([0-9a-f]{7,12})\b/);
-      if (range) { hash = range[2]; break; }
-      const plain = r.text.match(HASH_RE);
-      if (plain) { hash = plain[1]; break; }
+    // ── Case 1: tool_call (agent calls bash tool) ──
+    if (b.kind === "tool_call" && b.name === "bash") {
+      const cmd = (b.args && typeof b.args.command === "string") ? b.args.command : "";
+      const message = tryExtractMessage(cmd);
+      if (!message) continue;
+
+      let hash: string | undefined;
+      for (let j = i + 1; j < Math.min(blocks.length, i + 3); j++) {
+        const r = blocks[j];
+        if (r.kind !== "tool_result") continue;
+        hash = extractHashFromOutput(r.text);
+        if (hash) break;
+      }
+      addCommit(hash, message);
+      continue;
     }
 
-    // Dedup by message+hash
-    const key = `${hash ?? ""}::${message}`;
-    if (!commits.some((c) => `${c.hash ?? ""}::${c.message}` === key)) {
-      commits.push({ hash, message });
+    // ── Case 2: bash execution message ──
+    if (b.kind === "bash") {
+      const message = tryExtractMessage(b.command);
+      if (!message) continue;
+      const hash = extractHashFromOutput(b.output);
+      addCommit(hash, message);
+      continue;
+    }
+
+    // ── Case 3: user message wrapping a bash execution (post-convertToLlm) ──
+    if (b.kind === "user") {
+      // Detect "Ran `git commit -m "..."`" pattern in user text
+      const ranCmd = b.text.match(/Ran\s+`((?:[^`\\]|\\.)*)`/);
+      if (!ranCmd) continue;
+      const message = tryExtractMessage(ranCmd[1]);
+      if (!message) continue;
+      // Extract output from the code block following the command
+      const codeBlock = b.text.match(/```\n([\s\S]*?)```/);
+      const hash = codeBlock ? extractHashFromOutput(codeBlock[1]) : undefined;
+      addCommit(hash, message);
     }
   }
 

package/src/extract/goals.ts

@@ -40,6 +40,11 @@ const isSubstantiveGoal = (text: string): boolean => {
   return true;
 };
 
+const FIRST_MSG_CLIP = 80;
+
+const indexSuffix = (sourceIndex?: number): string =>
+  sourceIndex != null ? ` (#${sourceIndex})` : "";
+
 // Test scope-change / task intent only on the leading portion of a user block
 // so that pasted outputs below the actual instruction do not trigger matches.
 const LEADING_CHARS = 200;
@@ -47,6 +52,7 @@ const LEADING_CHARS = 200;
 export const extractGoals = (blocks: NormalizedBlock[]): string[] => {
   const goals: string[] = [];
   let latestScopeChange: string[] | null = null;
+  let latestScopeIndex: number | undefined;
 
   for (const b of blocks) {
     if (b.kind !== "user") continue;
@@ -58,21 +64,26 @@ export const extractGoals = (blocks: NormalizedBlock[]): string[] => {
     if (lines.length === 0) continue;
 
     if (goals.length === 0) {
-      goals.push(...lines.slice(0, 6));
+      goals.push(...lines.slice(0, 6).map((l) => clip(l, FIRST_MSG_CLIP) + indexSuffix(b.sourceIndex)));
       continue;
     }
 
     const leading = b.text.slice(0, LEADING_CHARS);
     if (SCOPE_CHANGE_RE.test(leading)) {
       latestScopeChange = lines.slice(0, 3).map((l) => clip(l, MAX_GOAL_CHARS));
+      latestScopeIndex = b.sourceIndex;
     } else if (TASK_RE.test(leading) && lines[0].length > 15) {
       latestScopeChange = lines.slice(0, 2).map((l) => clip(l, MAX_GOAL_CHARS));
+      latestScopeIndex = b.sourceIndex;
     }
   }
 
   // Only emit the [Scope change] marker when we actually captured bullets.
   if (latestScopeChange && latestScopeChange.length > 0) {
-    goals.push("[Scope change]", ...latestScopeChange);
+    goals.push("[Scope change]");
+    for (const line of latestScopeChange) {
+      goals.push(line + indexSuffix(latestScopeIndex));
+    }
   }
 
   return goals.slice(0, 8);

package/src/hooks/before-compact.ts

@@ -48,6 +48,37 @@ const formatTokens = (n: number): string => {
   return String(n);
 };
 
+export interface CompactionStats {
+  summarized: number;
+  kept: number;
+  keptTokensEst: number;
+  compactAll: boolean;
+  totalUserTurns: number;
+  keptUserTurns: number;
+  requestedKeepUserTurns: number;
+  keepUserTurnsExplicit: boolean;
+  keepFallbackToCompactAll: boolean;
+  smartKeepAdjusted: boolean;
+  smartFromKeep: number;
+}
+
+/**
+ * Format compaction stats for user-visible notification.
+ * Example output:
+ *   blackhole: 6 source entries processed; tail kept 1/4 user turns (~0.5k tok).
+ */
+export const formatCompactionStats = (stats: CompactionStats): string => {
+  const parts: string[] = [`${stats.summarized} source entries processed`];
+  parts.push(`tail kept ${stats.keptUserTurns}/${stats.totalUserTurns} user turns`);
+  if (stats.smartKeepAdjusted) {
+    parts.push(`smart keep:${stats.smartFromKeep}→${stats.keptUserTurns}`);
+  }
+  if (stats.keepFallbackToCompactAll) {
+    parts.push(`compact-all`);
+  }
+  return `blackhole: ${parts.join("; ")} (~${formatTokens(stats.keptTokensEst)} tok).`;
+};
+
 const dbg = (debug: boolean, data: Record<string, unknown>) => {
   if (!debug) return;
   try { writeFileSync("/tmp/pi-blackhole-debug.json", JSON.stringify(data, null, 2)); } catch {}
@@ -240,7 +271,7 @@ const REASON_MESSAGES: Record<OwnCutCancelReason, string> = {
 export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime) => {
   pi.on("session_before_compact", (event, ctx) => {
     const { preparation, branchEntries, customInstructions } = event;
-    omRuntime.ensureConfig(ctx.cwd ?? process.cwd());
+    omRuntime.ensureConfig(ctx.cwd ?? process.cwd(), (msg) => ctx.ui?.notify?.(msg, "warning"));
     const trace = (ev: string, d?: Record<string, unknown>) => debugLog(ev, d, omRuntime.config.debugLog === true);
 
     trace("before_compact.enter", {
@@ -291,9 +322,7 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
     // Determine effective tail behavior for buildOwnCut
     // Both /blackhole and auto-triggered default to "minimal" (aggressive cut);
     // users can opt into "pi-default" (gentler) by setting tailBehavior in config.
-    const effectiveTailBehavior = isPiVcc
-      ? (omRuntime.config.tailBehavior ?? "minimal")
-      : (omRuntime.config.tailBehavior ?? "minimal");
+    const effectiveTailBehavior = omRuntime.config.tailBehavior ?? "minimal";
 
     trace("before_compact.tail_behavior", {
       effectiveTailBehavior,
@@ -400,10 +429,22 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
       }, 0);
       return sum;
     }, 0);
+    const totalUserTurns = (branchEntries as any[]).filter((e: any) => e.type === "message" && e.message?.role === "user").length;
+    const keptUserTurns = ownCut.compactAll
+      ? 0
+      : (branchEntries as any[]).slice(keptIdx).filter((e: any) => e.type === "message" && e.message?.role === "user").length;
     omRuntime.compactionStats = {
       summarized: agentMessages.length,
       kept: keptEntries.length,
       keptTokensEst: Math.round(keptChars / 4),
+      compactAll: ownCut.compactAll,
+      totalUserTurns,
+      keptUserTurns,
+      requestedKeepUserTurns: 1,
+      keepUserTurnsExplicit: false,
+      keepFallbackToCompactAll: ownCut.compactAll,
+      smartKeepAdjusted: false,
+      smartFromKeep: 1,
     };
 
     const summary = compile({
@@ -460,7 +501,10 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
       const projection = buildCompactionProjection(
       branchEntries as any[],
       firstKeptEntryId,
-      { observationsPoolMaxTokens: omRuntime.config.observationsPoolMaxTokens },
+      {
+        observationsPoolMaxTokens: omRuntime.config.observationsPoolMaxTokens,
+        fullFoldAlways: omRuntime.config.fullFoldAlways,
+      },
     );
       omContent = renderSummary(projection.reflections, projection.observations);
       omDetails = projection.details;
@@ -488,10 +532,7 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
     const sessionId = ctx.sessionManager.getSessionId();
     setTimeout(() => {
       try {
-        ctx?.ui?.notify?.(
-          `blackhole: ${stats.summarized} source entries processed; tail kept ${stats.kept} (~${formatTokens(stats.keptTokensEst)} tok).`,
-          "info",
-        );
+        ctx?.ui?.notify?.(formatCompactionStats(stats), "info");
         notifyMigrationReminder(sessionId, (msg, level) => ctx?.ui?.notify?.(msg, level as any));
       } catch {}
     }, 500);

package/src/om/agents/dropper/agent.ts

@@ -8,7 +8,7 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { createBridgeStreamFn } from "../../provider-stream.js";
-import { streamSimple } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import { Type } from "typebox";
 import type { Static } from "typebox";
 import { debugLog } from "../../debug-log.js";

package/src/om/agents/observer/agent.ts

@@ -9,7 +9,7 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { createBridgeStreamFn } from "../../provider-stream.js";
-import { streamSimple } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import { Type } from "typebox";
 import type { Static } from "typebox";
 import { hashId } from "../../ids.js";

package/src/om/agents/reflector/agent.ts

@@ -8,7 +8,7 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { createBridgeStreamFn } from "../../provider-stream.js";
-import { streamSimple } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import { Type } from "typebox";
 import type { Static } from "typebox";
 import { hashId } from "../../ids.js";