pi-blackhole 0.2.0

package/src/core/lineage.ts

@@ -0,0 +1,26 @@
+export interface LineageEntryLike {
+  id?: string;
+}
+
+export interface LineageSessionManagerLike {
+  getBranch: () => LineageEntryLike[];
+  getEntries?: () => LineageEntryLike[];
+}
+
+export const getActiveLineageEntryIds = (sessionManager: LineageSessionManagerLike): Set<string> => {
+  try {
+    const branch = sessionManager.getBranch() ?? [];
+    if (branch.length > 0) {
+      return new Set(branch.map((e) => e.id).filter((id): id is string => Boolean(id)));
+    }
+  } catch {
+    // fall through to defensive fallback
+  }
+
+  try {
+    const all = sessionManager.getEntries?.() ?? [];
+    return new Set(all.map((e) => e.id).filter((id): id is string => Boolean(id)));
+  } catch {
+    return new Set();
+  }
+};

package/src/core/load-messages.ts

@@ -0,0 +1,41 @@
+import { readFileSync } from "fs";
+import type { Message } from "@earendil-works/pi-ai";
+import { renderMessage, type RenderedEntry } from "./render-entries";
+
+export interface LoadedMessages {
+  rendered: RenderedEntry[];
+  rawMessages: Message[];
+  entryIds: string[];
+}
+
+export const loadAllMessages = (
+  sessionFile: string,
+  full: boolean,
+  allowedEntryIds?: Set<string>,
+): LoadedMessages => {
+  const content = readFileSync(sessionFile, "utf-8");
+  const entries: any[] = [];
+  for (const line of content.split("\n")) {
+    if (!line.trim()) continue;
+    try { entries.push(JSON.parse(line)); } catch {}
+  }
+  const rendered: RenderedEntry[] = [];
+  const rawMessages: Message[] = [];
+  const entryIds: string[] = [];
+
+  let messageIndex = 0;
+  for (const e of entries) {
+    const isMessage = e.type === "message" && e.message;
+    if (!isMessage) continue;
+
+    const allowed = !allowedEntryIds || allowedEntryIds.has(e.id);
+    if (allowed) {
+      rendered.push(renderMessage(e.message, messageIndex, String(e.id), full));
+      rawMessages.push(e.message);
+      entryIds.push(String(e.id));
+    }
+    messageIndex++;
+  }
+
+  return { rendered, rawMessages, entryIds };
+};

package/src/core/normalize.ts

@@ -0,0 +1,79 @@
+/**
+ * Message normalization — converts raw messages to NormalizedBlock array.
+ *
+ * Upstream: https://github.com/sting8k/pi-vcc (src/core/normalize.ts)
+ * Unmodified.
+ */
+import type { Message } from "@earendil-works/pi-ai";
+import type { NormalizedBlock } from "../types";
+import { textOf } from "./content";
+import { sanitize } from "./sanitize";
+
+const normalizeOne = (msg: Message, msgIndex: number): NormalizedBlock[] => {
+  if (msg.role === "user") {
+    const blocks: NormalizedBlock[] = [];
+    const text = sanitize(textOf(msg.content));
+    if (text) blocks.push({ kind: "user", text, sourceIndex: msgIndex });
+    if (msg.content && typeof msg.content !== "string") {
+      for (const part of msg.content) {
+        if (part.type === "image") {
+          blocks.push({ kind: "user", text: `[image: ${part.mimeType}]`, sourceIndex: msgIndex });
+        }
+      }
+    }
+    return blocks.length > 0 ? blocks : [{ kind: "user", text: "", sourceIndex: msgIndex }];
+  }
+
+  if ((msg as any).role === "bashExecution") {
+    const cmd = (msg as any).command ?? "";
+    const out = (msg as any).output ?? "";
+    const exit = (msg as any).exitCode;
+    return [{ kind: "bash", command: cmd, output: out, exitCode: exit, sourceIndex: msgIndex }];
+  }
+
+  if (msg.role === "toolResult") {
+    return [{
+      kind: "tool_result",
+      name: msg.toolName,
+      text: sanitize(textOf(msg.content)),
+      isError: msg.isError,
+      sourceIndex: msgIndex,
+    }];
+  }
+
+  if (msg.role === "assistant") {
+    if (!msg.content) return [];
+    if (typeof msg.content === "string") {
+      return [{ kind: "assistant", text: sanitize(msg.content), sourceIndex: msgIndex }];
+    }
+
+    const blocks: NormalizedBlock[] = [];
+    for (const part of msg.content) {
+      if (part.type === "text") {
+        blocks.push({ kind: "assistant", text: sanitize(part.text), sourceIndex: msgIndex });
+      } else if (part.type === "thinking") {
+        blocks.push({
+          kind: "thinking",
+          text: sanitize(part.thinking),
+          redacted: part.redacted ?? false,
+          sourceIndex: msgIndex,
+        });
+      } else if (part.type === "toolCall") {
+        blocks.push({
+          kind: "tool_call",
+          name: part.name,
+          args: part.arguments,
+          sourceIndex: msgIndex,
+        });
+      }
+    }
+    return blocks;
+  }
+
+  return [];
+};
+
+export const normalize = (messages: Message[]): NormalizedBlock[] =>
+  messages.flatMap((msg, i) => normalizeOne(msg, i));
+
+

package/src/core/recall-scope.ts

@@ -0,0 +1,14 @@
+export type RecallScope = "lineage" | "all";
+
+const SCOPE_RE = /\bscope:(lineage|all)\b/i;
+
+export const normalizeRecallScope = (scope?: unknown): RecallScope =>
+  typeof scope === "string" && scope.toLowerCase() === "all" ? "all" : "lineage";
+
+export const parseRecallScope = (text: string): { scope: RecallScope; text: string } => {
+  const match = text.match(SCOPE_RE);
+  return {
+    scope: normalizeRecallScope(match?.[1]),
+    text: text.replace(SCOPE_RE, "").replace(/\s+/g, " ").trim(),
+  };
+};

package/src/core/render-entries.ts

@@ -0,0 +1,56 @@
+import type { Message } from "@earendil-works/pi-ai";
+import { clip, textOf } from "./content";
+import { summarizeToolArgs } from "./tool-args";
+import { extractPath } from "./tool-args";
+
+export interface RenderedEntry {
+  index: number;
+  id: string;
+  role: string;
+  summary: string;
+  files?: string[];
+}
+
+const toolCalls = (content: Message["content"]): string => {
+  if (!content || typeof content === "string") return "";
+  return content
+    .filter((c) => c.type === "toolCall")
+    .map((c) => `${c.name}(${summarizeToolArgs(c.arguments)})`)
+    .join(", ");
+};
+
+const extractFilesFromContent = (content: Message["content"]): string[] => {
+  if (!content || typeof content === "string") return [];
+  return content
+    .filter((c) => c.type === "toolCall")
+    .map((c) => extractPath(c.arguments))
+    .filter((p): p is string => p !== null);
+};
+
+export const renderMessage = (msg: Message, index: number, id: string, full = false): RenderedEntry => {
+  if (msg.role === "user") {
+    return { index, id, role: "user", summary: full ? textOf(msg.content) : clip(textOf(msg.content), 300) };
+  }
+  if (msg.role === "toolResult") {
+    const prefix = msg.isError ? "ERROR " : "";
+    const text = full ? textOf(msg.content) : clip(textOf(msg.content), 200);
+    return {
+      index, id, role: "tool_result",
+      summary: `${prefix}[${msg.toolName}] ${text}`,
+    };
+  }
+  // bashExecution has command+output instead of content
+  if ((msg as any).role === "bashExecution") {
+    const cmd = (msg as any).command ?? "";
+    const out = (msg as any).output ?? "";
+    const text = full ? `$ ${cmd}\n${out}` : clip(`$ ${cmd}\n${out}`, 300);
+    return { index, id, role: "bash", summary: text };
+  }
+  const text = full ? textOf(msg.content) : clip(textOf(msg.content), 300);
+  const tools = toolCalls(msg.content);
+  const files = extractFilesFromContent(msg.content);
+  const summary = tools ? `${tools}\n${text}` : text;
+  return { index, id, role: "assistant", summary, ...(files.length > 0 && { files }) };
+};
+
+

package/src/core/report.ts

@@ -0,0 +1,237 @@
+import type { Message } from "@earendil-works/pi-ai";
+import { buildSections } from "./build-sections";
+import { clip } from "./content";
+import { normalize } from "./normalize";
+import { renderMessage } from "./render-entries";
+import { searchEntries } from "./search-entries";
+import { type CompileInput, compile } from "./summarize";
+
+const SECTION_HEADERS = ["Session Goal", "Files And Changes", "Commits", "Outstanding Context"];
+
+interface RoleCounts {
+  user: number;
+  assistant: number;
+  toolResult: number;
+}
+
+interface BlockCounts {
+  user: number;
+  assistant: number;
+  toolCalls: number;
+  toolResults: number;
+  thinking: number;
+}
+
+export interface RecallProbe {
+  label: string;
+  sourceText: string;
+  query: string;
+  summaryMentioned: boolean;
+  recallHits: number;
+}
+
+export interface CompactReport {
+  summary: string;
+  before: {
+    messageCount: number;
+    roleCounts: RoleCounts;
+    blockCounts: BlockCounts;
+    inputChars: number;
+    estimatedTokens: number;
+    topFiles: string[];
+    preview: string;
+  };
+  after: {
+    summaryLength: number;
+    estimatedTokens: number;
+    sectionCount: number;
+    summaryPreview: string;
+    goalsCount: number;
+    blockersCount: number;
+    briefTranscriptLines: number;
+  };
+  compression: {
+    charsBefore: number;
+    charsAfter: number;
+    ratio: number;
+    messagesBefore: number;
+  };
+  recall: {
+    probes: RecallProbe[];
+  };
+}
+
+const estimateTokensFromChars = (chars: number): number =>
+  Math.ceil(chars / 4);
+
+const countRoles = (messages: Message[]): RoleCounts => {
+  const counts: RoleCounts = { user: 0, assistant: 0, toolResult: 0 };
+  for (const msg of messages) {
+    if (msg.role === "user") counts.user += 1;
+    else if (msg.role === "assistant") counts.assistant += 1;
+    else if (msg.role === "toolResult") counts.toolResult += 1;
+  }
+  return counts;
+};
+
+const countBlocks = (messages: Message[]): BlockCounts => {
+  const counts: BlockCounts = {
+    user: 0,
+    assistant: 0,
+    toolCalls: 0,
+    toolResults: 0,
+    thinking: 0,
+  };
+
+  for (const block of normalize(messages)) {
+    if (block.kind === "user") counts.user += 1;
+    else if (block.kind === "assistant") counts.assistant += 1;
+    else if (block.kind === "tool_call") counts.toolCalls += 1;
+    else if (block.kind === "tool_result") counts.toolResults += 1;
+    else if (block.kind === "thinking") counts.thinking += 1;
+  }
+
+  return counts;
+};
+
+const inputCharsOf = (messages: Message[]): number =>
+  messages
+    .map((msg, index) => renderMessage(msg, index, "", true).summary.length)
+    .reduce((sum, len) => sum + len, 0);
+
+const topFilesOf = (messages: Message[]): string[] => {
+  const files = new Set<string>();
+  for (const block of normalize(messages)) {
+    if (block.kind === "tool_call") {
+      for (const key of ["path", "file_path", "filePath", "file"]) {
+        const val = block.args[key];
+        if (typeof val === "string") { files.add(val); break; }
+      }
+    }
+  }
+  return [...files].slice(0, 10);
+};
+
+const previewOf = (messages: Message[], edgeCount = 3): string => {
+  const rendered = messages.map((msg, index) => renderMessage(msg, index, ""));
+  if (rendered.length === 0) return "(empty)";
+  if (rendered.length <= edgeCount * 2) {
+    return rendered
+      .map((entry) => `#${entry.index} [${entry.role}] ${clip(entry.summary, 220)}`)
+      .join("\n");
+  }
+
+  const first = rendered.slice(0, edgeCount);
+  const last = rendered.slice(-edgeCount);
+  return [
+    ...first.map((entry) => `#${entry.index} [${entry.role}] ${clip(entry.summary, 220)}`),
+    "...",
+    ...last.map((entry) => `#${entry.index} [${entry.role}] ${clip(entry.summary, 220)}`),
+  ].join("\n");
+};
+
+const sectionCountOf = (summary: string): number =>
+  SECTION_HEADERS.filter((header) => summary.includes(`[${header}]`)).length;
+
+const briefLineCountOf = (summary: string): number => {
+  const sep = "\n\n---\n\n";
+  const idx = summary.indexOf(sep);
+  if (idx < 0) return 0;
+  return summary.slice(idx + sep.length).split("\n").length;
+};
+
+const queryTermsOf = (text: string): string[] =>
+  (text.match(/[\p{L}\p{N}_./-]{3,}/gu) ?? [])
+    .map((part) => part.trim())
+    .filter(Boolean);
+
+const queryOf = (text: string): string => {
+  const terms = queryTermsOf(text);
+  return terms.slice(0, 6).join(" ");
+};
+
+const matchesQuery = (text: string, query: string): boolean => {
+  const hay = text.toLowerCase();
+  return query
+    .toLowerCase()
+    .split(/\s+/)
+    .filter(Boolean)
+    .every((term) => hay.includes(term));
+};
+
+const probesOf = (messages: Message[], summary: string): RecallProbe[] => {
+  const blocks = normalize(messages);
+  const data = buildSections({ blocks });
+
+  // Find first file from tool calls
+  let firstFile = "";
+  for (const b of blocks) {
+    if (b.kind === "tool_call") {
+      for (const key of ["path", "file_path", "filePath", "file"]) {
+        if (typeof b.args[key] === "string") { firstFile = b.args[key] as string; break; }
+      }
+      if (firstFile) break;
+    }
+  }
+
+  const rawProbes = [
+    { label: "goal", text: data.sessionGoal[0] ?? "" },
+    { label: "file", text: firstFile },
+    { label: "problem", text: data.outstandingContext[0] ?? "" },
+  ];
+
+  const rendered = messages.map((msg, index) => renderMessage(msg, index, ""));
+
+  return rawProbes
+    .map(({ label, text }) => {
+      const sourceText = text.trim();
+      const query = queryOf(sourceText);
+      if (!query) return null;
+      return {
+        label,
+        sourceText,
+        query,
+        summaryMentioned: matchesQuery(summary, query),
+        recallHits: searchEntries(rendered, messages, query).length,
+      };
+    })
+    .filter((probe): probe is RecallProbe => probe !== null);
+};
+
+export const buildCompactReport = (input: CompileInput): CompactReport => {
+  const summary = compile(input);
+  const data = buildSections({ blocks: normalize(input.messages) });
+  const inputChars = inputCharsOf(input.messages);
+  const topFiles = topFilesOf(input.messages);
+
+  return {
+    summary,
+    before: {
+      messageCount: input.messages.length,
+      roleCounts: countRoles(input.messages),
+      blockCounts: countBlocks(input.messages),
+      inputChars,
+      estimatedTokens: estimateTokensFromChars(inputChars),
+      topFiles,
+      preview: previewOf(input.messages),
+    },
+    after: {
+      summaryLength: summary.length,
+      estimatedTokens: estimateTokensFromChars(summary.length),
+      sectionCount: sectionCountOf(summary),
+      summaryPreview: summary,
+      goalsCount: data.sessionGoal.length,
+      blockersCount: data.outstandingContext.length,
+      briefTranscriptLines: briefLineCountOf(summary),
+    },
+    compression: {
+      charsBefore: inputChars,
+      charsAfter: summary.length,
+      ratio: summary.length === 0 ? 0 : Number((inputChars / summary.length).toFixed(2)),
+      messagesBefore: input.messages.length,
+    },
+    recall: {
+      probes: probesOf(input.messages, summary),
+    },
+  };
+};

package/src/core/sanitize.ts

@@ -0,0 +1,5 @@
+const ANSI_RE = /\x1b\[[0-9;]*[A-Za-z]/g;
+const CTRL_RE = /[\x00-\x08\x0b\x0c\x0e-\x1f]/g;
+
+export const sanitize = (text: string): string =>
+  text.replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(ANSI_RE, "").replace(CTRL_RE, "");

package/src/core/search-entries.ts

@@ -0,0 +1,227 @@
+/**
+ * Search entries — BM25 + regex search over session history.
+ *
+ * Upstream: https://github.com/sting8k/pi-vcc (src/core/search-entries.ts)
+ * Unmodified.
+ */
+import type { Message } from "@earendil-works/pi-ai";
+import type { RenderedEntry } from "./render-entries";
+import { textOf } from "./content";
+
+export interface SearchHit extends RenderedEntry {
+  /** Context snippet around the first matched term (only when query provided) */
+  snippet?: string;
+  /** Number of query terms matched (for ranking) */
+  matchCount?: number;
+}
+
+const escapeRegex = (s: string): string =>
+  s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/** Try to compile as regex; fall back to escaped literal. */
+const safeRegex = (pattern: string): RegExp => {
+  try {
+    return new RegExp(pattern, "i");
+  } catch {
+    return new RegExp(escapeRegex(pattern), "i");
+  }
+};
+
+/** Detect if the query looks like a single regex pattern (contains regex metacharacters). */
+const looksLikeRegex = (query: string): boolean =>
+  /[|*+?{}()[\]\\^$.]/.test(query);
+
+/** Build a regex for snippet highlighting — matches first available term. */
+const snippetRegex = (terms: string[]): RegExp => {
+  const alts = terms.map((t) => {
+    try {
+      // Validate that it's a valid regex
+      new RegExp(t, "i");
+      return t;
+    } catch {
+      return escapeRegex(t);
+    }
+  });
+  return new RegExp(alts.join("|"), "i");
+};
+
+// ── Stopwords for natural language queries ──
+const STOPWORDS = new Set([
+  // English
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "can", "shall", "of", "in", "to", "for",
+  "with", "on", "at", "from", "by", "as", "into", "through", "during",
+  "before", "after", "above", "below", "between", "out", "off", "over",
+  "under", "again", "further", "then", "once", "here", "there", "when",
+  "where", "why", "how", "all", "both", "each", "few", "more", "most",
+  "other", "some", "such", "no", "nor", "not", "only", "own", "same",
+  "so", "than", "too", "very", "just", "about", "it", "its", "that",
+  "this", "what", "which", "who", "whom", "these", "those",
+]);
+
+/** Remove stopwords, keep meaningful terms. */
+const filterStopwords = (terms: string[]): string[] => {
+  const meaningful = terms.filter((t) => !STOPWORDS.has(t.toLowerCase()) && t.length > 1);
+  // If all terms were stopwords, return original (don't lose everything)
+  return meaningful.length > 0 ? meaningful : terms;
+};
+
+/** Count how many distinct terms match the haystack. */
+const countMatches = (hay: string, terms: string[]): number => {
+  let count = 0;
+  for (const t of terms) {
+    if (safeRegex(t).test(hay)) count++;
+  }
+  return count;
+};
+
+// ── BM25-lite scoring ──
+const BM25_K = 1.2;
+const BM25_B = 0.75;
+
+/** Count occurrences of a regex pattern in text. */
+const termFreq = (text: string, pattern: RegExp): number => {
+  const matches = text.match(new RegExp(pattern.source, "gi"));
+  return matches ? matches.length : 0;
+};
+
+interface BM25Context {
+  n: number;         // total docs
+  avgDl: number;     // average doc length (words)
+  df: Map<string, number>; // term -> number of docs containing it
+}
+
+/** Precompute IDF and avgDl across all docs. */
+const buildBM25Context = (docs: string[], terms: string[]): BM25Context => {
+  const n = docs.length;
+  const df = new Map<string, number>();
+  let totalLen = 0;
+
+  for (const doc of docs) {
+    totalLen += doc.split(/\s+/).length;
+    for (const t of terms) {
+      if (safeRegex(t).test(doc)) {
+        df.set(t, (df.get(t) ?? 0) + 1);
+      }
+    }
+  }
+
+  return { n, avgDl: totalLen / Math.max(n, 1), df };
+};
+
+/** BM25 score for a single doc against query terms. */
+const bm25Score = (doc: string, terms: string[], ctx: BM25Context): number => {
+  const dl = doc.split(/\s+/).length;
+  let score = 0;
+
+  for (const t of terms) {
+    const tf = termFreq(doc, safeRegex(t));
+    if (tf === 0) continue;
+
+    const docFreq = ctx.df.get(t) ?? 0;
+    // IDF: log((N - df + 0.5) / (df + 0.5) + 1)
+    const idf = Math.log((ctx.n - docFreq + 0.5) / (docFreq + 0.5) + 1);
+    // TF saturation with length normalization
+    const tfNorm = (tf * (BM25_K + 1)) / (tf + BM25_K * (1 - BM25_B + BM25_B * dl / ctx.avgDl));
+    score += idf * tfNorm;
+  }
+
+  return score;
+};
+
+/** Line-based snippet: ±contextLines around first regex match. */
+const lineSnippet = (text: string, regex: RegExp, contextLines = 2): string | undefined => {
+  const lines = text.split("\n");
+  let matchIdx = -1;
+  for (let i = 0; i < lines.length; i++) {
+    if (regex.test(lines[i])) {
+      matchIdx = i;
+      break;
+    }
+  }
+  if (matchIdx === -1) return undefined;
+
+  const start = Math.max(0, matchIdx - contextLines);
+  const end = Math.min(lines.length, matchIdx + contextLines + 1);
+  const slice = lines.slice(start, end);
+
+  const parts: string[] = [];
+  if (start > 0) parts.push(`...(${start} lines above)`);
+  parts.push(...slice);
+  if (end < lines.length) parts.push(`...(${lines.length - end} lines below)`);
+  return parts.join("\n");
+};
+
+/** Build full searchable text for a message. */
+const fullText = (msg: Message): string => {
+  if ((msg as any).role === "bashExecution") {
+    return `${(msg as any).command ?? ""} ${(msg as any).output ?? ""}`;
+  }
+  return textOf(msg.content);
+};
+
+export const searchEntries = (
+  entries: RenderedEntry[],
+  messages: Message[],
+  query?: string,
+): SearchHit[] => {
+  if (!query?.trim()) return entries;
+
+  const rawQuery = query.trim();
+
+  // If query looks like a single regex pattern (contains metacharacters),
+  // treat the whole thing as one pattern — don't split into terms
+  if (looksLikeRegex(rawQuery)) {
+    const regex = safeRegex(rawQuery);
+    const hits: SearchHit[] = [];
+    for (let i = 0; i < entries.length; i++) {
+      const e = entries[i];
+      const msg = messages[i];
+      const text = msg ? fullText(msg) : e.summary;
+      const filePart = e.files?.join(" ") ?? "";
+      const hay = `${e.role} ${text} ${filePart}`;
+      if (regex.test(hay)) {
+        const snip = lineSnippet(text, regex);
+        hits.push({ ...e, snippet: snip, matchCount: 1 });
+      }
+    }
+    return hits;
+  }
+
+  // Natural language / multi-word query: BM25 scoring
+  const rawTerms = rawQuery.split(/\s+/);
+  const terms = filterStopwords(rawTerms);
+  const snipRe = snippetRegex(terms);
+
+  // Build all docs for BM25 context
+  const docs: string[] = [];
+  for (let i = 0; i < entries.length; i++) {
+    const e = entries[i];
+    const msg = messages[i];
+    const text = msg ? fullText(msg) : e.summary;
+    const filePart = e.files?.join(" ") ?? "";
+    docs.push(`${e.role} ${text} ${filePart}`);
+  }
+
+  const ctx = buildBM25Context(docs, terms);
+
+  const scored: Array<{ hit: SearchHit; score: number }> = [];
+  for (let i = 0; i < entries.length; i++) {
+    const e = entries[i];
+    const hay = docs[i];
+    const mc = countMatches(hay, terms);
+    if (mc === 0) continue;
+    const score = bm25Score(hay, terms, ctx);
+    const text = messages[i] ? fullText(messages[i]) : e.summary;
+    const snip = lineSnippet(text, snipRe);
+    scored.push({
+      hit: { ...e, snippet: snip, matchCount: mc },
+      score,
+    });
+  }
+
+  // Sort by BM25 score desc
+  scored.sort((a, b) => b.score - a.score);
+  return scored.map((s) => s.hit);
+};