pi-blackhole 0.2.0

package/src/om/serialize.ts

@@ -0,0 +1,224 @@
+/**
+ * Serialize branch entries and render source-addressed chunks.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/serialize.ts)
+ * Unmodified.
+ */
+import type { Message, TextContent, ToolResultMessage } from "@earendil-works/pi-ai";
+
+function pad(n: number): string {
+	return n.toString().padStart(2, "0");
+}
+
+function fmtLocal(d: Date): string {
+	return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
+}
+
+function formatTimestamp(v: number | string | undefined): string {
+	if (v === undefined) return "????-??-?? ??:??";
+	const d = new Date(v);
+	return Number.isNaN(d.getTime()) ? "????-??-?? ??:??" : fmtLocal(d);
+}
+
+function formatRecallTimestamp(...values: Array<number | string | undefined>): string {
+	for (const v of values) {
+		if (v === undefined) continue;
+		const d = new Date(v);
+		if (!Number.isNaN(d.getTime())) return fmtLocal(d);
+	}
+	return "Unknown time";
+}
+
+function textAndPlaceholders(
+	content: unknown,
+	options: { omitRedactedThinking?: boolean; includeThinking?: boolean } = {},
+): string {
+	if (typeof content === "string") return content;
+	if (!Array.isArray(content)) return "[non-text content omitted]";
+
+	const parts: string[] = [];
+	for (const block of content as Array<Record<string, unknown>>) {
+		if (!block || typeof block !== "object") {
+			parts.push("[non-text content omitted]");
+			continue;
+		}
+		if (block.type === "text" && typeof block.text === "string") {
+			parts.push(block.text);
+			continue;
+		}
+		if (block.type === "thinking") {
+			if (options.omitRedactedThinking && block.redacted === true) continue;
+			if (options.includeThinking && typeof block.thinking === "string") {
+				parts.push(`[thinking: ${block.thinking}]`);
+				continue;
+			}
+			parts.push("[non-text content omitted]");
+			continue;
+		}
+		if (block.type === "toolCall" && typeof block.name === "string") {
+			parts.push(`[${block.name}(${JSON.stringify(block.arguments ?? {})})]`);
+			continue;
+		}
+		parts.push("[non-text content omitted]");
+	}
+	return parts.join("\n");
+}
+
+function textOnly(content: unknown): string {
+	if (content == null) return "";
+	if (typeof content === "string") return content;
+	if (!Array.isArray(content)) return "";
+	return content
+		.filter((b): b is TextContent => b?.type === "text" && typeof b.text === "string")
+		.map((b) => b.text)
+		.join("\n");
+}
+
+export function serializeConversation(messages: Message[]): string {
+	return messages
+		.map((msg): string | null => {
+			const time = formatTimestamp(msg.timestamp);
+			if (msg.role === "user") {
+				const text = textOnly(msg.content);
+				return `[User @ ${time}]: ${text}`;
+			}
+			if (msg.role === "assistant") {
+				const body = textAndPlaceholders(msg.content, {
+					includeThinking: true,
+					omitRedactedThinking: true,
+				})
+					.split("\n")
+					.filter(Boolean)
+					.join("\n");
+				if (!body) return null;
+				return `[Assistant @ ${time}]: ${body}`;
+			}
+			const text = textOnly(msg.content);
+			return `[Tool result for ${(msg as ToolResultMessage).toolName} @ ${time}]: ${text}`;
+		})
+		.filter((line): line is string => line !== null)
+		.join("\n\n");
+}
+
+export function nowTimestamp(): string {
+	return fmtLocal(new Date());
+}
+
+export const MAX_RECORD_CONTENT_CHARS = 10_000;
+
+export function truncateRecordContent(content: string): string {
+	if (content.length <= MAX_RECORD_CONTENT_CHARS) return content;
+	const head = content.slice(0, MAX_RECORD_CONTENT_CHARS);
+	const dropped = content.length - MAX_RECORD_CONTENT_CHARS;
+	return `${head} … [truncated ${dropped} chars]`;
+}
+
+export type RenderableEntry = {
+	type: string;
+	id?: string;
+	timestamp?: string;
+	message?: unknown;
+	customType?: string;
+	content?: unknown;
+	summary?: unknown;
+};
+
+function renderCustomMessage(entry: RenderableEntry, options: { recallFormat: boolean }): string {
+	const time = options.recallFormat ? formatRecallTimestamp(entry.timestamp) : formatTimestamp(entry.timestamp);
+	const text = options.recallFormat
+		? textAndPlaceholders(entry.content)
+		: typeof entry.content === "string"
+			? entry.content
+			: Array.isArray(entry.content)
+				? (entry.content as Array<{ type?: string; text?: string }>)
+						.filter((b) => b?.type === "text" && typeof b.text === "string")
+						.map((b) => b.text as string)
+						.join("\n")
+				: "";
+	if (options.recallFormat) {
+		const origin = entry.customType ? `Custom message (${entry.customType})` : "Custom message";
+		return `[${origin} @ ${time}]: ${text}`;
+	}
+	const tag = entry.customType ? `Custom (${entry.customType})` : "Custom";
+	return `[${tag} @ ${time}]: ${text}`;
+}
+
+export function serializeBranchEntries(entries: RenderableEntry[]): string {
+	const blocks: string[] = [];
+	for (const entry of entries) {
+		if (entry.type === "message" && entry.message) {
+			const part = serializeConversation([entry.message as Message]);
+			if (part) blocks.push(part);
+			continue;
+		}
+		if (entry.type === "custom_message") {
+			blocks.push(renderCustomMessage(entry, { recallFormat: false }));
+			continue;
+		}
+		if (entry.type === "branch_summary" && typeof entry.summary === "string") {
+			const time = formatTimestamp(entry.timestamp);
+			blocks.push(`[Branch summary @ ${time}]: ${entry.summary}`);
+		}
+	}
+	return blocks.join("\n\n");
+}
+
+export type SourceAddressedSerialization = {
+	text: string;
+	sourceEntryIds: string[];
+};
+
+function isSourceRenderableEntry(entry: RenderableEntry): boolean {
+	return entry.type === "message" || entry.type === "custom_message" || entry.type === "branch_summary";
+}
+
+export function serializeSourceAddressedBranchEntries(entries: RenderableEntry[]): SourceAddressedSerialization {
+	const blocks: string[] = [];
+	const sourceEntryIds: string[] = [];
+	for (const entry of entries) {
+		if (!entry.id || !isSourceRenderableEntry(entry)) continue;
+		const rendered = serializeBranchEntries([entry]);
+		if (!rendered.trim()) continue;
+		sourceEntryIds.push(entry.id);
+		blocks.push(`[Source entry id: ${entry.id}]\n${rendered}`);
+	}
+	return { text: blocks.join("\n\n"), sourceEntryIds };
+}
+
+function renderRecallMessage(entry: RenderableEntry): string | null {
+	if (!entry.message || typeof entry.message !== "object") return null;
+	const msg = entry.message as Message;
+	const time = formatRecallTimestamp(msg.timestamp, entry.timestamp);
+	if (msg.role === "user") {
+		return `[User @ ${time}]: ${textAndPlaceholders(msg.content)}`;
+	}
+	if (msg.role === "assistant") {
+		const body = textAndPlaceholders(msg.content, {
+			includeThinking: true,
+			omitRedactedThinking: true,
+		})
+			.split("\n")
+			.filter(Boolean)
+			.join("\n");
+		if (!body) return null;
+		return `[Assistant @ ${time}]: ${body}`;
+	}
+	return `[Tool result: ${(msg as ToolResultMessage).toolName} @ ${time}]: ${textAndPlaceholders(msg.content)}`;
+}
+
+export function renderRecallSourceEntry(entry: RenderableEntry): string | null {
+	if (entry.type === "message") return renderRecallMessage(entry);
+	if (entry.type === "custom_message") return renderCustomMessage(entry, { recallFormat: true });
+	if (entry.type === "branch_summary" && typeof entry.summary === "string") {
+		const time = formatRecallTimestamp(entry.timestamp);
+		return `[Branch summary @ ${time}]: ${entry.summary}`;
+	}
+	return null;
+}
+
+export function renderRecallSourceEntries(entries: RenderableEntry[]): string {
+	return entries
+		.map(renderRecallSourceEntry)
+		.filter((block): block is string => block !== null && block.trim().length > 0)
+		.join("\n\n");
+}

package/src/om/tokens.ts

@@ -0,0 +1,33 @@
+/**
+ * Token estimation for serialized entries.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/tokens.ts)
+ * Unmodified.
+ */
+import { estimateTokens as estimateMessageTokens } from "@earendil-works/pi-coding-agent";
+
+export function estimateStringTokens(text: string): number {
+	return Math.ceil(text.length / 4);
+}
+
+export function estimateEntryTokens(entry: { type: string; message?: unknown; content?: unknown; summary?: unknown }): number {
+	if (entry.type === "message" && entry.message) {
+		return estimateMessageTokens(entry.message as Parameters<typeof estimateMessageTokens>[0]);
+	}
+	if (entry.type === "custom_message" && entry.content) {
+		const content = entry.content;
+		if (typeof content === "string") return estimateStringTokens(content);
+		if (Array.isArray(content)) {
+			let total = 0;
+			for (const block of content) {
+				if (block.type === "text" && block.text) total += estimateStringTokens(block.text);
+			}
+			return total;
+		}
+	}
+	if (entry.type === "branch_summary" && typeof entry.summary === "string") {
+		return estimateStringTokens(entry.summary);
+	}
+	return 0;
+}
+

package/src/sections.ts

@@ -0,0 +1,18 @@
+/**
+ * Pi-vcc section types.
+ *
+ * Upstream: https://github.com/sting8k/pi-vcc (src/sections.ts)
+ * Unmodified.
+ */
+import type { TranscriptEntry } from "./core/brief";
+
+export interface SectionData {
+  sessionGoal: string[];
+  outstandingContext: string[];
+  filesAndChanges: string[];
+  commits: string[];
+  userPreferences: string[];
+  briefTranscript: string;
+  /** Structured transcript entries (verbose object format) */
+  transcriptEntries: TranscriptEntry[];
+}

package/src/tools/recall.ts

@@ -0,0 +1,212 @@
+/**
+ * Unified recall tool — handles #N transcript indices, 12-char hex om memory ids,
+ * and free-text search (BM25 + regex).
+ *
+ * Created by pi-vcc-om. Replaces pi-vcc's vcc_recall and OM's standalone recall-observation.
+ */
+import { Type } from "@earendil-works/pi-ai";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { loadAllMessages } from "../core/load-messages";
+import { searchEntries } from "../core/search-entries";
+import { formatRecallOutput } from "../core/format-recall";
+import { getActiveLineageEntryIds } from "../core/lineage";
+import { normalizeRecallScope } from "../core/recall-scope";
+import {
+	recallMemorySources,
+	type Entry,
+} from "../om/ledger/recall.js";
+import { renderRecallSourceEntries } from "../om/serialize.js";
+import {
+	findObservationsForEntryIds,
+	findReflectionsForEntryIds,
+	formatRelatedObservations,
+	buildIndexMap,
+	formatEntryIndexAnnotation,
+} from "../om/reverse-recall.js";
+
+// ── Pi-vcc recall logic ──────────────────────────────────────────────────
+
+const DEFAULT_RECENT = 25;
+const PAGE_SIZE = 5;
+
+const invalidExpandIndices = (requested: number[], available: Set<number>): number[] =>
+	requested.filter((i) => !Number.isInteger(i) || !available.has(i));
+
+async function vccRecall(params: { query?: string; expand?: number[]; page?: number; scope?: "lineage" | "all" }, ctx: any) {
+	const sessionFile = ctx.sessionManager.getSessionFile();
+	if (!sessionFile) {
+		return { content: [{ type: "text" as const, text: "No session file available." }], details: undefined };
+	}
+	const scope = normalizeRecallScope(params.scope);
+	const lineageEntryIds = scope === "lineage" ? getActiveLineageEntryIds(ctx.sessionManager) : undefined;
+	const expandSet = new Set(params.expand ?? []);
+	const hasExpand = expandSet.size > 0;
+
+	if (hasExpand && !params.query) {
+		const { rendered: fullMsgs } = loadAllMessages(sessionFile, true, lineageEntryIds);
+		const requested = [...expandSet];
+		const byIndex = new Map(fullMsgs.map((m) => [m.index, m]));
+		const invalid = invalidExpandIndices(requested, new Set(byIndex.keys()));
+		if (invalid.length > 0) {
+			return { content: [{ type: "text" as const, text: `Cannot expand indices outside ${scope === "all" ? "session history" : "active lineage"}: ${invalid.join(", ")}` }], details: undefined };
+		}
+		const expanded = requested.map((i) => byIndex.get(i)).filter((m): m is NonNullable<typeof m> => Boolean(m));
+		let output = (scope === "all" ? "Scope: all\n\n" : "") + formatRecallOutput(expanded);
+
+		// Coupling: look up related OM observations
+		const expandedIds = expanded.map((e) => e.id).filter(Boolean);
+		if (expandedIds.length > 0) {
+			try {
+				const branchEntries = ctx.sessionManager.getBranch() as Entry[];
+				const obs = findObservationsForEntryIds(branchEntries, expandedIds);
+				const refs = findReflectionsForEntryIds(branchEntries, expandedIds);
+				if (obs.length > 0 || refs.length > 0) {
+					output += "\n\n" + formatRelatedObservations(obs, refs);
+				}
+			} catch { /* branch may not be available */ }
+		}
+
+		return { content: [{ type: "text" as const, text: output }], details: undefined };
+	}
+
+	const { rendered: msgs, rawMessages } = loadAllMessages(sessionFile, false, lineageEntryIds);
+	const allResults = params.query?.trim()
+		? searchEntries(msgs, rawMessages, params.query)
+		: msgs.slice(-DEFAULT_RECENT);
+
+	if (params.query?.trim()) {
+		const page = Math.max(1, params.page ?? 1);
+		const start = (page - 1) * PAGE_SIZE;
+		const pageResults = allResults.slice(start, start + PAGE_SIZE);
+		const totalPages = Math.ceil(allResults.length / PAGE_SIZE);
+		const scopeSuffix = scope === "all" ? " (scope: all)" : "";
+		const header = totalPages > 1
+			? `Page ${page}/${totalPages} (${allResults.length} total matches${scopeSuffix})`
+			: `${allResults.length} matches${scopeSuffix}`;
+		const footer = page < totalPages
+			? `\n--- Use page:${page + 1}${scope === "all" ? " with scope:'all'" : ""} for more results ---`
+			: "";
+		let output = formatRecallOutput(pageResults, params.query, header) + footer;
+
+		// Coupling: augment search results with related observations
+		const pageResultIds = pageResults.map((r) => r.id).filter(Boolean);
+		if (pageResultIds.length > 0) {
+			try {
+				const branchEntries = ctx.sessionManager.getBranch() as Entry[];
+				const obs = findObservationsForEntryIds(branchEntries, pageResultIds);
+				const refs = findReflectionsForEntryIds(branchEntries, pageResultIds);
+				if (obs.length > 0 || refs.length > 0) {
+					output += "\n\n" + formatRelatedObservations(obs, refs);
+				}
+			} catch { /* branch may not be available */ }
+		}
+
+		return { content: [{ type: "text" as const, text: output }], details: undefined };
+	}
+
+	const output = (scope === "all" ? "Scope: all\n\n" : "") + formatRecallOutput(allResults, params.query);
+	return { content: [{ type: "text" as const, text: output }], details: undefined };
+}
+
+// ── Observational-memory recall logic ─────────────────────────────────────
+
+const MEMORY_ID_PATTERN = /^[a-f0-9]{12}$/;
+const VCC_ENTRY_PATTERN = /^#(\d+)$/;
+
+async function omRecall(memoryId: string, ctx: any) {
+	if (!MEMORY_ID_PATTERN.test(memoryId)) {
+		return { content: [{ type: "text" as const, text: `Memory id must be 12 lowercase hex characters. Received: ${memoryId}` }], details: undefined };
+	}
+	const branchEntries = ctx.sessionManager.getBranch() as Entry[];
+	const result = recallMemorySources(branchEntries, memoryId);
+	if (result.status === "not_found") {
+		return { content: [{ type: "text" as const, text: `No observation or reflection with id ${memoryId} was found on the current branch.` }], details: undefined };
+	}
+	const lines: string[] = [];
+	if (result.collision) lines.push(`ID ${result.memoryId} matched multiple items.`);
+	for (const ref of result.reflections) {
+		lines.push(`[${ref.reflection.id}] ${ref.reflection.content}`);
+	}
+	for (const obs of result.observations) {
+		const dropped = obs.status === "dropped" ? " [dropped]" : "";
+		lines.push(`[${obs.observation.id}]${dropped} ${obs.observation.timestamp} [${obs.observation.relevance}] ${obs.observation.content}`);
+	}
+	if (result.sourceEntries.length > 0) {
+		lines.push("");
+		lines.push("Sources:");
+		// Cross-format nav: annotate source entries with #N indices
+		try {
+			const sessionFile = ctx.sessionManager.getSessionFile();
+			if (sessionFile) {
+				const { rendered } = await Promise.resolve(loadAllMessages(sessionFile, false));
+				const idToIndex = buildIndexMap(rendered);
+				const indexAnnotation = formatEntryIndexAnnotation(
+					result.observations.flatMap((o) => o.sourceEntryIds),
+					idToIndex,
+				);
+				if (indexAnnotation) lines.push(indexAnnotation);
+			}
+		} catch { /* ignore errors from index mapping */ }
+		lines.push(renderRecallSourceEntries(result.sourceEntries));
+	}
+	const text = lines.join("\n") || `Memory ${memoryId} found, but no evidence rendered.`;
+	return { content: [{ type: "text" as const, text }], details: undefined };
+}
+
+// ── Unified recall tool ──────────────────────────────────────────────────
+
+export function registerRecallTool(pi: ExtensionAPI): void {
+	pi.registerTool({
+		name: "recall",
+		label: "Recall",
+		description:
+			"Recall session history or memory evidence. This is text/pattern matching, NOT semantic search. Accepts:\n" +
+			"- A 12-char hex id [a1b2c3d4e5f6] to recover observation/reflection source evidence.\n" +
+			"- A #N entry index to expand a specific transcript entry from compacted output.\n" +
+			"- A text/regex query to search conversation content. Multi-word queries use BM25 ranking with stopword filtering.\n" +
+			"Search tips: use unique concrete words (file names, function names, exact phrases), not conceptual questions. Regex metacharacters (|, *, .) trigger regex mode. Default scope is active lineage; use scope:'all' for off-lineage branches.",
+		promptSnippet:
+			"recall: Text/regex search (not semantic). Also: 12-char hex ids recover obs/reflection sources; #N indices expand transcript entries. Use concrete words for search. scope:'all' for off-lineage.",
+		promptGuidelines: [
+			"Use recall with a 12-char hex id before making an important decision that depends on a compacted observation or reflection whose details are unclear.",
+			"Use recall with a search query when you need to find specific conversation content. Use unique concrete terms (file paths, function names, error messages, exact phrases), not conceptual questions. Example: 'merge_design.md' not 'what did we decide about merging'.",
+			"Use recall with #N to expand a transcript entry reference from compacted output.",
+			"After compaction, the summary includes observation/reflection ids in brackets. Use recall with those ids to recover full source evidence.",
+			"If you get no results, try fewer terms, use a distinctive single word, or use a regex pattern (e.g. 'fork.*pi-vcc').",
+		],
+		parameters: Type.Object({
+			query: Type.Optional(
+				Type.String({ description: "12-char hex id, #N entry index, or text/regex search terms. NOT semantic — use concrete words (file names, quoted phrases, identifiers). Regex metacharacters trigger regex mode. Multi-word = BM25-ranked OR." }),
+			),
+			expand: Type.Optional(
+				Type.Array(Type.Number(), { description: "Entry indices to return full untruncated content for (blackhole format only)." }),
+			),
+			page: Type.Optional(
+				Type.Number({ description: "Page number (1-based) for paginated search results. Default: 1." }),
+			),
+			scope: Type.Optional(
+				Type.Union([
+					Type.Literal("lineage"),
+					Type.Literal("all"),
+				], { description: "Search scope. Default: lineage; all includes off-lineage branches." }),
+			),
+		}),
+		async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
+			const q = params.query?.trim();
+			// Dispatch by format
+			if (q && VCC_ENTRY_PATTERN.test(q)) {
+				// #N → expand entry indices
+				const match = q.match(VCC_ENTRY_PATTERN);
+				const index = match ? parseInt(match[1], 10) : NaN;
+				if (!Number.isNaN(index)) {
+					return vccRecall({ query: "", expand: [index] }, ctx);
+				}
+			}
+			if (q && MEMORY_ID_PATTERN.test(q)) {
+				return omRecall(q, ctx);
+			}
+			// Default: pi-vcc search
+			return vccRecall(params, ctx);
+		},
+	});
+}

package/src/types.ts

@@ -0,0 +1,19 @@
+/**
+ * Normalized block types shared across pi-vcc pipeline.
+ *
+ * Upstream: https://github.com/sting8k/pi-vcc (src/types.ts)
+ * Unmodified.
+ */
+export interface FileOps {
+  readFiles?: string[];
+  modifiedFiles?: string[];
+  createdFiles?: string[];
+}
+
+export type NormalizedBlock =
+  | { kind: "user"; text: string; sourceIndex?: number }
+  | { kind: "assistant"; text: string; sourceIndex?: number }
+  | { kind: "tool_call"; name: string; args: Record<string, unknown>; sourceIndex?: number }
+  | { kind: "tool_result"; name: string; text: string; isError: boolean; sourceIndex?: number }
+  | { kind: "bash"; command: string; output: string; exitCode: number | undefined; sourceIndex?: number }
+  | { kind: "thinking"; text: string; redacted: boolean; sourceIndex?: number };

package/vitest.config.ts

@@ -0,0 +1,41 @@
+import { defineConfig } from "vitest/config";
+
+// ── pnpm global store paths for @earendil-works packages ──────────────────
+
+const GLOBAL_PNPM =
+	"/home/kovalik/.local/share/pnpm/global/5/.pnpm";
+
+const PKGS: Record<string, string> = {
+	"@earendil-works/pi-ai": `${GLOBAL_PNPM}/@earendil-works+pi-ai@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-ai`,
+	"@earendil-works/pi-ai/oauth": `${GLOBAL_PNPM}/@earendil-works+pi-ai@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-ai/oauth`,
+	"@earendil-works/pi-agent-core": `${GLOBAL_PNPM}/@earendil-works+pi-agent-core@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-agent-core`,
+	"@earendil-works/pi-coding-agent": `${GLOBAL_PNPM}/@earendil-works+pi-coding-agent@0.75.4_ws@8.20.1_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent`,
+	"@earendil-works/pi-tui": `${GLOBAL_PNPM}/@earendil-works+pi-tui@0.75.4/node_modules/@earendil-works/pi-tui`,
+};
+
+export default defineConfig({
+	test: {
+		globals: true,
+		environment: "node",
+		testTimeout: 10000,
+		include: ["tests/**/*.test.ts"],
+	},
+	resolve: {
+		alias: [
+			// Map @earendil-works/* to global pnpm store
+			...Object.entries(PKGS).map(([name, path]) => ({
+				find: new RegExp(`^${escapeRegex(name)}$`),
+				replacement: path,
+			})),
+			// Resolve .js → extension-less for our source files
+			{
+				find: /\.js$/,
+				replacement: "",
+			},
+		],
+	},
+});
+
+function escapeRegex(s: string): string {
+	return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}