pi-blackhole 0.2.0

package/src/om/debug-log.ts

@@ -0,0 +1,55 @@
+/**
+ * Debug logging — writes JSONL to ~/.pi/agent/pi-blackhole/debug.ndjson.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/debug-log.ts)
+ * Modified: path changed from observational-memory/ to pi-blackhole/.
+ */
+import { AsyncLocalStorage } from "node:async_hooks";
+import { existsSync, mkdirSync, renameSync, statSync, unlinkSync, appendFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+export const DEBUG_LOG_MAX_BYTES = 10 * 1024 * 1024;
+export const DEBUG_LOG_RELATIVE_PATH = join("pi-blackhole", "debug.ndjson");
+
+interface DebugLogContext {
+	enabled: boolean;
+	cwd?: string;
+	runId?: string;
+}
+
+const storage = new AsyncLocalStorage<DebugLogContext>();
+
+export function withDebugLogContext<T>(context: DebugLogContext, fn: () => T): T {
+	const parent = storage.getStore();
+	return storage.run({ ...parent, ...context }, fn);
+}
+
+export function debugLog(event: string, data: Record<string, unknown> = {}): void {
+	const context = storage.getStore();
+	if (context?.enabled !== true) return;
+
+	try {
+		const path = join(getAgentDir(), DEBUG_LOG_RELATIVE_PATH);
+		mkdirSync(dirname(path), { recursive: true });
+		rotateIfNeeded(path);
+		const payload = {
+			ts: new Date().toISOString(),
+			event,
+			cwd: context.cwd,
+			runId: context.runId,
+			data,
+		};
+		appendFileSync(path, `${JSON.stringify(payload)}\n`, "utf-8");
+	} catch {
+		// Debug logging must never affect memory behavior.
+	}
+}
+
+function rotateIfNeeded(path: string): void {
+	if (!existsSync(path)) return;
+	if (statSync(path).size < DEBUG_LOG_MAX_BYTES) return;
+	const backupPath = `${path}.1`;
+	if (existsSync(backupPath)) unlinkSync(backupPath);
+	renameSync(path, backupPath);
+}

package/src/om/ids.ts

@@ -0,0 +1,5 @@
+import { createHash } from "node:crypto";
+
+export function hashId(content: string): string {
+	return createHash("sha256").update(content).digest("hex").slice(0, 12);
+}

package/src/om/ledger/fold.ts

@@ -0,0 +1,106 @@
+/**
+ * Ledger folding — deduplicates observations/reflections and applies tombstones.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/fold.ts)
+ * Unmodified.
+ */
+import {
+	isObservationsDroppedData,
+	isObservationsRecordedData,
+	isReflectionsRecordedData,
+	OM_OBSERVATIONS_DROPPED,
+	OM_OBSERVATIONS_RECORDED,
+	OM_REFLECTIONS_RECORDED,
+	type Entry,
+	type Observation,
+	type Reflection,
+} from "./types.js";
+
+export type FoldLedgerOptions = {
+	/** Fold entries from branch root through this entry id, inclusive. Omit to fold through branch tip. */
+	upToEntryId?: string;
+};
+
+export type FoldedLedger = {
+	/** All first-valid observation records encountered through the fold boundary, including dropped observations. */
+	observations: Observation[];
+	/** Observation records not tombstoned by a folded drop entry. */
+	activeObservations: Observation[];
+	/** Tombstoned observation ids, including ids that may not have a corresponding folded observation. */
+	droppedObservationIds: Set<string>;
+	/** All first-valid reflection records encountered through the fold boundary. */
+	reflections: Reflection[];
+	/** All first-valid observation records by id, including dropped observations. */
+	observationsById: Map<string, Observation>;
+	/** All first-valid reflection records by id. */
+	reflectionsById: Map<string, Reflection>;
+};
+
+function foldEndIndex(entries: Entry[], upToEntryId: string | undefined): number {
+	if (!upToEntryId) return entries.length - 1;
+	const idx = entries.findIndex((entry) => entry.id === upToEntryId);
+	return idx === -1 ? entries.length - 1 : idx;
+}
+
+function isCustomEntry(entry: Entry, customType: string): boolean {
+	return entry.type === "custom" && entry.customType === customType;
+}
+
+/**
+ * Fold valid V3 memory ledger entries from the branch root through the target entry.
+ *
+ * Unknown custom entries, old V2 entries, invalid V3-shaped data, and compaction details are ignored.
+ * Observations and reflections use first-valid-record-wins semantics. Drops are tombstones and are
+ * retained even when the dropped id is unknown at the time of folding.
+ */
+export function foldLedger(entries: Entry[], options: FoldLedgerOptions = {}): FoldedLedger {
+	const observationsById = new Map<string, Observation>();
+	const reflectionsById = new Map<string, Reflection>();
+	const droppedObservationIds = new Set<string>();
+	const endIdx = foldEndIndex(entries, options.upToEntryId);
+
+	for (let i = 0; i <= endIdx; i++) {
+		const entry = entries[i];
+		if (!entry) continue;
+
+		if (isCustomEntry(entry, OM_OBSERVATIONS_RECORDED)) {
+			if (!isObservationsRecordedData(entry.data)) continue;
+			for (const observation of entry.data.observations) {
+				if (!observationsById.has(observation.id)) {
+					observationsById.set(observation.id, observation);
+				}
+			}
+			continue;
+		}
+
+		if (isCustomEntry(entry, OM_REFLECTIONS_RECORDED)) {
+			if (!isReflectionsRecordedData(entry.data)) continue;
+			for (const reflection of entry.data.reflections) {
+				if (!reflectionsById.has(reflection.id)) {
+					reflectionsById.set(reflection.id, reflection);
+				}
+			}
+			continue;
+		}
+
+		if (isCustomEntry(entry, OM_OBSERVATIONS_DROPPED)) {
+			if (!isObservationsDroppedData(entry.data)) continue;
+			for (const observationId of entry.data.observationIds) {
+				droppedObservationIds.add(observationId);
+			}
+		}
+	}
+
+	const observations = Array.from(observationsById.values());
+	const activeObservations = observations.filter((observation) => !droppedObservationIds.has(observation.id));
+	const reflections = Array.from(reflectionsById.values());
+
+	return {
+		observations,
+		activeObservations,
+		droppedObservationIds,
+		reflections,
+		observationsById,
+		reflectionsById,
+	};
+}

package/src/om/ledger/index.ts

@@ -0,0 +1,6 @@
+export * from "./types.js";
+export * from "./progress.js";
+export * from "./fold.js";
+export * from "./projection.js";
+export * from "./recall.js";
+export * from "./render-summary.js";

package/src/om/ledger/progress.ts

@@ -0,0 +1,225 @@
+import { estimateEntryTokens } from "../tokens.js";
+import {
+	OM_OBSERVATIONS_DROPPED,
+	OM_OBSERVATIONS_RECORDED,
+	OM_REFLECTIONS_RECORDED,
+	isObservationsRecordedData,
+	isReflectionsRecordedData,
+	type Entry,
+	type Observation,
+	type Reflection,
+	type V3MemoryCustomType,
+} from "./types.js";
+
+const SOURCE_ENTRY_TYPES = new Set(["message", "custom_message", "branch_summary"]);
+
+export function isSourceEntry(entry: Entry): boolean {
+	return SOURCE_ENTRY_TYPES.has(entry.type);
+}
+
+export function entryIndexById(entries: Entry[]): Map<string, number> {
+	const idToIndex = new Map<string, number>();
+	for (let i = 0; i < entries.length; i++) idToIndex.set(entries[i].id, i);
+	return idToIndex;
+}
+
+export function entryIndexForId(entries: Entry[], entryId: string | undefined): number {
+	if (!entryId) return -1;
+	const idx = entryIndexById(entries).get(entryId);
+	return idx ?? -1;
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}
+
+function isNonEmptyArray(value: unknown): value is unknown[] {
+	return Array.isArray(value) && value.length > 0;
+}
+
+function isValidCoverageEntry(entry: Entry, customType: V3MemoryCustomType): entry is Entry & { data: { coversUpToId: string } } {
+	if (entry.type !== "custom" || entry.customType !== customType) return false;
+	if (!isObject(entry.data) || typeof entry.data.coversUpToId !== "string") return false;
+
+	if (customType === OM_OBSERVATIONS_RECORDED) return isNonEmptyArray(entry.data.observations);
+	if (customType === OM_REFLECTIONS_RECORDED) return isNonEmptyArray(entry.data.reflections);
+	return isNonEmptyArray(entry.data.observationIds);
+}
+
+export function latestCoverageIndex(entries: Entry[], customType: V3MemoryCustomType): number {
+	const idToIndex = entryIndexById(entries);
+	let latest = -1;
+
+	for (const entry of entries) {
+		if (!isValidCoverageEntry(entry, customType)) continue;
+		const coveredIndex = idToIndex.get(entry.data.coversUpToId);
+		if (coveredIndex === undefined) continue;
+		if (coveredIndex > latest) latest = coveredIndex;
+	}
+
+	return latest;
+}
+
+export function latestCoverageMarkerId(entries: Entry[], customType: V3MemoryCustomType): string | undefined {
+	const idToIndex = entryIndexById(entries);
+	let latestIndex = -1;
+	let latestMarkerId: string | undefined;
+
+	for (const entry of entries) {
+		if (!isValidCoverageEntry(entry, customType)) continue;
+		const coveredIndex = idToIndex.get(entry.data.coversUpToId);
+		if (coveredIndex === undefined) continue;
+		if (coveredIndex > latestIndex) {
+			latestIndex = coveredIndex;
+			latestMarkerId = entry.data.coversUpToId;
+		}
+	}
+
+	return latestMarkerId;
+}
+
+export function earlierCoverageMarkerId(entries: Entry[], firstId: string | undefined, secondId: string | undefined): string | undefined {
+	if (!firstId) return secondId;
+	if (!secondId) return firstId;
+
+	const idToIndex = entryIndexById(entries);
+	const firstIndex = idToIndex.get(firstId);
+	const secondIndex = idToIndex.get(secondId);
+	if (firstIndex === undefined) return secondIndex === undefined ? undefined : secondId;
+	if (secondIndex === undefined) return firstId;
+	return firstIndex <= secondIndex ? firstId : secondId;
+}
+
+export function rawTokensAfterIndex(entries: Entry[], index: number): number {
+	let total = 0;
+	for (let i = Math.max(0, index + 1); i < entries.length; i++) {
+		if (isSourceEntry(entries[i])) total += estimateEntryTokens(entries[i]);
+	}
+	return total;
+}
+
+export function rawTokensSinceCoverage(entries: Entry[], customType: V3MemoryCustomType): number {
+	return rawTokensAfterIndex(entries, latestCoverageIndex(entries, customType));
+}
+
+export function rawTokensSinceObservationCoverage(entries: Entry[]): number {
+	return rawTokensSinceCoverage(entries, OM_OBSERVATIONS_RECORDED);
+}
+
+export function rawTokensSinceReflectionCoverage(entries: Entry[]): number {
+	return rawTokensSinceCoverage(entries, OM_REFLECTIONS_RECORDED);
+}
+
+export function rawTokensSinceDropCoverage(entries: Entry[]): number {
+	return rawTokensSinceCoverage(entries, OM_OBSERVATIONS_DROPPED);
+}
+
+export function findLastCompactionIndex(entries: Entry[]): number {
+	for (let i = entries.length - 1; i >= 0; i--) {
+		if (entries[i].type === "compaction") return i;
+	}
+	return -1;
+}
+
+export function rawTokensSinceLastCompaction(entries: Entry[]): number {
+	const compactionIndex = findLastCompactionIndex(entries);
+	if (compactionIndex === -1) return rawTokensAfterIndex(entries, -1);
+
+	const firstKeptEntryId = entries[compactionIndex].firstKeptEntryId;
+	const firstKeptIndex = entryIndexForId(entries, firstKeptEntryId);
+
+	if (firstKeptIndex === -1) return rawTokensAfterIndex(entries, compactionIndex);
+	return rawTokensAfterIndex(entries, firstKeptIndex - 1);
+}
+
+/**
+ * Extract observations created since the given entry index.
+ * Walks the branch and collects observations from OM_OBSERVATIONS_RECORDED
+ * entries that were appended AFTER the given index.
+ */
+export function observationsCreatedAfterIndex(
+	entries: Entry[],
+	sinceIndex: number,
+): Observation[] {
+	const observations: Observation[] = [];
+	const seen = new Set<string>();
+
+	for (let i = sinceIndex + 1; i < entries.length; i++) {
+		const entry = entries[i];
+		if (entry.type !== "custom") continue;
+		if (entry.customType !== OM_OBSERVATIONS_RECORDED) continue;
+		if (!isObservationsRecordedData(entry.data)) continue;
+		for (const obs of entry.data.observations) {
+			if (!seen.has(obs.id)) {
+				seen.add(obs.id);
+				observations.push(obs);
+			}
+		}
+	}
+	return observations;
+}
+
+/**
+ * Extract reflections created since the given entry index.
+ */
+export function reflectionsCreatedAfterIndex(
+	entries: Entry[],
+	sinceIndex: number,
+): Reflection[] {
+	const reflections: Reflection[] = [];
+	const seen = new Set<string>();
+
+	for (let i = sinceIndex + 1; i < entries.length; i++) {
+		const entry = entries[i];
+		if (entry.type !== "custom") continue;
+		if (entry.customType !== OM_REFLECTIONS_RECORDED) continue;
+		if (!isReflectionsRecordedData(entry.data)) continue;
+		for (const ref of entry.data.reflections) {
+			if (!seen.has(ref.id)) {
+				seen.add(ref.id);
+				reflections.push(ref);
+			}
+		}
+	}
+	return reflections;
+}
+
+/**
+ * Build a compact one-line summary of existing observations for context.
+ * Capped at maxTokens.
+ */
+export function buildExistingObservationsSummary(
+	observations: Observation[],
+	maxTokens: number,
+): string {
+	const lines: string[] = [];
+	let tokens = 0;
+	for (const obs of observations) {
+		const line = `[${obs.id}] ${obs.timestamp} [${obs.relevance}] ${obs.content}`;
+		const lineTokens = Math.ceil(line.length / 4);
+		if (tokens + lineTokens > maxTokens && lines.length > 0) break;
+		lines.push(line);
+		tokens += lineTokens;
+	}
+	return lines.join("\n");
+}
+
+/**
+ * Build a compact one-line summary of existing reflections for context.
+ * Capped at maxTokens.
+ */
+export function buildExistingReflectionsSummary(
+	reflections: Reflection[],
+	maxTokens: number,
+): string {
+	const lines: string[] = [];
+	let tokens = 0;
+	for (const ref of reflections) {
+		const line = `[${ref.id}] ${ref.content}`;
+		const lineTokens = Math.ceil(line.length / 4);
+		if (tokens + lineTokens > maxTokens && lines.length > 0) break;
+		lines.push(line);
+		tokens += lineTokens;
+	}
+	return lines.join("\n");
+}

package/src/om/ledger/projection.ts

@@ -0,0 +1,237 @@
+/**
+ * Compaction projection — builds projection slices for compaction events.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/projection.ts)
+ * Unmodified.
+ */
+import {
+	OM_FOLDED,
+	isMemoryDetails,
+	isObservationsDroppedEntry,
+	isObservationsRecordedEntry,
+	isReflectionsRecordedEntry,
+	type Entry,
+	type MemoryDetails,
+	type Observation,
+	type Reflection,
+} from "./types.js";
+
+export type Projection = {
+	observations: Observation[];
+	reflections: Reflection[];
+};
+
+export type ProjectionDiff = {
+	observationsOnlyInFull: Observation[];
+	reflectionsOnlyInFull: Reflection[];
+	droppedOnlyInFull: Observation[];
+};
+
+export type CompactionProjectionConfig = {
+	observationsPoolMaxTokens: number;
+};
+
+export type CompactionProjection = Projection & {
+	fullFold: boolean;
+	details: MemoryDetails;
+};
+
+type ProjectionBoundary =
+	| { kind: "entry"; entryId: string }
+	| { kind: "tip" }
+	| { kind: "none" };
+
+type ProjectionFoldOptions = {
+	observationsBoundary: ProjectionBoundary;
+	reflectionsBoundary: ProjectionBoundary;
+	dropsBoundary: ProjectionBoundary;
+};
+
+function entryIndexById(entries: Entry[]): Map<string, number> {
+	const indexes = new Map<string, number>();
+	for (let i = 0; i < entries.length; i++) indexes.set(entries[i].id, i);
+	return indexes;
+}
+
+function entryBoundary(entryId: string): ProjectionBoundary {
+	return { kind: "entry", entryId };
+}
+
+function tipBoundary(): ProjectionBoundary {
+	return { kind: "tip" };
+}
+
+function noneBoundary(): ProjectionBoundary {
+	return { kind: "none" };
+}
+
+function boundaryIndex(entries: Entry[], indexes: Map<string, number>, boundary: ProjectionBoundary): number {
+	if (boundary.kind === "tip") return entries.length - 1;
+	if (boundary.kind === "none") return -1;
+	return indexes.get(boundary.entryId) ?? -1;
+}
+
+function coverageIndex(entry: Entry & { data: { coversUpToId: string } }, indexes: Map<string, number>): number {
+	return indexes.get(entry.data.coversUpToId) ?? -1;
+}
+
+function isAtOrBefore(index: number, boundaryIndex: number): boolean {
+	return index >= 0 && boundaryIndex >= 0 && index <= boundaryIndex;
+}
+
+function isCoveredAtOrBefore(
+	entry: Entry & { data: { coversUpToId: string } },
+	indexes: Map<string, number>,
+	boundaryIndex: number,
+): boolean {
+	return isAtOrBefore(coverageIndex(entry, indexes), boundaryIndex);
+}
+
+function foldProjection(entries: Entry[], options: ProjectionFoldOptions): Projection {
+	const indexes = entryIndexById(entries);
+	const observationsBoundary = boundaryIndex(entries, indexes, options.observationsBoundary);
+	const reflectionsBoundary = boundaryIndex(entries, indexes, options.reflectionsBoundary);
+	const dropsBoundary = boundaryIndex(entries, indexes, options.dropsBoundary);
+	const observations: Observation[] = [];
+	const reflections: Reflection[] = [];
+	const observationsById = new Set<string>();
+	const reflectionsById = new Set<string>();
+	const droppedObservationIds = new Set<string>();
+
+	for (const entry of entries) {
+		if (isObservationsRecordedEntry(entry) && isCoveredAtOrBefore(entry, indexes, observationsBoundary)) {
+			for (const observation of entry.data.observations) {
+				if (observationsById.has(observation.id)) continue;
+				observationsById.add(observation.id);
+				observations.push(observation);
+			}
+			continue;
+		}
+
+		if (isReflectionsRecordedEntry(entry) && isCoveredAtOrBefore(entry, indexes, reflectionsBoundary)) {
+			for (const reflection of entry.data.reflections) {
+				if (reflectionsById.has(reflection.id)) continue;
+				reflectionsById.add(reflection.id);
+				reflections.push(reflection);
+			}
+			continue;
+		}
+
+		if (isObservationsDroppedEntry(entry) && isCoveredAtOrBefore(entry, indexes, dropsBoundary)) {
+			for (const observationId of entry.data.observationIds) droppedObservationIds.add(observationId);
+		}
+	}
+
+	return {
+		observations: observations.filter((observation) => !droppedObservationIds.has(observation.id)),
+		reflections,
+	};
+}
+
+function projectionFromMemoryDetails(details: MemoryDetails): Projection {
+	return {
+		observations: [...details.observations],
+		reflections: [...details.reflections],
+	};
+}
+
+function latestV3CompactionDetails(entries: Entry[]): MemoryDetails | undefined {
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		if (entry.type !== "compaction") continue;
+		const details = unwrapMemoryDetails(entry);
+		if (details) return details;
+	}
+	return undefined;
+}
+
+export function fullProjection(entries: Entry[], upToEntryId?: string): Projection {
+	const boundary = upToEntryId ? entryBoundary(upToEntryId) : tipBoundary();
+	return foldProjection(entries, {
+		observationsBoundary: boundary,
+		reflectionsBoundary: boundary,
+		dropsBoundary: boundary,
+	});
+}
+
+export function visibleProjection(entries: Entry[], upToEntryId?: string): Projection {
+	if (!upToEntryId) {
+		const details = latestV3CompactionDetails(entries);
+		return details ? projectionFromMemoryDetails(details) : { observations: [], reflections: [] };
+	}
+
+	return buildCompactionProjection(entries, upToEntryId, { observationsPoolMaxTokens: Number.POSITIVE_INFINITY });
+}
+
+function unwrapMemoryDetails(entry: Entry): MemoryDetails | undefined {
+	if (isMemoryDetails(entry.details)) return entry.details;
+	if (entry.details && typeof entry.details === "object" && !Array.isArray(entry.details)) {
+		const nested = (entry.details as Record<string, unknown>)["om.folded"];
+		if (isMemoryDetails(nested)) return nested;
+	}
+	return undefined;
+}
+
+export function latestFullFoldBoundaryId(entries: Entry[]): string | undefined {
+	const indexes = entryIndexById(entries);
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		if (entry.type !== "compaction") continue;
+		const details = unwrapMemoryDetails(entry);
+		if (!details) continue;
+		if (!details.fullFold) continue;
+		if (!entry.firstKeptEntryId) continue;
+		if (!indexes.has(entry.firstKeptEntryId)) continue;
+		return entry.firstKeptEntryId;
+	}
+	return undefined;
+}
+
+export function buildCompactionProjection(
+	entries: Entry[],
+	firstKeptEntryId: string,
+	config: CompactionProjectionConfig,
+): CompactionProjection {
+	const fullFoldBoundaryId = latestFullFoldBoundaryId(entries);
+	const maintenanceBoundary = fullFoldBoundaryId ? entryBoundary(fullFoldBoundaryId) : noneBoundary();
+	const normalProjection = foldProjection(entries, {
+		observationsBoundary: entryBoundary(firstKeptEntryId),
+		reflectionsBoundary: maintenanceBoundary,
+		dropsBoundary: maintenanceBoundary,
+	});
+	const observationTokens = normalProjection.observations.reduce(
+		(total, observation) => total + observation.tokenCount,
+		0,
+	);
+	const fullFold = observationTokens >= config.observationsPoolMaxTokens;
+	const projection = fullFold
+		? fullProjection(entries, firstKeptEntryId)
+		: normalProjection;
+
+	const details: MemoryDetails = {
+		type: OM_FOLDED,
+		version: 1,
+		fullFold,
+		observations: projection.observations,
+		reflections: projection.reflections,
+	};
+
+	return {
+		fullFold,
+		observations: projection.observations,
+		reflections: projection.reflections,
+		details,
+	};
+}
+
+export function diffProjection(visible: Projection, full: Projection): ProjectionDiff {
+	const visibleObservationIds = new Set(visible.observations.map((observation) => observation.id));
+	const fullObservationIds = new Set(full.observations.map((observation) => observation.id));
+	const visibleReflectionIds = new Set(visible.reflections.map((reflection) => reflection.id));
+
+	return {
+		observationsOnlyInFull: full.observations.filter((observation) => !visibleObservationIds.has(observation.id)),
+		reflectionsOnlyInFull: full.reflections.filter((reflection) => !visibleReflectionIds.has(reflection.id)),
+		droppedOnlyInFull: visible.observations.filter((observation) => !fullObservationIds.has(observation.id)),
+	};
+}