pi-blackhole 0.2.0

package/src/om/ledger/recall.ts

@@ -0,0 +1,243 @@
+/**
+ * Memory recall — resolves source evidence for observations and reflections.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/recall.ts)
+ * Unmodified.
+ */
+import {
+	isObservationsDroppedEntry,
+	isObservationsRecordedEntry,
+	isReflectionsRecordedEntry,
+	type Entry,
+	type Observation,
+	type Reflection,
+} from "./types.js";
+
+const SOURCE_TYPES = new Set(["message", "custom_message", "branch_summary"]);
+
+export type { Entry, Observation, Reflection };
+
+type ObservationLedgerLocation = {
+	entryId: string;
+	entryIndex: number;
+	recordIndex: number;
+};
+
+type ReflectionLedgerLocation = {
+	entryId: string;
+	entryIndex: number;
+	recordIndex: number;
+};
+
+export type RecalledObservation = {
+	observation: Observation;
+	observationEntryId: string;
+	observationRecordIndex: number;
+	status: "active" | "dropped";
+	sourceEntryIds: string[];
+	sourceEntries: Entry[];
+	missingSourceEntryIds: string[];
+	nonSourceEntryIds: string[];
+};
+
+export type RecalledReflection = {
+	reflection: Reflection;
+	reflectionEntryId: string;
+	reflectionRecordIndex: number;
+};
+
+export type RecallResult =
+	| {
+			status: "not_found";
+			memoryId: string;
+			kind: undefined;
+			reflections: [];
+			observations: [];
+			sourceEntries: [];
+			missingSourceEntryIds: [];
+			nonSourceEntryIds: [];
+			missingSupportingObservationIds: [];
+			collision: false;
+			partial: false;
+	  }
+	| {
+			status: "found";
+			memoryId: string;
+			kind: "observation" | "reflection" | "mixed";
+			reflections: RecalledReflection[];
+			observations: RecalledObservation[];
+			sourceEntries: Entry[];
+			missingSourceEntryIds: string[];
+			nonSourceEntryIds: string[];
+			missingSupportingObservationIds: string[];
+			collision: boolean;
+			partial: boolean;
+	  };
+
+type IndexedObservation = ObservationLedgerLocation & { observation: Observation };
+type IndexedReflection = ReflectionLedgerLocation & { reflection: Reflection };
+
+function isSourceEntry(entry: Entry): boolean {
+	return SOURCE_TYPES.has(entry.type);
+}
+
+function uniqueById(entries: Entry[]): Entry[] {
+	const seen = new Set<string>();
+	const result: Entry[] = [];
+	for (const entry of entries) {
+		if (seen.has(entry.id)) continue;
+		seen.add(entry.id);
+		result.push(entry);
+	}
+	return result;
+}
+
+function uniqueStrings(values: string[]): string[] {
+	return Array.from(new Set(values));
+}
+
+export function indexLedger(entries: Entry[]): {
+	observations: IndexedObservation[];
+	reflections: IndexedReflection[];
+	droppedIds: Set<string>;
+} {
+	const observations: IndexedObservation[] = [];
+	const reflections: IndexedReflection[] = [];
+	const droppedIds = new Set<string>();
+
+	for (let entryIndex = 0; entryIndex < entries.length; entryIndex++) {
+		const entry = entries[entryIndex];
+		if (isObservationsRecordedEntry(entry)) {
+			entry.data.observations.forEach((observation, recordIndex) => {
+				observations.push({ observation, entryId: entry.id, entryIndex, recordIndex });
+			});
+			continue;
+		}
+		if (isReflectionsRecordedEntry(entry)) {
+			entry.data.reflections.forEach((reflection, recordIndex) => {
+				reflections.push({ reflection, entryId: entry.id, entryIndex, recordIndex });
+			});
+			continue;
+		}
+		if (isObservationsDroppedEntry(entry)) {
+			entry.data.observationIds.forEach((id) => droppedIds.add(id));
+		}
+	}
+
+	return { observations, reflections, droppedIds };
+}
+
+function resolveObservationSources(entries: Entry[], observation: Observation, location: ObservationLedgerLocation): RecalledObservation {
+	const sourceEntryIds = uniqueStrings(observation.sourceEntryIds);
+	const byId = new Map(entries.map((entry) => [entry.id, entry]));
+	const sourceEntries: Entry[] = [];
+	const missingSourceEntryIds: string[] = [];
+	const nonSourceEntryIds: string[] = [];
+
+	for (const sourceEntryId of sourceEntryIds) {
+		const sourceEntry = byId.get(sourceEntryId);
+		if (!sourceEntry) {
+			missingSourceEntryIds.push(sourceEntryId);
+			continue;
+		}
+		if (!isSourceEntry(sourceEntry)) {
+			nonSourceEntryIds.push(sourceEntryId);
+			continue;
+		}
+		sourceEntries.push(sourceEntry);
+	}
+
+	return {
+		observation,
+		observationEntryId: location.entryId,
+		observationRecordIndex: location.recordIndex,
+		status: "active",
+		sourceEntryIds,
+		sourceEntries,
+		missingSourceEntryIds,
+		nonSourceEntryIds,
+	};
+}
+
+function notFound(memoryId: string): RecallResult {
+	return {
+		status: "not_found",
+		memoryId,
+		kind: undefined,
+		reflections: [],
+		observations: [],
+		sourceEntries: [],
+		missingSourceEntryIds: [],
+		nonSourceEntryIds: [],
+		missingSupportingObservationIds: [],
+		collision: false,
+		partial: false,
+	};
+}
+
+export function recallMemorySources(entries: Entry[], memoryId: string): RecallResult {
+	const { observations: indexedObservations, reflections: indexedReflections, droppedIds } = indexLedger(entries);
+	const directObservationMatches = indexedObservations.filter(({ observation }) => observation.id === memoryId);
+	const reflectionMatches = indexedReflections.filter(({ reflection }) => reflection.id === memoryId);
+
+	if (directObservationMatches.length === 0 && reflectionMatches.length === 0) return notFound(memoryId);
+
+	const observationsById = new Map<string, IndexedObservation>();
+	for (const indexed of indexedObservations) {
+		if (!observationsById.has(indexed.observation.id)) observationsById.set(indexed.observation.id, indexed);
+	}
+
+	const recalledByKey = new Map<string, RecalledObservation>();
+	const missingSupportingObservationIds: string[] = [];
+
+	function addObservation(indexed: IndexedObservation): void {
+		const key = `${indexed.entryId}:${indexed.recordIndex}`;
+		if (recalledByKey.has(key)) return;
+		const recalled = resolveObservationSources(entries, indexed.observation, indexed);
+		recalled.status = droppedIds.has(indexed.observation.id) ? "dropped" : "active";
+		recalledByKey.set(key, recalled);
+	}
+
+	for (const match of directObservationMatches) addObservation(match);
+
+	for (const { reflection } of reflectionMatches) {
+		for (const observationId of uniqueStrings(reflection.supportingObservationIds)) {
+			const indexed = observationsById.get(observationId);
+			if (!indexed) {
+				missingSupportingObservationIds.push(observationId);
+				continue;
+			}
+			addObservation(indexed);
+		}
+	}
+
+	const recalledObservations = Array.from(recalledByKey.values());
+	const recalledReflections: RecalledReflection[] = reflectionMatches.map(({ reflection, entryId, recordIndex }) => ({
+		reflection,
+		reflectionEntryId: entryId,
+		reflectionRecordIndex: recordIndex,
+	}));
+	const sourceEntries = uniqueById(recalledObservations.flatMap((match) => match.sourceEntries));
+	const missingSourceEntryIds = uniqueStrings(recalledObservations.flatMap((match) => match.missingSourceEntryIds));
+	const nonSourceEntryIds = uniqueStrings(recalledObservations.flatMap((match) => match.nonSourceEntryIds));
+	const uniqueMissingSupportingObservationIds = uniqueStrings(missingSupportingObservationIds);
+	const matchCount = directObservationMatches.length + reflectionMatches.length;
+
+	return {
+		status: "found",
+		memoryId,
+		kind: directObservationMatches.length > 0 && reflectionMatches.length > 0
+			? "mixed"
+			: reflectionMatches.length > 0
+				? "reflection"
+				: "observation",
+		reflections: recalledReflections,
+		observations: recalledObservations,
+		sourceEntries,
+		missingSourceEntryIds,
+		nonSourceEntryIds,
+		missingSupportingObservationIds: uniqueMissingSupportingObservationIds,
+		collision: matchCount > 1,
+		partial: missingSourceEntryIds.length > 0 || nonSourceEntryIds.length > 0 || uniqueMissingSupportingObservationIds.length > 0,
+	};
+}

package/src/om/ledger/render-summary.ts

@@ -0,0 +1,44 @@
+/**
+ * Summary rendering — formats observations/reflections for compaction output.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/render-summary.ts)
+ * Unmodified.
+ */
+import type { Observation, Reflection } from "./types.js";
+
+const RECALL_NOTE =
+	"Use `recall` to search for prior work, decisions, and context from before this summary. " +
+	"Do not redo work already completed.";
+
+const CONTEXT_USAGE_INSTRUCTIONS = `These are condensed memories from earlier in this session.
+
+- Reflections: stable, long-lived facts about the user, project, decisions, and constraints. New reflection lines may include ids in brackets.
+- Observations: timestamped events from the conversation history, in chronological order. Observation lines include ids in brackets.
+
+Treat these as past records. When entries conflict, the most recent observation reflects the latest known state. Work that prior observations describe as completed should not be redone unless the user explicitly asks to revisit it.
+
+When exact source context is needed for precision or traceability, use the \`recall\` tool with the relevant observation or reflection id. This is especially useful when a reflection materially affects a decision or is too compressed to continue confidently. Do not use \`recall\` as broad search or inject raw source unless it is needed.`;
+
+export const OM_FOOTER = `----\n${RECALL_NOTE}\n\n${CONTEXT_USAGE_INSTRUCTIONS}\n----`;
+
+export function observationToSummaryLine(observation: Observation): string {
+	return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] ${observation.content}`;
+}
+
+export function reflectionToSummaryLine(reflection: Reflection): string {
+	return `[${reflection.id}] ${reflection.content}`;
+}
+
+export function renderSummary(reflections: Reflection[], observations: Observation[]): string {
+	if (reflections.length === 0 && observations.length === 0) return "";
+
+	const parts: string[] = [];
+	if (reflections.length > 0) {
+		parts.push(`## Reflections\n${reflections.map(reflectionToSummaryLine).join("\n")}`);
+	}
+	if (observations.length > 0) {
+		parts.push(`## Observations\n${observations.map(observationToSummaryLine).join("\n")}`);
+	}
+	parts.push(OM_FOOTER);
+	return parts.join("\n\n");
+}

package/src/om/ledger/types.ts

@@ -0,0 +1,206 @@
+/**
+ * Observational memory ledger types — validated Entry, Observation, Reflection.
+ *
+ * Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/types.ts)
+ * Unmodified.
+ */
+export const OM_OBSERVATIONS_RECORDED = "om.observations.recorded";
+export const OM_REFLECTIONS_RECORDED = "om.reflections.recorded";
+export const OM_OBSERVATIONS_DROPPED = "om.observations.dropped";
+export const OM_FOLDED = "om.folded";
+
+export const RELEVANCE_VALUES = ["low", "medium", "high", "critical"] as const;
+export type Relevance = (typeof RELEVANCE_VALUES)[number];
+
+export const MEMORY_ID_PATTERN = /^[a-f0-9]{12}$/;
+
+export type Entry = {
+	type: string;
+	id: string;
+	timestamp?: string;
+	message?: unknown;
+	content?: unknown;
+	customType?: string;
+	summary?: unknown;
+	fromId?: string;
+	data?: unknown;
+	details?: unknown;
+	firstKeptEntryId?: string;
+};
+
+export type Observation = {
+	id: string;
+	content: string;
+	timestamp: string;
+	relevance: Relevance;
+	sourceEntryIds: string[];
+	tokenCount: number;
+};
+
+export type Reflection = {
+	id: string;
+	content: string;
+	supportingObservationIds: string[];
+	tokenCount: number;
+};
+
+export type ObservationsRecordedEntryData = {
+	observations: Observation[];
+	coversUpToId: string;
+};
+
+export type ReflectionsRecordedEntryData = {
+	reflections: Reflection[];
+	coversUpToId: string;
+};
+
+export type ObservationsDroppedEntryData = {
+	observationIds: string[];
+	coversUpToId: string;
+};
+
+export type MemoryDetails = {
+	type: typeof OM_FOLDED;
+	version: 1;
+	fullFold: boolean;
+	observations: Observation[];
+	reflections: Reflection[];
+};
+
+export type V3MemoryCustomType =
+	| typeof OM_OBSERVATIONS_RECORDED
+	| typeof OM_REFLECTIONS_RECORDED
+	| typeof OM_OBSERVATIONS_DROPPED;
+
+export function isRelevance(value: unknown): value is Relevance {
+	return typeof value === "string" && (RELEVANCE_VALUES as readonly string[]).includes(value);
+}
+
+export function isNonEmptyString(value: unknown): value is string {
+	return typeof value === "string" && value.length > 0;
+}
+
+export function isNonEmptyStringArray(value: unknown): value is string[] {
+	return Array.isArray(value) && value.length > 0 && value.every(isNonEmptyString);
+}
+
+export function isMemoryId(value: unknown): value is string {
+	return typeof value === "string" && MEMORY_ID_PATTERN.test(value);
+}
+
+function isTokenCount(value: unknown): value is number {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+	return !!value && typeof value === "object";
+}
+
+export function isObservation(value: unknown): value is Observation {
+	if (!isPlainRecord(value)) return false;
+	return (
+		isMemoryId(value.id) &&
+		isNonEmptyString(value.content) &&
+		isNonEmptyString(value.timestamp) &&
+		isRelevance(value.relevance) &&
+		isNonEmptyStringArray(value.sourceEntryIds) &&
+		isTokenCount(value.tokenCount)
+	);
+}
+
+export function isReflection(value: unknown): value is Reflection {
+	if (!isPlainRecord(value)) return false;
+	return (
+		isMemoryId(value.id) &&
+		isNonEmptyString(value.content) &&
+		!/\r|\n/.test(value.content) &&
+		isNonEmptyStringArray(value.supportingObservationIds) &&
+		isTokenCount(value.tokenCount)
+	);
+}
+
+export function isObservationsRecordedData(value: unknown): value is ObservationsRecordedEntryData {
+	if (!isPlainRecord(value)) return false;
+	return (
+		Array.isArray(value.observations) &&
+		value.observations.length > 0 &&
+		value.observations.every(isObservation) &&
+		isNonEmptyString(value.coversUpToId)
+	);
+}
+
+export function isReflectionsRecordedData(value: unknown): value is ReflectionsRecordedEntryData {
+	if (!isPlainRecord(value)) return false;
+	return (
+		Array.isArray(value.reflections) &&
+		value.reflections.length > 0 &&
+		value.reflections.every(isReflection) &&
+		isNonEmptyString(value.coversUpToId)
+	);
+}
+
+export function isObservationsDroppedData(value: unknown): value is ObservationsDroppedEntryData {
+	if (!isPlainRecord(value)) return false;
+	return isNonEmptyStringArray(value.observationIds) && isNonEmptyString(value.coversUpToId);
+}
+
+export function isMemoryDetails(value: unknown): value is MemoryDetails {
+	if (!isPlainRecord(value)) return false;
+	return (
+		value.type === OM_FOLDED &&
+		value.version === 1 &&
+		typeof value.fullFold === "boolean" &&
+		Array.isArray(value.observations) &&
+		value.observations.every(isObservation) &&
+		Array.isArray(value.reflections) &&
+		value.reflections.every(isReflection)
+	);
+}
+
+export function isObservationsRecordedEntry(entry: Entry): entry is Entry & {
+	type: "custom";
+	customType: typeof OM_OBSERVATIONS_RECORDED;
+	data: ObservationsRecordedEntryData;
+} {
+	return entry.type === "custom" && entry.customType === OM_OBSERVATIONS_RECORDED && isObservationsRecordedData(entry.data);
+}
+
+export function isReflectionsRecordedEntry(entry: Entry): entry is Entry & {
+	type: "custom";
+	customType: typeof OM_REFLECTIONS_RECORDED;
+	data: ReflectionsRecordedEntryData;
+} {
+	return entry.type === "custom" && entry.customType === OM_REFLECTIONS_RECORDED && isReflectionsRecordedData(entry.data);
+}
+
+export function isObservationsDroppedEntry(entry: Entry): entry is Entry & {
+	type: "custom";
+	customType: typeof OM_OBSERVATIONS_DROPPED;
+	data: ObservationsDroppedEntryData;
+} {
+	return entry.type === "custom" && entry.customType === OM_OBSERVATIONS_DROPPED && isObservationsDroppedData(entry.data);
+}
+
+export function buildObservationsRecordedData(
+	observations: Observation[],
+	coversUpToId: string,
+): ObservationsRecordedEntryData | undefined {
+	if (observations.length === 0 || !isNonEmptyString(coversUpToId)) return undefined;
+	return { observations, coversUpToId };
+}
+
+export function buildReflectionsRecordedData(
+	reflections: Reflection[],
+	coversUpToId: string,
+): ReflectionsRecordedEntryData | undefined {
+	if (reflections.length === 0 || !isNonEmptyString(coversUpToId)) return undefined;
+	return { reflections, coversUpToId };
+}
+
+export function buildObservationsDroppedData(
+	observationIds: string[],
+	coversUpToId: string,
+): ObservationsDroppedEntryData | undefined {
+	if (observationIds.length === 0 || !isNonEmptyString(coversUpToId)) return undefined;
+	return { observationIds, coversUpToId };
+}

package/src/om/model-budget.ts

@@ -0,0 +1,9 @@
+import type { Model } from "@earendil-works/pi-ai";
+
+export const AGENT_LOOP_MAX_TOKENS = 32_000;
+
+export function boundedMaxTokens(model: Model<any>, requested: number = AGENT_LOOP_MAX_TOKENS): number {
+	return typeof model.maxTokens === "number" && model.maxTokens > 0
+		? Math.min(model.maxTokens, requested)
+		: requested;
+}