pi-observational-memory 2.4.3 → 3.0.0

package/src/session-ledger/progress.ts

@@ -0,0 +1,129 @@
+import { estimateEntryTokens } from "../tokens.js";
+import {
+	OM_OBSERVATIONS_DROPPED,
+	OM_OBSERVATIONS_RECORDED,
+	OM_REFLECTIONS_RECORDED,
+	type Entry,
+	type V3MemoryCustomType,
+} from "./types.js";
+
+const SOURCE_ENTRY_TYPES = new Set(["message", "custom_message", "branch_summary"]);
+
+export function isSourceEntry(entry: Entry): boolean {
+	return SOURCE_ENTRY_TYPES.has(entry.type);
+}
+
+export function entryIndexById(entries: Entry[]): Map<string, number> {
+	const idToIndex = new Map<string, number>();
+	for (let i = 0; i < entries.length; i++) idToIndex.set(entries[i].id, i);
+	return idToIndex;
+}
+
+export function entryIndexForId(entries: Entry[], entryId: string | undefined): number {
+	if (!entryId) return -1;
+	const idx = entryIndexById(entries).get(entryId);
+	return idx ?? -1;
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}
+
+function isNonEmptyArray(value: unknown): value is unknown[] {
+	return Array.isArray(value) && value.length > 0;
+}
+
+function isValidCoverageEntry(entry: Entry, customType: V3MemoryCustomType): entry is Entry & { data: { coversUpToId: string } } {
+	if (entry.type !== "custom" || entry.customType !== customType) return false;
+	if (!isObject(entry.data) || typeof entry.data.coversUpToId !== "string") return false;
+
+	if (customType === OM_OBSERVATIONS_RECORDED) return isNonEmptyArray(entry.data.observations);
+	if (customType === OM_REFLECTIONS_RECORDED) return isNonEmptyArray(entry.data.reflections);
+	return isNonEmptyArray(entry.data.observationIds);
+}
+
+export function latestCoverageIndex(entries: Entry[], customType: V3MemoryCustomType): number {
+	const idToIndex = entryIndexById(entries);
+	let latest = -1;
+
+	for (const entry of entries) {
+		if (!isValidCoverageEntry(entry, customType)) continue;
+		const coveredIndex = idToIndex.get(entry.data.coversUpToId);
+		if (coveredIndex === undefined) continue;
+		if (coveredIndex > latest) latest = coveredIndex;
+	}
+
+	return latest;
+}
+
+export function latestCoverageMarkerId(entries: Entry[], customType: V3MemoryCustomType): string | undefined {
+	const idToIndex = entryIndexById(entries);
+	let latestIndex = -1;
+	let latestMarkerId: string | undefined;
+
+	for (const entry of entries) {
+		if (!isValidCoverageEntry(entry, customType)) continue;
+		const coveredIndex = idToIndex.get(entry.data.coversUpToId);
+		if (coveredIndex === undefined) continue;
+		if (coveredIndex > latestIndex) {
+			latestIndex = coveredIndex;
+			latestMarkerId = entry.data.coversUpToId;
+		}
+	}
+
+	return latestMarkerId;
+}
+
+export function earlierCoverageMarkerId(entries: Entry[], firstId: string | undefined, secondId: string | undefined): string | undefined {
+	if (!firstId) return secondId;
+	if (!secondId) return firstId;
+
+	const idToIndex = entryIndexById(entries);
+	const firstIndex = idToIndex.get(firstId);
+	const secondIndex = idToIndex.get(secondId);
+	if (firstIndex === undefined) return secondIndex === undefined ? undefined : secondId;
+	if (secondIndex === undefined) return firstId;
+	return firstIndex <= secondIndex ? firstId : secondId;
+}
+
+export function rawTokensAfterIndex(entries: Entry[], index: number): number {
+	let total = 0;
+	for (let i = Math.max(0, index + 1); i < entries.length; i++) {
+		if (isSourceEntry(entries[i])) total += estimateEntryTokens(entries[i]);
+	}
+	return total;
+}
+
+export function rawTokensSinceCoverage(entries: Entry[], customType: V3MemoryCustomType): number {
+	return rawTokensAfterIndex(entries, latestCoverageIndex(entries, customType));
+}
+
+export function rawTokensSinceObservationCoverage(entries: Entry[]): number {
+	return rawTokensSinceCoverage(entries, OM_OBSERVATIONS_RECORDED);
+}
+
+export function rawTokensSinceReflectionCoverage(entries: Entry[]): number {
+	return rawTokensSinceCoverage(entries, OM_REFLECTIONS_RECORDED);
+}
+
+export function rawTokensSinceDropCoverage(entries: Entry[]): number {
+	return rawTokensSinceCoverage(entries, OM_OBSERVATIONS_DROPPED);
+}
+
+export function findLastCompactionIndex(entries: Entry[]): number {
+	for (let i = entries.length - 1; i >= 0; i--) {
+		if (entries[i].type === "compaction") return i;
+	}
+	return -1;
+}
+
+export function rawTokensSinceLastCompaction(entries: Entry[]): number {
+	const compactionIndex = findLastCompactionIndex(entries);
+	if (compactionIndex === -1) return rawTokensAfterIndex(entries, -1);
+
+	const firstKeptEntryId = entries[compactionIndex].firstKeptEntryId;
+	const firstKeptIndex = entryIndexForId(entries, firstKeptEntryId);
+
+	if (firstKeptIndex === -1) return rawTokensAfterIndex(entries, compactionIndex);
+	return rawTokensAfterIndex(entries, firstKeptIndex - 1);
+}

package/src/session-ledger/projection.ts

@@ -0,0 +1,220 @@
+import {
+	OM_FOLDED,
+	isMemoryDetails,
+	isObservationsDroppedEntry,
+	isObservationsRecordedEntry,
+	isReflectionsRecordedEntry,
+	type Entry,
+	type MemoryDetails,
+	type Observation,
+	type Reflection,
+} from "./types.js";
+
+export type Projection = {
+	observations: Observation[];
+	reflections: Reflection[];
+};
+
+export type ProjectionDiff = {
+	observationsOnlyInFull: Observation[];
+	reflectionsOnlyInFull: Reflection[];
+	droppedOnlyInFull: Observation[];
+};
+
+export type CompactionProjectionConfig = {
+	observationsPoolMaxTokens: number;
+};
+
+export type CompactionProjection = Projection & {
+	fullFold: boolean;
+	details: MemoryDetails;
+};
+
+type ProjectionBoundary =
+	| { kind: "entry"; entryId: string }
+	| { kind: "tip" }
+	| { kind: "none" };
+
+type ProjectionFoldOptions = {
+	observationsBoundary: ProjectionBoundary;
+	reflectionsBoundary: ProjectionBoundary;
+	dropsBoundary: ProjectionBoundary;
+};
+
+function entryIndexById(entries: Entry[]): Map<string, number> {
+	const indexes = new Map<string, number>();
+	for (let i = 0; i < entries.length; i++) indexes.set(entries[i].id, i);
+	return indexes;
+}
+
+function entryBoundary(entryId: string): ProjectionBoundary {
+	return { kind: "entry", entryId };
+}
+
+function tipBoundary(): ProjectionBoundary {
+	return { kind: "tip" };
+}
+
+function noneBoundary(): ProjectionBoundary {
+	return { kind: "none" };
+}
+
+function boundaryIndex(entries: Entry[], indexes: Map<string, number>, boundary: ProjectionBoundary): number {
+	if (boundary.kind === "tip") return entries.length - 1;
+	if (boundary.kind === "none") return -1;
+	return indexes.get(boundary.entryId) ?? -1;
+}
+
+function coverageIndex(entry: Entry & { data: { coversUpToId: string } }, indexes: Map<string, number>): number {
+	return indexes.get(entry.data.coversUpToId) ?? -1;
+}
+
+function isAtOrBefore(index: number, boundaryIndex: number): boolean {
+	return index >= 0 && boundaryIndex >= 0 && index <= boundaryIndex;
+}
+
+function isCoveredAtOrBefore(
+	entry: Entry & { data: { coversUpToId: string } },
+	indexes: Map<string, number>,
+	boundaryIndex: number,
+): boolean {
+	return isAtOrBefore(coverageIndex(entry, indexes), boundaryIndex);
+}
+
+function foldProjection(entries: Entry[], options: ProjectionFoldOptions): Projection {
+	const indexes = entryIndexById(entries);
+	const observationsBoundary = boundaryIndex(entries, indexes, options.observationsBoundary);
+	const reflectionsBoundary = boundaryIndex(entries, indexes, options.reflectionsBoundary);
+	const dropsBoundary = boundaryIndex(entries, indexes, options.dropsBoundary);
+	const observations: Observation[] = [];
+	const reflections: Reflection[] = [];
+	const observationsById = new Set<string>();
+	const reflectionsById = new Set<string>();
+	const droppedObservationIds = new Set<string>();
+
+	for (const entry of entries) {
+		if (isObservationsRecordedEntry(entry) && isCoveredAtOrBefore(entry, indexes, observationsBoundary)) {
+			for (const observation of entry.data.observations) {
+				if (observationsById.has(observation.id)) continue;
+				observationsById.add(observation.id);
+				observations.push(observation);
+			}
+			continue;
+		}
+
+		if (isReflectionsRecordedEntry(entry) && isCoveredAtOrBefore(entry, indexes, reflectionsBoundary)) {
+			for (const reflection of entry.data.reflections) {
+				if (reflectionsById.has(reflection.id)) continue;
+				reflectionsById.add(reflection.id);
+				reflections.push(reflection);
+			}
+			continue;
+		}
+
+		if (isObservationsDroppedEntry(entry) && isCoveredAtOrBefore(entry, indexes, dropsBoundary)) {
+			for (const observationId of entry.data.observationIds) droppedObservationIds.add(observationId);
+		}
+	}
+
+	return {
+		observations: observations.filter((observation) => !droppedObservationIds.has(observation.id)),
+		reflections,
+	};
+}
+
+function projectionFromMemoryDetails(details: MemoryDetails): Projection {
+	return {
+		observations: [...details.observations],
+		reflections: [...details.reflections],
+	};
+}
+
+function latestV3CompactionDetails(entries: Entry[]): MemoryDetails | undefined {
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		if (entry.type !== "compaction") continue;
+		if (isMemoryDetails(entry.details)) return entry.details;
+	}
+	return undefined;
+}
+
+export function fullProjection(entries: Entry[], upToEntryId?: string): Projection {
+	const boundary = upToEntryId ? entryBoundary(upToEntryId) : tipBoundary();
+	return foldProjection(entries, {
+		observationsBoundary: boundary,
+		reflectionsBoundary: boundary,
+		dropsBoundary: boundary,
+	});
+}
+
+export function visibleProjection(entries: Entry[], upToEntryId?: string): Projection {
+	if (!upToEntryId) {
+		const details = latestV3CompactionDetails(entries);
+		return details ? projectionFromMemoryDetails(details) : { observations: [], reflections: [] };
+	}
+
+	return buildCompactionProjection(entries, upToEntryId, { observationsPoolMaxTokens: Number.POSITIVE_INFINITY });
+}
+
+export function latestFullFoldBoundaryId(entries: Entry[]): string | undefined {
+	const indexes = entryIndexById(entries);
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		if (entry.type !== "compaction") continue;
+		if (!isMemoryDetails(entry.details)) continue;
+		if (!entry.details.fullFold) continue;
+		if (!entry.firstKeptEntryId) continue;
+		if (!indexes.has(entry.firstKeptEntryId)) continue;
+		return entry.firstKeptEntryId;
+	}
+	return undefined;
+}
+
+export function buildCompactionProjection(
+	entries: Entry[],
+	firstKeptEntryId: string,
+	config: CompactionProjectionConfig,
+): CompactionProjection {
+	const fullFoldBoundaryId = latestFullFoldBoundaryId(entries);
+	const maintenanceBoundary = fullFoldBoundaryId ? entryBoundary(fullFoldBoundaryId) : noneBoundary();
+	const normalProjection = foldProjection(entries, {
+		observationsBoundary: entryBoundary(firstKeptEntryId),
+		reflectionsBoundary: maintenanceBoundary,
+		dropsBoundary: maintenanceBoundary,
+	});
+	const observationTokens = normalProjection.observations.reduce(
+		(total, observation) => total + observation.tokenCount,
+		0,
+	);
+	const fullFold = observationTokens >= config.observationsPoolMaxTokens;
+	const projection = fullFold
+		? fullProjection(entries, firstKeptEntryId)
+		: normalProjection;
+
+	const details: MemoryDetails = {
+		type: OM_FOLDED,
+		version: 1,
+		fullFold,
+		observations: projection.observations,
+		reflections: projection.reflections,
+	};
+
+	return {
+		fullFold,
+		observations: projection.observations,
+		reflections: projection.reflections,
+		details,
+	};
+}
+
+export function diffProjection(visible: Projection, full: Projection): ProjectionDiff {
+	const visibleObservationIds = new Set(visible.observations.map((observation) => observation.id));
+	const fullObservationIds = new Set(full.observations.map((observation) => observation.id));
+	const visibleReflectionIds = new Set(visible.reflections.map((reflection) => reflection.id));
+
+	return {
+		observationsOnlyInFull: full.observations.filter((observation) => !visibleObservationIds.has(observation.id)),
+		reflectionsOnlyInFull: full.reflections.filter((reflection) => !visibleReflectionIds.has(reflection.id)),
+		droppedOnlyInFull: visible.observations.filter((observation) => !fullObservationIds.has(observation.id)),
+	};
+}

package/src/session-ledger/recall.ts

@@ -0,0 +1,237 @@
+import {
+	isObservationsDroppedEntry,
+	isObservationsRecordedEntry,
+	isReflectionsRecordedEntry,
+	type Entry,
+	type Observation,
+	type Reflection,
+} from "./types.js";
+
+const SOURCE_TYPES = new Set(["message", "custom_message", "branch_summary"]);
+
+export type { Entry, Observation, Reflection };
+
+type ObservationLedgerLocation = {
+	entryId: string;
+	entryIndex: number;
+	recordIndex: number;
+};
+
+type ReflectionLedgerLocation = {
+	entryId: string;
+	entryIndex: number;
+	recordIndex: number;
+};
+
+export type RecalledObservation = {
+	observation: Observation;
+	observationEntryId: string;
+	observationRecordIndex: number;
+	status: "active" | "dropped";
+	sourceEntryIds: string[];
+	sourceEntries: Entry[];
+	missingSourceEntryIds: string[];
+	nonSourceEntryIds: string[];
+};
+
+export type RecalledReflection = {
+	reflection: Reflection;
+	reflectionEntryId: string;
+	reflectionRecordIndex: number;
+};
+
+export type RecallResult =
+	| {
+			status: "not_found";
+			memoryId: string;
+			kind: undefined;
+			reflections: [];
+			observations: [];
+			sourceEntries: [];
+			missingSourceEntryIds: [];
+			nonSourceEntryIds: [];
+			missingSupportingObservationIds: [];
+			collision: false;
+			partial: false;
+	  }
+	| {
+			status: "found";
+			memoryId: string;
+			kind: "observation" | "reflection" | "mixed";
+			reflections: RecalledReflection[];
+			observations: RecalledObservation[];
+			sourceEntries: Entry[];
+			missingSourceEntryIds: string[];
+			nonSourceEntryIds: string[];
+			missingSupportingObservationIds: string[];
+			collision: boolean;
+			partial: boolean;
+	  };
+
+type IndexedObservation = ObservationLedgerLocation & { observation: Observation };
+type IndexedReflection = ReflectionLedgerLocation & { reflection: Reflection };
+
+function isSourceEntry(entry: Entry): boolean {
+	return SOURCE_TYPES.has(entry.type);
+}
+
+function uniqueById(entries: Entry[]): Entry[] {
+	const seen = new Set<string>();
+	const result: Entry[] = [];
+	for (const entry of entries) {
+		if (seen.has(entry.id)) continue;
+		seen.add(entry.id);
+		result.push(entry);
+	}
+	return result;
+}
+
+function uniqueStrings(values: string[]): string[] {
+	return Array.from(new Set(values));
+}
+
+function indexLedger(entries: Entry[]): {
+	observations: IndexedObservation[];
+	reflections: IndexedReflection[];
+	droppedIds: Set<string>;
+} {
+	const observations: IndexedObservation[] = [];
+	const reflections: IndexedReflection[] = [];
+	const droppedIds = new Set<string>();
+
+	for (let entryIndex = 0; entryIndex < entries.length; entryIndex++) {
+		const entry = entries[entryIndex];
+		if (isObservationsRecordedEntry(entry)) {
+			entry.data.observations.forEach((observation, recordIndex) => {
+				observations.push({ observation, entryId: entry.id, entryIndex, recordIndex });
+			});
+			continue;
+		}
+		if (isReflectionsRecordedEntry(entry)) {
+			entry.data.reflections.forEach((reflection, recordIndex) => {
+				reflections.push({ reflection, entryId: entry.id, entryIndex, recordIndex });
+			});
+			continue;
+		}
+		if (isObservationsDroppedEntry(entry)) {
+			entry.data.observationIds.forEach((id) => droppedIds.add(id));
+		}
+	}
+
+	return { observations, reflections, droppedIds };
+}
+
+function resolveObservationSources(entries: Entry[], observation: Observation, location: ObservationLedgerLocation): RecalledObservation {
+	const sourceEntryIds = uniqueStrings(observation.sourceEntryIds);
+	const byId = new Map(entries.map((entry) => [entry.id, entry]));
+	const sourceEntries: Entry[] = [];
+	const missingSourceEntryIds: string[] = [];
+	const nonSourceEntryIds: string[] = [];
+
+	for (const sourceEntryId of sourceEntryIds) {
+		const sourceEntry = byId.get(sourceEntryId);
+		if (!sourceEntry) {
+			missingSourceEntryIds.push(sourceEntryId);
+			continue;
+		}
+		if (!isSourceEntry(sourceEntry)) {
+			nonSourceEntryIds.push(sourceEntryId);
+			continue;
+		}
+		sourceEntries.push(sourceEntry);
+	}
+
+	return {
+		observation,
+		observationEntryId: location.entryId,
+		observationRecordIndex: location.recordIndex,
+		status: "active",
+		sourceEntryIds,
+		sourceEntries,
+		missingSourceEntryIds,
+		nonSourceEntryIds,
+	};
+}
+
+function notFound(memoryId: string): RecallResult {
+	return {
+		status: "not_found",
+		memoryId,
+		kind: undefined,
+		reflections: [],
+		observations: [],
+		sourceEntries: [],
+		missingSourceEntryIds: [],
+		nonSourceEntryIds: [],
+		missingSupportingObservationIds: [],
+		collision: false,
+		partial: false,
+	};
+}
+
+export function recallMemorySources(entries: Entry[], memoryId: string): RecallResult {
+	const { observations: indexedObservations, reflections: indexedReflections, droppedIds } = indexLedger(entries);
+	const directObservationMatches = indexedObservations.filter(({ observation }) => observation.id === memoryId);
+	const reflectionMatches = indexedReflections.filter(({ reflection }) => reflection.id === memoryId);
+
+	if (directObservationMatches.length === 0 && reflectionMatches.length === 0) return notFound(memoryId);
+
+	const observationsById = new Map<string, IndexedObservation>();
+	for (const indexed of indexedObservations) {
+		if (!observationsById.has(indexed.observation.id)) observationsById.set(indexed.observation.id, indexed);
+	}
+
+	const recalledByKey = new Map<string, RecalledObservation>();
+	const missingSupportingObservationIds: string[] = [];
+
+	function addObservation(indexed: IndexedObservation): void {
+		const key = `${indexed.entryId}:${indexed.recordIndex}`;
+		if (recalledByKey.has(key)) return;
+		const recalled = resolveObservationSources(entries, indexed.observation, indexed);
+		recalled.status = droppedIds.has(indexed.observation.id) ? "dropped" : "active";
+		recalledByKey.set(key, recalled);
+	}
+
+	for (const match of directObservationMatches) addObservation(match);
+
+	for (const { reflection } of reflectionMatches) {
+		for (const observationId of uniqueStrings(reflection.supportingObservationIds)) {
+			const indexed = observationsById.get(observationId);
+			if (!indexed) {
+				missingSupportingObservationIds.push(observationId);
+				continue;
+			}
+			addObservation(indexed);
+		}
+	}
+
+	const recalledObservations = Array.from(recalledByKey.values());
+	const recalledReflections: RecalledReflection[] = reflectionMatches.map(({ reflection, entryId, recordIndex }) => ({
+		reflection,
+		reflectionEntryId: entryId,
+		reflectionRecordIndex: recordIndex,
+	}));
+	const sourceEntries = uniqueById(recalledObservations.flatMap((match) => match.sourceEntries));
+	const missingSourceEntryIds = uniqueStrings(recalledObservations.flatMap((match) => match.missingSourceEntryIds));
+	const nonSourceEntryIds = uniqueStrings(recalledObservations.flatMap((match) => match.nonSourceEntryIds));
+	const uniqueMissingSupportingObservationIds = uniqueStrings(missingSupportingObservationIds);
+	const matchCount = directObservationMatches.length + reflectionMatches.length;
+
+	return {
+		status: "found",
+		memoryId,
+		kind: directObservationMatches.length > 0 && reflectionMatches.length > 0
+			? "mixed"
+			: reflectionMatches.length > 0
+				? "reflection"
+				: "observation",
+		reflections: recalledReflections,
+		observations: recalledObservations,
+		sourceEntries,
+		missingSourceEntryIds,
+		nonSourceEntryIds,
+		missingSupportingObservationIds: uniqueMissingSupportingObservationIds,
+		collision: matchCount > 1,
+		partial: missingSourceEntryIds.length > 0 || nonSourceEntryIds.length > 0 || uniqueMissingSupportingObservationIds.length > 0,
+	};
+}

package/src/session-ledger/render-summary.ts

@@ -0,0 +1,31 @@
+import type { Observation, Reflection } from "./types.js";
+
+const CONTEXT_USAGE_INSTRUCTIONS = `These are condensed memories from earlier in this session.
+
+- Reflections: stable, long-lived facts about the user, project, decisions, and constraints. New reflection lines may include ids in brackets.
+- Observations: timestamped events from the conversation history, in chronological order. Observation lines include ids in brackets.
+
+Treat these as past records. When entries conflict, the most recent observation reflects the latest known state. Work that prior observations describe as completed should not be redone unless the user explicitly asks to revisit it.
+
+When exact source context is needed for precision or traceability, use the recall tool with the relevant observation or reflection id. This is especially useful when a reflection materially affects a decision or is too compressed to continue confidently. Do not use recall as broad search or inject raw source unless it is needed.`;
+
+export function observationToSummaryLine(observation: Observation): string {
+	return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] ${observation.content}`;
+}
+
+export function reflectionToSummaryLine(reflection: Reflection): string {
+	return `[${reflection.id}] ${reflection.content}`;
+}
+
+export function renderSummary(reflections: Reflection[], observations: Observation[]): string {
+	if (reflections.length === 0 && observations.length === 0) return "";
+
+	const parts: string[] = [CONTEXT_USAGE_INSTRUCTIONS];
+	if (reflections.length > 0) {
+		parts.push(`## Reflections\n${reflections.map(reflectionToSummaryLine).join("\n")}`);
+	}
+	if (observations.length > 0) {
+		parts.push(`## Observations\n${observations.map(observationToSummaryLine).join("\n")}`);
+	}
+	return parts.join("\n\n");
+}