pi-observational-memory 2.4.3 → 3.0.1

package/src/agents/dropper/agent.ts

@@ -0,0 +1,281 @@
+import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
+import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
+import { Type } from "@earendil-works/pi-ai";
+import type { Static } from "typebox";
+import { debugLog } from "../../debug-log.js";
+import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
+import { reflectionToSummaryLine, type Observation, type Reflection } from "../../session-ledger/index.js";
+import { DROPPER_SYSTEM } from "./prompts.js";
+import {
+	REFLECTION_COVERAGE_DROP_RANK,
+	coverageTierForObservation,
+	reflectionCoverageMap,
+	summarizeCoverageByRelevance,
+	summarizeCoverageByRelevanceForIds,
+	observationToDropperLine,
+} from "./coverage.js";
+import { observationPoolMetrics } from "./pool.js";
+export {
+	maxDropCountForPool,
+	observationPoolFullness,
+	observationPoolMetrics,
+} from "./pool.js";
+export type { ObservationPoolMetrics } from "./pool.js";
+export {
+	REFLECTION_COVERAGE_TIERS,
+	coverageTierForObservation,
+	emptyCoverageSummaryByRelevance,
+	observationToDropperLine,
+	reflectionCoverageMap,
+	reflectionCoverageTierForCount,
+	reflectionSupportCounts,
+	summarizeCoverageByRelevance,
+	summarizeCoverageByRelevanceForIds,
+	summarizeCoverageTransitionsByRelevance,
+} from "./coverage.js";
+export type { CoverageSummaryByRelevance, CoverageTransitionSummaryByRelevance, ReflectionCoverageTier } from "./coverage.js";
+
+interface RunDropperArgs {
+	model: Model<any>;
+	apiKey: string;
+	headers?: Record<string, string>;
+	reflections: Reflection[];
+	observations: Observation[];
+	targetTokens: number;
+	signal?: AbortSignal;
+	agentLoop?: typeof agentLoop;
+	maxTurns?: number;
+	thinkingLevel?: ModelThinkingLevel;
+}
+
+const RELEVANCE_DROP_RANK: Record<Observation["relevance"], number> = {
+	low: 0,
+	medium: 1,
+	high: 2,
+	critical: 3,
+};
+
+const DropObservationsSchema = Type.Object({
+	ids: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+	reason: Type.Optional(Type.String()),
+});
+
+type DropObservationsArgs = Static<typeof DropObservationsSchema>;
+
+function joinOrEmpty(items: string[]): string {
+	return items.length ? items.join("\n") : "(none yet)";
+}
+
+function relevanceCounts(observations: readonly Observation[]): Record<Observation["relevance"], number> {
+	return observations.reduce<Record<Observation["relevance"], number>>((counts, observation) => {
+		counts[observation.relevance]++;
+		return counts;
+	}, { low: 0, medium: 0, high: 0, critical: 0 });
+}
+
+export function normalizeDropObservationIds(
+	ids: readonly string[] | undefined,
+	observations: readonly Observation[],
+): string[] | undefined {
+	if (!ids || ids.length === 0) return undefined;
+	const allowed = new Map(observations.map((observation) => [observation.id, observation]));
+	const result: string[] = [];
+	const seen = new Set<string>();
+	for (const id of ids) {
+		const observation = allowed.get(id);
+		if (!observation) continue;
+		if (seen.has(id)) continue;
+		seen.add(id);
+		result.push(id);
+	}
+	return result.length > 0 ? result : undefined;
+}
+
+function timestampRank(timestamp: string): number {
+	const parsed = Date.parse(timestamp);
+	return Number.isFinite(parsed) ? parsed : Number.POSITIVE_INFINITY;
+}
+
+export function selectDropCandidates(
+	ids: readonly string[],
+	observations: readonly Observation[],
+	maxDrops: number,
+	reflections: readonly Reflection[] = [],
+): string[] {
+	if (maxDrops <= 0 || ids.length === 0) return [];
+
+	const byId = new Map(observations.map((observation) => [observation.id, observation]));
+	const coverageById = reflectionCoverageMap(observations, reflections);
+	const firstProposalIndex = new Map<string, number>();
+	for (let i = 0; i < ids.length; i++) {
+		const id = ids[i];
+		if (!firstProposalIndex.has(id)) firstProposalIndex.set(id, i);
+	}
+
+	return Array.from(firstProposalIndex.entries())
+		.map(([id, index]) => ({ id, index, observation: byId.get(id) }))
+		.filter((candidate): candidate is { id: string; index: number; observation: Observation } =>
+			candidate.observation !== undefined
+		)
+		.sort((a, b) => {
+			const coverageDelta = REFLECTION_COVERAGE_DROP_RANK[coverageTierForObservation(a.observation, coverageById)]
+				- REFLECTION_COVERAGE_DROP_RANK[coverageTierForObservation(b.observation, coverageById)];
+			const relevanceDelta = RELEVANCE_DROP_RANK[a.observation.relevance] - RELEVANCE_DROP_RANK[b.observation.relevance];
+			const ageDelta = timestampRank(a.observation.timestamp) - timestampRank(b.observation.timestamp);
+			return coverageDelta || relevanceDelta || ageDelta || a.index - b.index;
+		})
+		.slice(0, maxDrops)
+		.map((candidate) => candidate.id);
+}
+
+export async function runDropper(args: RunDropperArgs): Promise<string[] | undefined> {
+	const { model, apiKey, headers, reflections, observations, targetTokens, signal } = args;
+	if (observations.length === 0) return undefined;
+
+	const metrics = observationPoolMetrics(observations, targetTokens);
+	const { observationTokens, fullness, tokensOverTarget, maxDropsAllowed } = metrics;
+	const coverageById = reflectionCoverageMap(observations, reflections);
+	const coverageSummaryByRelevance = summarizeCoverageByRelevance(observations, coverageById);
+	debugLog("dropper.agent_start", {
+		activeObservationCount: observations.length,
+		reflectionCount: reflections.length,
+		observationTokens,
+		targetTokens,
+		tokensOverTarget,
+		fullness,
+		maxDropsAllowed,
+		relevanceCounts: relevanceCounts(observations),
+		coverageSummaryByRelevance,
+	});
+	if (maxDropsAllowed <= 0) {
+		debugLog("dropper.result", {
+			reason: "not_over_target",
+			toolCallCount: 0,
+			rawRequestedIdsCount: 0,
+			acceptedCandidateCount: 0,
+			selectedDropsCount: 0,
+			selectedDropTokens: 0,
+			selectedCoverageSummaryByRelevance: summarizeCoverageByRelevanceForIds([], observations, coverageById),
+			maxDropsAllowed,
+		});
+		return undefined;
+	}
+
+	const proposedDropIds: string[] = [];
+	const proposed = new Set<string>();
+	const allowed = new Map(observations.map((observation) => [observation.id, observation]));
+	let toolCallCount = 0;
+	let rawRequestedIdsCount = 0;
+	let missingIdsCount = 0;
+	let criticalCandidateIdsCount = 0;
+	let duplicateInRequestCount = 0;
+	let duplicateInRunCount = 0;
+
+	const dropObservations: AgentTool<typeof DropObservationsSchema> = {
+		name: "drop_observations",
+		label: "Drop observations",
+		description: "Propose active observation ids that are safe to remove from compacted memory.",
+		parameters: DropObservationsSchema,
+		execute: async (_id, params: DropObservationsArgs) => {
+			toolCallCount++;
+			rawRequestedIdsCount += params.ids.length;
+			const seenInRequest = new Set<string>();
+			let added = 0;
+			let requestMissingIds = 0;
+			let requestCriticalCandidateIds = 0;
+			let requestDuplicateIds = 0;
+			let requestDuplicateInRunIds = 0;
+			for (const id of params.ids) {
+				const observation = allowed.get(id);
+				if (!observation) {
+					missingIdsCount++;
+					requestMissingIds++;
+					continue;
+				}
+				if (seenInRequest.has(id)) {
+					duplicateInRequestCount++;
+					requestDuplicateIds++;
+					continue;
+				}
+				seenInRequest.add(id);
+				if (proposed.has(id)) {
+					duplicateInRunCount++;
+					requestDuplicateInRunIds++;
+					continue;
+				}
+				proposed.add(id);
+				proposedDropIds.push(id);
+				if (observation.relevance === "critical") {
+					criticalCandidateIdsCount++;
+					requestCriticalCandidateIds++;
+				}
+				added++;
+			}
+			debugLog("dropper.tool_call", {
+				toolCallCount,
+				rawRequestedIdsCount: params.ids.length,
+				acceptedIdsCount: added,
+				missingIdsCount: requestMissingIds,
+				criticalCandidateIdsCount: requestCriticalCandidateIds,
+				duplicateInRequestCount: requestDuplicateIds,
+				duplicateInRunCount: requestDuplicateInRunIds,
+				totalCandidates: proposedDropIds.length,
+				maxDropsAllowed,
+			});
+			return {
+				content: [{ type: "text", text: `Queued ${added} drop candidate${added === 1 ? "" : "s"}. Candidates this run: ${proposedDropIds.length}. Maximum drops allowed: ${maxDropsAllowed}.` }],
+				details: { added, totalCandidates: proposedDropIds.length, maxDropsAllowed },
+			};
+		},
+	};
+
+	const fullnessPercent = Math.round(fullness * 100);
+	const userText = `CURRENT REFLECTIONS:\n${joinOrEmpty(reflections.map(reflectionToSummaryLine))}\n\nCURRENT OBSERVATIONS:\n${joinOrEmpty(observations.map((observation) => observationToDropperLine(observation, coverageTierForObservation(observation, coverageById))))}\n\nActive observation pool: ~${observationTokens.toLocaleString()} tokens; target: ~${targetTokens.toLocaleString()} tokens; fullness against target: ~${fullnessPercent.toLocaleString()}%; over target by ~${tokensOverTarget.toLocaleString()} tokens.\nMaximum drops allowed this run: ${maxDropsAllowed.toLocaleString()} observation${maxDropsAllowed === 1 ? "" : "s"}. This maximum is sized to move the active pool toward the target if every proposed drop is clearly safe.\nThis maximum is a hard upper bound, not a target. Drop fewer or none if fewer observations are clearly safe.`;
+	const prompts: Message[] = [{ role: "user", content: [{ type: "text", text: userText }], timestamp: Date.now() }];
+	const context: AgentContext = { systemPrompt: DROPPER_SYSTEM, messages: [], tools: [dropObservations as AgentTool<any>] };
+	const reasoning = (model as { reasoning?: unknown }).reasoning;
+	const thinkingLevel = args.thinkingLevel ?? "low";
+	const effectiveMaxTurns = args.maxTurns && args.maxTurns > 0 ? args.maxTurns : undefined;
+	let turnCount = 0;
+	const config: AgentLoopConfig = {
+		model,
+		apiKey,
+		headers,
+		maxTokens: boundedMaxTokens(model, AGENT_LOOP_MAX_TOKENS),
+		convertToLlm: (msgs) => msgs as Message[],
+		toolExecution: "sequential",
+		...(reasoning && thinkingLevel !== "off" ? { reasoning: thinkingLevel } : {}),
+		...(effectiveMaxTurns !== undefined ? { shouldStopAfterTurn: () => ++turnCount >= effectiveMaxTurns } : {}),
+	};
+
+	const loop = args.agentLoop ?? agentLoop;
+	const stream = loop(prompts, context, config, signal);
+	for await (const _event of stream) {
+		// Tool execution collects candidate ids.
+	}
+	await stream.result();
+	const droppedIds = selectDropCandidates(proposedDropIds, observations, maxDropsAllowed, reflections);
+	const reason = droppedIds.length > 0
+		? "selected_nonempty"
+		: toolCallCount === 0
+			? "no_tool_call"
+			: proposedDropIds.length === 0
+				? "all_filtered"
+				: "selected_empty";
+	const selectedDropTokens = droppedIds.reduce((sum, id) => sum + (allowed.get(id)?.tokenCount ?? 0), 0);
+	debugLog("dropper.result", {
+		reason,
+		toolCallCount,
+		rawRequestedIdsCount,
+		missingIdsCount,
+		criticalCandidateIdsCount,
+		duplicateInRequestCount,
+		duplicateInRunCount,
+		acceptedCandidateCount: proposedDropIds.length,
+		selectedDropsCount: droppedIds.length,
+		selectedDropTokens,
+		selectedCoverageSummaryByRelevance: summarizeCoverageByRelevanceForIds(droppedIds, observations, coverageById),
+		maxDropsAllowed,
+	});
+	return droppedIds.length > 0 ? droppedIds : undefined;
+}

package/src/agents/dropper/coverage.ts

@@ -0,0 +1,128 @@
+import type { Observation, Reflection } from "../../session-ledger/index.js";
+
+export const REFLECTION_COVERAGE_TIERS = ["none", "partial", "strong"] as const;
+export type ReflectionCoverageTier = typeof REFLECTION_COVERAGE_TIERS[number];
+
+type Relevance = Observation["relevance"];
+
+type CoverageBucket = Record<ReflectionCoverageTier, { count: number; tokens: number }>;
+export type CoverageSummaryByRelevance = Record<Relevance, CoverageBucket>;
+export type CoverageTransitionSummaryByRelevance = Record<Relevance, Record<string, { count: number; tokens: number }>>;
+
+export const REFLECTION_COVERAGE_DROP_RANK: Record<ReflectionCoverageTier, number> = {
+	strong: 0,
+	partial: 1,
+	none: 2,
+};
+
+export function reflectionSupportCounts(reflections: readonly Reflection[]): Map<string, number> {
+	const counts = new Map<string, number>();
+	for (const reflection of reflections) {
+		const uniqueIds = new Set(reflection.supportingObservationIds);
+		for (const id of uniqueIds) counts.set(id, (counts.get(id) ?? 0) + 1);
+	}
+	return counts;
+}
+
+export function reflectionCoverageTierForCount(count: number): ReflectionCoverageTier {
+	if (count <= 0) return "none";
+	if (count === 1) return "partial";
+	return "strong";
+}
+
+export function reflectionCoverageMap(
+	observations: readonly Observation[],
+	reflections: readonly Reflection[],
+): Map<string, ReflectionCoverageTier> {
+	const counts = reflectionSupportCounts(reflections);
+	return new Map(observations.map((observation) => [
+		observation.id,
+		reflectionCoverageTierForCount(counts.get(observation.id) ?? 0),
+	]));
+}
+
+function emptyCoverageBucket(): CoverageBucket {
+	return {
+		none: { count: 0, tokens: 0 },
+		partial: { count: 0, tokens: 0 },
+		strong: { count: 0, tokens: 0 },
+	};
+}
+
+export function emptyCoverageSummaryByRelevance(): CoverageSummaryByRelevance {
+	return {
+		low: emptyCoverageBucket(),
+		medium: emptyCoverageBucket(),
+		high: emptyCoverageBucket(),
+		critical: emptyCoverageBucket(),
+	};
+}
+
+export function summarizeCoverageByRelevance(
+	observations: readonly Observation[],
+	coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
+): CoverageSummaryByRelevance {
+	const summary = emptyCoverageSummaryByRelevance();
+	for (const observation of observations) {
+		const tier = coverageById.get(observation.id) ?? "none";
+		const bucket = summary[observation.relevance][tier];
+		bucket.count++;
+		bucket.tokens += observation.tokenCount;
+	}
+	return summary;
+}
+
+export function summarizeCoverageByRelevanceForIds(
+	ids: readonly string[],
+	observations: readonly Observation[],
+	coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
+): CoverageSummaryByRelevance {
+	const byId = new Map(observations.map((observation) => [observation.id, observation]));
+	const selected = ids.flatMap((id) => {
+		const observation = byId.get(id);
+		return observation ? [observation] : [];
+	});
+	return summarizeCoverageByRelevance(selected, coverageById);
+}
+
+export function emptyCoverageTransitionSummaryByRelevance(): CoverageTransitionSummaryByRelevance {
+	return {
+		low: {},
+		medium: {},
+		high: {},
+		critical: {},
+	};
+}
+
+export function summarizeCoverageTransitionsByRelevance(
+	observations: readonly Observation[],
+	beforeCoverageById: ReadonlyMap<string, ReflectionCoverageTier>,
+	afterCoverageById: ReadonlyMap<string, ReflectionCoverageTier>,
+): CoverageTransitionSummaryByRelevance {
+	const summary = emptyCoverageTransitionSummaryByRelevance();
+	for (const observation of observations) {
+		const before = beforeCoverageById.get(observation.id) ?? "none";
+		const after = afterCoverageById.get(observation.id) ?? "none";
+		if (before === after) continue;
+		const key = `${before}->${after}`;
+		const bucket = summary[observation.relevance][key] ?? { count: 0, tokens: 0 };
+		bucket.count++;
+		bucket.tokens += observation.tokenCount;
+		summary[observation.relevance][key] = bucket;
+	}
+	return summary;
+}
+
+export function observationToDropperLine(
+	observation: Observation,
+	coverage: ReflectionCoverageTier,
+): string {
+	return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] [coverage: ${coverage}] ${observation.content}`;
+}
+
+export function coverageTierForObservation(
+	observation: Observation,
+	coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
+): ReflectionCoverageTier {
+	return coverageById.get(observation.id) ?? "none";
+}

package/src/agents/dropper/pool.ts

@@ -0,0 +1,67 @@
+import type { Observation } from "../../session-ledger/index.js";
+
+export type ObservationPoolMetrics = {
+	observationTokens: number;
+	targetTokens: number;
+	tokensOverTarget: number;
+	fullness: number;
+	activeObservationCount: number;
+	droppableCount: number;
+	maxDropsAllowed: number;
+	overTarget: boolean;
+	ready: boolean;
+};
+
+export function observationTokenSum(observations: readonly { tokenCount: number }[]): number {
+	return observations.reduce((sum, observation) => sum + observation.tokenCount, 0);
+}
+
+export function observationPoolFullness(observationTokens: number, targetTokens: number): number {
+	if (!Number.isFinite(observationTokens) || observationTokens <= 0) return 0;
+	if (!Number.isFinite(targetTokens) || targetTokens <= 0) return 0;
+	return observationTokens / targetTokens;
+}
+
+export function droppableObservationCount(observations: readonly Observation[]): number {
+	return observations.length;
+}
+
+export function maxDropCountForPool(observations: readonly Observation[], observationTokens: number, targetTokens: number): number {
+	const activeObservationCount = observations.length;
+	if (activeObservationCount === 0) return 0;
+	if (!Number.isFinite(observationTokens) || observationTokens <= 0) return 0;
+	if (!Number.isFinite(targetTokens) || targetTokens < 0) return 0;
+
+	const tokensOverTarget = observationTokens - targetTokens;
+	if (tokensOverTarget <= 0) return 0;
+
+	const averageObservationTokens = observationTokens / activeObservationCount;
+	if (!Number.isFinite(averageObservationTokens) || averageObservationTokens <= 0) return 0;
+
+	const estimatedDrops = Math.ceil(tokensOverTarget / averageObservationTokens);
+	return Math.min(activeObservationCount, Math.max(1, estimatedDrops));
+}
+
+export function observationPoolMetrics(
+	observations: readonly Observation[],
+	targetTokens: number,
+): ObservationPoolMetrics {
+	const observationTokens = observationTokenSum(observations);
+	const fullness = observationPoolFullness(observationTokens, targetTokens);
+	const activeObservationCount = observations.length;
+	const droppableCount = droppableObservationCount(observations);
+	const tokensOverTarget = Math.max(0, observationTokens - targetTokens);
+	const maxDropsAllowed = maxDropCountForPool(observations, observationTokens, targetTokens);
+	const overTarget = Number.isFinite(targetTokens) && targetTokens >= 0 && observationTokens > targetTokens;
+	return {
+		observationTokens,
+		targetTokens,
+		tokensOverTarget,
+		fullness,
+		activeObservationCount,
+		droppableCount,
+		maxDropsAllowed,
+		overTarget,
+		ready: overTarget && maxDropsAllowed > 0,
+	};
+}

package/src/agents/dropper/prompts.ts

@@ -0,0 +1,48 @@
+export const DROPPER_SYSTEM = `You are the dropper agent for a coding assistant.
+
+These records are the ONLY information the assistant will have about past interactions once the raw conversation is compacted out of context. Dropping the wrong observation can make future work repeat, contradict, or misremember the user. Take this seriously.
+
+Your job is to identify only the safest active observations to remove from compacted memory by calling drop_observations with their ids. Default action is KEEP. When uncertain, keep the observation.
+
+Active-memory framing. Dropping an observation removes it from active compacted memory; it does not erase the ledger history or source evidence. Still, future compressed context will no longer show the observation, so only drop it when its durable meaning is safely captured elsewhere or it is genuinely low-signal and carries no unique future value.
+
+The user message includes the active observation pool target and "Maximum drops allowed this run". The maximum is a hard upper bound sized to move the pool toward the target if every proposed drop is clearly safe. It is not a target. Do not try to fill it. Drop fewer or none when fewer observations are safely removable. When the active pool is far over target, make a thorough pass over safe candidates rather than stopping after a few obvious examples.
+
+What to drop, in priority order:
+- Redundant observations whose durable meaning is already captured by current reflections with equivalent fidelity.
+- Superseded observations where a later observation clearly replaces the older state.
+- Repeated routine tool acknowledgements or low-signal progress updates that do not carry decisions, constraints, exact errors, or user-specific facts.
+- Older observations that no longer carry working context and are covered by a reflection or a newer observation.
+
+Age-gradient rule. Recent observations carry working context the assistant may still need; older observations have usually been summarized elsewhere or are no longer load-bearing. Prefer older safe drops before newer working context, but age alone is not enough to drop important or uniquely load-bearing observations.
+
+Reflection coverage guidance. Each observation line includes [coverage: none|partial|strong]. Coverage is evidence, not an automatic decision:
+- none: no current reflection cites this observation id. Be cautious, especially for high or critical observations.
+- partial: one current reflection cites this observation id. Compare the observation to the reflection before dropping.
+- strong: two or more current reflections cite this observation id. This is stronger evidence that the durable meaning is preserved, but you must still keep uniquely load-bearing or uncertain observations.
+
+Relevance guidance. Relevance is importance/resistance, not an absolute keep/drop lock:
+- low: consider first, but drop only when it carries no unique detail, decision, state, error, identifier, or user-specific fact.
+- medium: drop when redundant with reflections or other observations, or when the work state is clearly obsolete.
+- high: drop only when clearly superseded or already captured by a reflection with equivalent fidelity.
+- critical: highest importance and strongest resistance. Do not drop fresh or uniquely load-bearing critical observations. Critical observations may be dropped only with strong semantic evidence such as age plus partial/strong reflection coverage, supersession by newer memory, redundancy, or clear obsolescence.
+
+User assertions and concrete completions must be preserved unless a current reflection or newer observation preserves the exact assertion/completion and its important details with equivalent fidelity.
+
+Preservation floor. Regardless of relevance label, budget pressure, coverage, or age, do not drop observations that uniquely carry any of the following:
+- User preferences, constraints, corrections, or identity/role facts.
+- Concrete completions that future runs must not redo.
+- Named identifiers, file paths, function names, package names, tickets, commit SHAs, handles, or exact commands.
+- Exact error messages, diagnostic output, or test failure names.
+- Architectural or technical decisions and their rationale.
+- Dates of specific events, deadlines, meetings, migrations, or incidents.
+- Current unresolved blockers, TODOs, partial work, or decisions waiting on the user.
+- Non-standard user terminology or unusual phrasing needed for future recognition.
+
+What you cannot do:
+- You cannot merge observations.
+- You cannot rewrite or edit observations.
+- You cannot add new observations or reflections.
+- You can only call drop_observations with ids from the current observations list.
+
+Do not force drops you do not believe in. If no observations are safe to drop, do not call the tool and reply briefly. Hitting the budget or maximum count is less important than preserving load-bearing memory.`;

package/src/{observer.ts → agents/observer/agent.ts}

@@ -1,12 +1,13 @@
-import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@mariozechner/pi-agent-core";
-import type { Message, Model, ModelThinkingLevel } from "@mariozechner/pi-ai";
-import { Type } from "@mariozechner/pi-ai";
+import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
+import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
+import { Type } from "@earendil-works/pi-ai";
 import type { Static } from "typebox";
-import { hashId } from "./ids.js";
-import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "./model-budget.js";
+import { hashId } from "../../ids.js";
+import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
 import { OBSERVER_SYSTEM } from "./prompts.js";
-import { nowTimestamp, truncateRecordContent } from "./serialize.js";
-import type { ObservationRecord, Relevance } from "./types.js";
+import { nowTimestamp, truncateRecordContent } from "../../serialize.js";
+import type { Observation, Relevance } from "../../session-ledger/index.js";
+import { estimateStringTokens } from "../../tokens.js";
 
 interface RunObserverArgs {
 	model: Model<any>;
@@ -80,12 +81,12 @@ export function normalizeSourceEntryIds(
 	return Array.from(seen).sort((a, b) => (allowedOrder.get(a) ?? 0) - (allowedOrder.get(b) ?? 0));
 }
 
-export async function runObserver(args: RunObserverArgs): Promise<ObservationRecord[] | undefined> {
+export async function runObserver(args: RunObserverArgs): Promise<Observation[] | undefined> {
 	const { model, apiKey, headers, priorReflections, priorObservations, chunk, allowedSourceEntryIds, signal } = args;
 	const conversation = chunk.trim();
 	if (!conversation) return undefined;
 
-	const accumulated = new Map<string, ObservationRecord>();
+	const accumulated = new Map<string, Observation>();
 
 	const recordObservations: AgentTool<typeof RecordObservationsSchema> = {
 		name: "record_observations",
@@ -117,6 +118,7 @@ export async function runObserver(args: RunObserverArgs): Promise<ObservationRec
 					timestamp: obs.timestamp,
 					relevance: obs.relevance as Relevance,
 					sourceEntryIds,
+					tokenCount: estimateStringTokens(content),
 				});
 				added++;
 			}
@@ -193,7 +195,3 @@ ${conversation}`;
 	if (accumulated.size === 0) return undefined;
 	return Array.from(accumulated.values());
 }
-
-export function observationsToPromptLines(records: ObservationRecord[]): string[] {
-	return records.map((r) => `[${r.id}] ${r.timestamp} [${r.relevance}] ${r.content}`);
-}