pi-observational-memory 2.3.0 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -22
- package/package.json +1 -1
- package/src/branch.ts +275 -6
- package/src/commands/status.ts +37 -11
- package/src/commands/view.ts +6 -5
- package/src/compaction.ts +330 -44
- package/src/config.ts +21 -2
- package/src/hooks/compaction-hook.ts +27 -22
- package/src/hooks/compaction-trigger.ts +31 -23
- package/src/hooks/observer-trigger.ts +13 -7
- package/src/prompts.ts +37 -13
- package/src/runtime.ts +6 -1
- package/src/serialize.ts +3 -1
- package/src/tools/recall-observation.ts +315 -111
- package/src/types.ts +91 -9
package/README.md
CHANGED
|
@@ -22,31 +22,34 @@ What the agent sees after compaction looks like this:
|
|
|
22
22
|
|
|
23
23
|
```
|
|
24
24
|
## Reflections
|
|
25
|
-
User works at Acme Corp building Acme Dashboard on Next.js 15 with Supabase auth.
|
|
26
|
-
Hard constraint: ship by January 22nd 2026.
|
|
27
|
-
Public API uses GraphQL (switched from REST to reduce mobile over-fetching).
|
|
25
|
+
[a1b2c3d4e5f6] User works at Acme Corp building Acme Dashboard on Next.js 15 with Supabase auth.
|
|
26
|
+
[b2c3d4e5f6a1] Hard constraint: ship by January 22nd 2026.
|
|
27
|
+
[c3d4e5f6a1b2] Public API uses GraphQL (switched from REST to reduce mobile over-fetching).
|
|
28
28
|
|
|
29
29
|
## Observations
|
|
30
|
-
2026-01-15 14:30 [high] User decided to switch from REST to GraphQL for the public API; motivation was reducing over-fetching on mobile clients.
|
|
31
|
-
2026-01-15 14:35 [medium] Agent scaffolded GraphQL schema in src/schema.ts.
|
|
32
|
-
2026-01-15 14:50 [medium] GraphQL migration completed; user confirmed queries working.
|
|
33
|
-
2026-01-15 15:10 [critical] User wants rate limiting on all public endpoints; prefers token bucket algorithm at 100 req/min per API key.
|
|
30
|
+
[d4e5f6a1b2c3] 2026-01-15 14:30 [high] User decided to switch from REST to GraphQL for the public API; motivation was reducing over-fetching on mobile clients.
|
|
31
|
+
[e5f6a1b2c3d4] 2026-01-15 14:35 [medium] Agent scaffolded GraphQL schema in src/schema.ts.
|
|
32
|
+
[f6a1b2c3d4e5] 2026-01-15 14:50 [medium] GraphQL migration completed; user confirmed queries working.
|
|
33
|
+
[a6b1c2d3e4f5] 2026-01-15 15:10 [critical] User wants rate limiting on all public endpoints; prefers token bucket algorithm at 100 req/min per API key.
|
|
34
34
|
```
|
|
35
35
|
|
|
36
36
|
Two layers of memory, two different jobs:
|
|
37
37
|
|
|
38
|
-
- **Reflections** are durable patterns — who you are, what you've decided, hard constraints.
|
|
39
|
-
- **Observations** are timestamped events with a per-entry relevance tier (`low` / `medium` / `high` / `critical`). They're written near-real-time, then pruned over time — but never paraphrased.
|
|
38
|
+
- **Reflections** are durable patterns — who you are, what you've decided, hard constraints. They render as plain prose with an id handle when recallable, and persist across future compactions.
|
|
39
|
+
- **Observations** are timestamped events with an id and a per-entry relevance tier (`low` / `medium` / `high` / `critical`). They're written near-real-time, then pruned over time — but never paraphrased.
|
|
40
|
+
|
|
41
|
+
Those ids are not decoration. When the agent needs exact evidence behind a compacted memory item, it can call the agent-facing `recall` tool with a reflection or observation id. The TUI shows a compact evidence summary, while the agent receives the full raw source context that produced the memory.
|
|
40
42
|
|
|
41
43
|
Hour six should feel like hour one. The agent knows who you are, what you've built together, and what's left to do.
|
|
42
44
|
|
|
43
45
|
## What you actually get from it
|
|
44
46
|
|
|
45
|
-
- **Continuity
|
|
47
|
+
- **Continuity across many compactions.** The summary is built by mechanical concatenation, not an LLM rewrite. Kept observations and reflections are carried forward without paraphrase; observations may be pruned later, but they are never rewritten into summary-of-summary drift.
|
|
46
48
|
- **Temporal reasoning.** Every observation carries a per-minute timestamp. The agent can reason about *when* something happened, not just *that* it happened.
|
|
49
|
+
- **Source-backed recall.** Observation and reflection ids let the agent recover the exact prior conversation/tool evidence behind compacted memory when precision matters.
|
|
47
50
|
- **Relevance-aware pruning.** Four relevance tiers drive what gets dropped first when the observation pool grows. Trivia goes; user assertions, decisions, and verbatim errors stay.
|
|
48
51
|
- **Reflections that crystallize.** Identity, constraints, and durable preferences settle into a separate layer that doesn't get re-paraphrased on each compaction.
|
|
49
|
-
- **Predictable token cost.** Properly configured for your use case, this can save real money. The reflector + pruner only run above a configurable gate, so
|
|
52
|
+
- **Predictable token cost.** Properly configured for your use case, this can save real money. The reflector + pruner only run above a configurable gate, so below-gate compactions skip those LLM calls; they only call a model if sync catch-up observation is needed. The observer can be pointed at a cheap fast model independently of your main coding model.
|
|
50
53
|
- **Cache-friendly by design.** Memory updates are batched at compaction boundaries instead of injected into every turn, so prompt prefix caching keeps working between compactions.
|
|
51
54
|
- **Fewer mid-work surprises.** The extension proactively triggers compaction when the agent is idle, and this will not affect your current work as after compaction you still keep the tail of your session intact.
|
|
52
55
|
|
|
@@ -73,19 +76,19 @@ flowchart TD
|
|
|
73
76
|
Conv([Conversation accumulates])
|
|
74
77
|
Obs[Observer<br/>async, fire-and-forget<br/>compresses each chunk into timestamped,<br/>relevance-tagged observations<br/>stored as silent tree entries]
|
|
75
78
|
Comp[Compaction<br/>extension-owned; merges accumulated<br/>observations with prior compaction state]
|
|
76
|
-
RP[Reflector + Pruner<br/>Reflector
|
|
77
|
-
Sum[Summary mechanically assembled<br/>## Reflections<br/>
|
|
79
|
+
RP[Reflector + Pruner<br/>Reflector runs focused passes<br/>to crystallize durable patterns<br/>Pruner drops observations by id<br/>across up to 5 passes]
|
|
80
|
+
Sum[Summary mechanically assembled<br/>## Reflections<br/> [id] durable insight<br/>## Observations<br/> [id] YYYY-MM-DD HH:MM relevance ...<br/>Becomes the compactionSummary<br/>the agent sees on the next turn]
|
|
78
81
|
|
|
79
82
|
Conv -->|every ~1k raw tokens since last bound| Obs
|
|
80
|
-
Obs -->|
|
|
83
|
+
Obs -->|live tail reaches ~50k raw tokens| Comp
|
|
81
84
|
Comp -->|observation pool ≥ 30k tokens| RP
|
|
82
85
|
RP --> Sum
|
|
83
|
-
Comp -.->|pool below gate — skip
|
|
86
|
+
Comp -.->|pool below gate — skip reflector/pruner| Sum
|
|
84
87
|
```
|
|
85
88
|
|
|
86
89
|
- **Observer** runs in the background as turns complete. The user never waits on it.
|
|
87
90
|
- **Compaction** is owned by the extension. The summary is *mechanically concatenated* from current reflections + current observations — never an LLM rewrite. This is what eliminates the summary-of-a-summary problem.
|
|
88
|
-
- **Reflector + Pruner** run as an inseparable pair, and only when there's enough material to crystallize. Below the gate, compaction
|
|
91
|
+
- **Reflector + Pruner** run as an inseparable pair, and only when there's enough material to crystallize. Below the gate, those roles are skipped; compaction only calls a model if sync catch-up observation is needed for uncovered raw history.
|
|
89
92
|
|
|
90
93
|
The agent only ever sees the most recent compaction summary, packaged as a normal `compactionSummary` message. Observations and reflections are never injected into the live message stream — that would invalidate prefix caching with every observation. By batching memory updates at compaction boundaries, the prefix stays stable between compactions and prefix caching keeps working.
|
|
91
94
|
|
|
@@ -104,7 +107,8 @@ Settings live in Pi's `settings.json` — globally at `~/.pi/agent/settings.json
|
|
|
104
107
|
"observational-memory": {
|
|
105
108
|
"observationThresholdTokens": 1000,
|
|
106
109
|
"compactionThresholdTokens": 50000,
|
|
107
|
-
"reflectionThresholdTokens": 30000
|
|
110
|
+
"reflectionThresholdTokens": 30000,
|
|
111
|
+
"passive": false
|
|
108
112
|
},
|
|
109
113
|
"compaction": {
|
|
110
114
|
"keepRecentTokens": 20000
|
|
@@ -122,26 +126,30 @@ To run the background memory work (observer, reflector, pruner) on a cheaper / f
|
|
|
122
126
|
}
|
|
123
127
|
```
|
|
124
128
|
|
|
125
|
-
The
|
|
129
|
+
The six settings most worth knowing:
|
|
126
130
|
|
|
127
131
|
| Setting | Default | What it controls |
|
|
128
132
|
|---|---|---|
|
|
129
133
|
| `observationThresholdTokens` | `1,000` | How often the observer fires in the background |
|
|
130
134
|
| `compactionThresholdTokens` | `50,000` | How often the extension proactively triggers compaction |
|
|
131
135
|
| `reflectionThresholdTokens` | `30,000` | The observation pool size at which reflector + pruner engage |
|
|
136
|
+
| `passive` | `false` | Disables proactive observation and extension-triggered compaction while keeping manual/Pi compaction and commands active |
|
|
132
137
|
| `compactionModel` | session model | Which model runs the observer / reflector / pruner — point at a cheaper one to save cost |
|
|
133
138
|
| `compaction.keepRecentTokens` | `20,000` | How much recent conversation Pi keeps verbatim post-compaction (Pi setting; structural to the extension) |
|
|
134
139
|
|
|
140
|
+
For shell/session-level control, `PI_OBSERVATIONAL_MEMORY_PASSIVE` overrides global and project settings. Use `1`, `true`, `yes`, or `on` to enable passive mode; use `0`, `false`, `no`, or `off` to force it off.
|
|
141
|
+
|
|
135
142
|
For the full list and tuning recipes, see **[docs/configuration.md](docs/configuration.md)**.
|
|
136
143
|
|
|
137
144
|
> **Upgrading from `pi-observational-memory@1.x`?** The config keys changed: v1's `observationThreshold` is now `compactionThresholdTokens`, v1's `reflectionThreshold` is now `reflectionThresholdTokens`, and `observationThresholdTokens` is new. Old v1 keys are silently ignored — update your `settings.json`.
|
|
138
145
|
|
|
139
|
-
## Commands
|
|
146
|
+
## Commands and agent tool
|
|
140
147
|
|
|
141
|
-
|
|
|
148
|
+
| Surface | What it does |
|
|
142
149
|
|---|---|
|
|
143
|
-
| `/om-status` | Memory totals, percent-to-threshold for each gate, and in-flight flags for observer and compaction |
|
|
144
|
-
| `/om-view` | Full dump of memory state: every reflection, every committed observation, every pending observation.
|
|
150
|
+
| `/om-status` | Memory totals, percent-to-threshold for each gate, passive-mode status, and in-flight flags for observer and compaction |
|
|
151
|
+
| `/om-view` | Full dump of memory state: every reflection, every committed observation, every pending observation. Reflection and observation ids shown here can be used with `recall` |
|
|
152
|
+
| `recall` agent tool | Recovers exact source evidence for a specific reflection or observation id on the current branch. It is for agent self-recovery and provenance, not a user command, semantic search, or transcript browser |
|
|
145
153
|
|
|
146
154
|
## Credits
|
|
147
155
|
|
package/package.json
CHANGED
package/src/branch.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
OBSERVATION_CUSTOM_TYPE,
|
|
3
|
-
isMemoryDetails,
|
|
4
3
|
isObservationEntryData,
|
|
5
|
-
|
|
4
|
+
isReflectionRecord,
|
|
5
|
+
isSupportedMemoryDetails,
|
|
6
|
+
type MemoryReflection,
|
|
6
7
|
type ObservationEntryData,
|
|
7
8
|
type ObservationRecord,
|
|
8
|
-
type
|
|
9
|
+
type ReflectionRecord,
|
|
10
|
+
type SupportedMemoryDetails,
|
|
9
11
|
} from "./types.js";
|
|
10
12
|
import { estimateEntryTokens } from "./tokens.js";
|
|
11
13
|
|
|
@@ -69,6 +71,86 @@ export type RecallObservationSourcesResult =
|
|
|
69
71
|
collision: boolean;
|
|
70
72
|
};
|
|
71
73
|
|
|
74
|
+
export type RecallMemoryObservation =
|
|
75
|
+
| {
|
|
76
|
+
status: "ok";
|
|
77
|
+
observation: ObservationRecord;
|
|
78
|
+
observationEntryId: string;
|
|
79
|
+
observationRecordIndex: number;
|
|
80
|
+
sourceEntryIds: string[];
|
|
81
|
+
sourceEntries: Entry[];
|
|
82
|
+
}
|
|
83
|
+
| {
|
|
84
|
+
status: "no_source";
|
|
85
|
+
observation: ObservationRecord;
|
|
86
|
+
observationEntryId: string;
|
|
87
|
+
observationRecordIndex: number;
|
|
88
|
+
}
|
|
89
|
+
| {
|
|
90
|
+
status: "source_unavailable";
|
|
91
|
+
observation: ObservationRecord;
|
|
92
|
+
observationEntryId: string;
|
|
93
|
+
observationRecordIndex: number;
|
|
94
|
+
sourceEntryIds: string[];
|
|
95
|
+
sourceEntries: Entry[];
|
|
96
|
+
missingSourceEntryIds: string[];
|
|
97
|
+
nonSourceEntryIds: string[];
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
export type RecallMemoryReflectionMatch = {
|
|
101
|
+
reflection: ReflectionRecord;
|
|
102
|
+
reflectionIndex: number;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
export type RecallUnavailableSupportingObservation = {
|
|
106
|
+
reflection: ReflectionRecord;
|
|
107
|
+
reflectionIndex: number;
|
|
108
|
+
observationId: string;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
export type RecallUnavailableReflectionProvenance = {
|
|
112
|
+
reflection: ReflectionRecord;
|
|
113
|
+
reflectionIndex: number;
|
|
114
|
+
reason: "legacy";
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
export type RecallMemorySourcesResult =
|
|
118
|
+
| {
|
|
119
|
+
status: "not_found";
|
|
120
|
+
memoryId: string;
|
|
121
|
+
reflectionMatches: [];
|
|
122
|
+
directObservationMatches: [];
|
|
123
|
+
observations: [];
|
|
124
|
+
sourceEntries: [];
|
|
125
|
+
unavailableSupportingObservations: [];
|
|
126
|
+
unavailableReflectionProvenance: [];
|
|
127
|
+
missingSourceEntryIds: [];
|
|
128
|
+
nonSourceEntryIds: [];
|
|
129
|
+
collision: false;
|
|
130
|
+
partial: false;
|
|
131
|
+
}
|
|
132
|
+
| {
|
|
133
|
+
status: "found";
|
|
134
|
+
memoryId: string;
|
|
135
|
+
reflectionMatches: RecallMemoryReflectionMatch[];
|
|
136
|
+
directObservationMatches: RecallMemoryObservation[];
|
|
137
|
+
observations: RecallMemoryObservation[];
|
|
138
|
+
sourceEntries: Entry[];
|
|
139
|
+
unavailableSupportingObservations: RecallUnavailableSupportingObservation[];
|
|
140
|
+
unavailableReflectionProvenance: RecallUnavailableReflectionProvenance[];
|
|
141
|
+
missingSourceEntryIds: string[];
|
|
142
|
+
nonSourceEntryIds: string[];
|
|
143
|
+
collision: boolean;
|
|
144
|
+
partial: boolean;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
type IndexedObservation = {
|
|
148
|
+
observation: ObservationRecord;
|
|
149
|
+
observationEntryId: string;
|
|
150
|
+
observationRecordIndex: number;
|
|
151
|
+
branchIndex: number;
|
|
152
|
+
};
|
|
153
|
+
|
|
72
154
|
export function findLastCompactionIndex(entries: Entry[]): number {
|
|
73
155
|
for (let i = entries.length - 1; i >= 0; i--) {
|
|
74
156
|
if (entries[i].type === "compaction") return i;
|
|
@@ -152,6 +234,23 @@ function uniqueIds(ids: string[]): string[] {
|
|
|
152
234
|
return Array.from(new Set(ids));
|
|
153
235
|
}
|
|
154
236
|
|
|
237
|
+
function collectIndexedObservations(entries: Entry[]): IndexedObservation[] {
|
|
238
|
+
const observations: IndexedObservation[] = [];
|
|
239
|
+
for (let branchIndex = 0; branchIndex < entries.length; branchIndex++) {
|
|
240
|
+
const entry = entries[branchIndex];
|
|
241
|
+
if (!isObservationEntry(entry)) continue;
|
|
242
|
+
if (!isObservationEntryData(entry.data)) continue;
|
|
243
|
+
entry.data.records.forEach((observation, observationRecordIndex) => {
|
|
244
|
+
observations.push({ observation, observationEntryId: entry.id, observationRecordIndex, branchIndex });
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
return observations;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function observationKey(observation: Pick<IndexedObservation, "observationEntryId" | "observationRecordIndex">): string {
|
|
251
|
+
return `${observation.observationEntryId}:${observation.observationRecordIndex}`;
|
|
252
|
+
}
|
|
253
|
+
|
|
155
254
|
function resolveSourceEntries(entries: Entry[], sourceEntryIds: string[]): {
|
|
156
255
|
status: "ok" | "source_unavailable";
|
|
157
256
|
sourceEntryIds: string[];
|
|
@@ -187,6 +286,81 @@ function resolveSourceEntries(entries: Entry[], sourceEntryIds: string[]): {
|
|
|
187
286
|
};
|
|
188
287
|
}
|
|
189
288
|
|
|
289
|
+
function resolveSourceEntriesPartial(entries: Entry[], sourceEntryIds: string[]): {
|
|
290
|
+
status: "ok" | "source_unavailable";
|
|
291
|
+
sourceEntryIds: string[];
|
|
292
|
+
sourceEntries: Entry[];
|
|
293
|
+
missingSourceEntryIds: string[];
|
|
294
|
+
nonSourceEntryIds: string[];
|
|
295
|
+
} {
|
|
296
|
+
const requested = uniqueIds(sourceEntryIds);
|
|
297
|
+
const requestedSet = new Set(requested);
|
|
298
|
+
const entriesById = new Map(entries.map((entry) => [entry.id, entry]));
|
|
299
|
+
const missingSourceEntryIds = requested.filter((id) => !entriesById.has(id));
|
|
300
|
+
const nonSourceEntryIds = requested.filter((id) => {
|
|
301
|
+
const entry = entriesById.get(id);
|
|
302
|
+
return entry !== undefined && !isSourceEntry(entry);
|
|
303
|
+
});
|
|
304
|
+
const sourceEntries = entries.filter((entry) => requestedSet.has(entry.id) && isSourceEntry(entry));
|
|
305
|
+
return {
|
|
306
|
+
status: missingSourceEntryIds.length > 0 || nonSourceEntryIds.length > 0 ? "source_unavailable" : "ok",
|
|
307
|
+
sourceEntryIds: requested,
|
|
308
|
+
sourceEntries,
|
|
309
|
+
missingSourceEntryIds,
|
|
310
|
+
nonSourceEntryIds,
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function memoryObservationFromIndexed(entries: Entry[], indexed: IndexedObservation): RecallMemoryObservation {
|
|
315
|
+
const { observation, observationEntryId, observationRecordIndex } = indexed;
|
|
316
|
+
if (!observation.sourceEntryIds || observation.sourceEntryIds.length === 0) {
|
|
317
|
+
return { status: "no_source", observation, observationEntryId, observationRecordIndex };
|
|
318
|
+
}
|
|
319
|
+
const resolved = resolveSourceEntriesPartial(entries, observation.sourceEntryIds);
|
|
320
|
+
if (resolved.status === "source_unavailable") {
|
|
321
|
+
return {
|
|
322
|
+
status: "source_unavailable",
|
|
323
|
+
observation,
|
|
324
|
+
observationEntryId,
|
|
325
|
+
observationRecordIndex,
|
|
326
|
+
sourceEntryIds: resolved.sourceEntryIds,
|
|
327
|
+
sourceEntries: resolved.sourceEntries,
|
|
328
|
+
missingSourceEntryIds: resolved.missingSourceEntryIds,
|
|
329
|
+
nonSourceEntryIds: resolved.nonSourceEntryIds,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
status: "ok",
|
|
334
|
+
observation,
|
|
335
|
+
observationEntryId,
|
|
336
|
+
observationRecordIndex,
|
|
337
|
+
sourceEntryIds: resolved.sourceEntryIds,
|
|
338
|
+
sourceEntries: resolved.sourceEntries,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function uniqueSourceEntriesInBranchOrder(entries: Entry[], observations: RecallMemoryObservation[]): Entry[] {
|
|
343
|
+
const requested = new Set<string>();
|
|
344
|
+
for (const observation of observations) {
|
|
345
|
+
if (observation.status === "ok" || observation.status === "source_unavailable") {
|
|
346
|
+
for (const entry of observation.sourceEntries) requested.add(entry.id);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
return entries.filter((entry) => requested.has(entry.id) && isSourceEntry(entry));
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function uniqueUnavailableSourceIds(
|
|
353
|
+
observations: RecallMemoryObservation[],
|
|
354
|
+
field: "missingSourceEntryIds" | "nonSourceEntryIds",
|
|
355
|
+
): string[] {
|
|
356
|
+
const ids: string[] = [];
|
|
357
|
+
for (const observation of observations) {
|
|
358
|
+
if (observation.status !== "source_unavailable") continue;
|
|
359
|
+
ids.push(...observation[field]);
|
|
360
|
+
}
|
|
361
|
+
return uniqueIds(ids);
|
|
362
|
+
}
|
|
363
|
+
|
|
190
364
|
export function recallObservationSources(entries: Entry[], observationId: string): RecallObservationSourcesResult {
|
|
191
365
|
const matches: RecallObservationMatch[] = [];
|
|
192
366
|
for (const entry of entries) {
|
|
@@ -226,11 +400,106 @@ export function recallObservationSources(entries: Entry[], observationId: string
|
|
|
226
400
|
return { status: "found", observationId, matches, collision: matches.length > 1 };
|
|
227
401
|
}
|
|
228
402
|
|
|
229
|
-
function getPriorMemoryDetails(entries: Entry[]):
|
|
403
|
+
function getPriorMemoryDetails(entries: Entry[]): SupportedMemoryDetails | undefined {
|
|
230
404
|
const idx = findLastCompactionIndex(entries);
|
|
231
405
|
if (idx === -1) return undefined;
|
|
232
406
|
const details = entries[idx].details;
|
|
233
|
-
return
|
|
407
|
+
return isSupportedMemoryDetails(details) ? details : undefined;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
export function recallMemorySources(entries: Entry[], memoryId: string): RecallMemorySourcesResult {
|
|
411
|
+
const indexedObservations = collectIndexedObservations(entries);
|
|
412
|
+
const observationsById = new Map<string, IndexedObservation[]>();
|
|
413
|
+
for (const observation of indexedObservations) {
|
|
414
|
+
const matches = observationsById.get(observation.observation.id) ?? [];
|
|
415
|
+
matches.push(observation);
|
|
416
|
+
observationsById.set(observation.observation.id, matches);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const priorDetails = getPriorMemoryDetails(entries);
|
|
420
|
+
const reflectionMatches: RecallMemoryReflectionMatch[] = [];
|
|
421
|
+
if (priorDetails) {
|
|
422
|
+
priorDetails.reflections.forEach((reflection, reflectionIndex) => {
|
|
423
|
+
if (!isReflectionRecord(reflection) || reflection.id !== memoryId) return;
|
|
424
|
+
reflectionMatches.push({ reflection, reflectionIndex });
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
const directIndexedObservations = observationsById.get(memoryId) ?? [];
|
|
429
|
+
const observationsByKey = new Map<string, IndexedObservation>();
|
|
430
|
+
for (const observation of directIndexedObservations) {
|
|
431
|
+
observationsByKey.set(observationKey(observation), observation);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
const unavailableSupportingObservations: RecallUnavailableSupportingObservation[] = [];
|
|
435
|
+
const unavailableReflectionProvenance: RecallUnavailableReflectionProvenance[] = [];
|
|
436
|
+
for (const reflectionMatch of reflectionMatches) {
|
|
437
|
+
if (reflectionMatch.reflection.legacy === true) {
|
|
438
|
+
unavailableReflectionProvenance.push({ ...reflectionMatch, reason: "legacy" });
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
for (const observationId of reflectionMatch.reflection.supportingObservationIds) {
|
|
442
|
+
const supportingObservations = observationsById.get(observationId);
|
|
443
|
+
if (!supportingObservations || supportingObservations.length === 0) {
|
|
444
|
+
unavailableSupportingObservations.push({ ...reflectionMatch, observationId });
|
|
445
|
+
continue;
|
|
446
|
+
}
|
|
447
|
+
for (const observation of supportingObservations) {
|
|
448
|
+
observationsByKey.set(observationKey(observation), observation);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
const indexedObservationBag = Array.from(observationsByKey.values()).sort((a, b) => {
|
|
454
|
+
if (a.branchIndex !== b.branchIndex) return a.branchIndex - b.branchIndex;
|
|
455
|
+
return a.observationRecordIndex - b.observationRecordIndex;
|
|
456
|
+
});
|
|
457
|
+
const observations = indexedObservationBag.map((observation) => memoryObservationFromIndexed(entries, observation));
|
|
458
|
+
const directObservationKeys = new Set(directIndexedObservations.map(observationKey));
|
|
459
|
+
const directObservationMatches = observations.filter((observation) => directObservationKeys.has(observationKey(observation)));
|
|
460
|
+
const sourceEntries = uniqueSourceEntriesInBranchOrder(entries, observations);
|
|
461
|
+
const missingSourceEntryIds = uniqueUnavailableSourceIds(observations, "missingSourceEntryIds");
|
|
462
|
+
const nonSourceEntryIds = uniqueUnavailableSourceIds(observations, "nonSourceEntryIds");
|
|
463
|
+
|
|
464
|
+
if (reflectionMatches.length === 0 && directObservationMatches.length === 0) {
|
|
465
|
+
return {
|
|
466
|
+
status: "not_found",
|
|
467
|
+
memoryId,
|
|
468
|
+
reflectionMatches: [],
|
|
469
|
+
directObservationMatches: [],
|
|
470
|
+
observations: [],
|
|
471
|
+
sourceEntries: [],
|
|
472
|
+
unavailableSupportingObservations: [],
|
|
473
|
+
unavailableReflectionProvenance: [],
|
|
474
|
+
missingSourceEntryIds: [],
|
|
475
|
+
nonSourceEntryIds: [],
|
|
476
|
+
collision: false,
|
|
477
|
+
partial: false,
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
const hasNoSourceObservationInMemoryRecall =
|
|
482
|
+
reflectionMatches.length > 0 && observations.some((observation) => observation.status === "no_source");
|
|
483
|
+
|
|
484
|
+
return {
|
|
485
|
+
status: "found",
|
|
486
|
+
memoryId,
|
|
487
|
+
reflectionMatches,
|
|
488
|
+
directObservationMatches,
|
|
489
|
+
observations,
|
|
490
|
+
sourceEntries,
|
|
491
|
+
unavailableSupportingObservations,
|
|
492
|
+
unavailableReflectionProvenance,
|
|
493
|
+
missingSourceEntryIds,
|
|
494
|
+
nonSourceEntryIds,
|
|
495
|
+
collision: reflectionMatches.length + directObservationMatches.length > 1,
|
|
496
|
+
partial:
|
|
497
|
+
unavailableReflectionProvenance.length > 0 ||
|
|
498
|
+
hasNoSourceObservationInMemoryRecall ||
|
|
499
|
+
unavailableSupportingObservations.length > 0 ||
|
|
500
|
+
missingSourceEntryIds.length > 0 ||
|
|
501
|
+
nonSourceEntryIds.length > 0,
|
|
502
|
+
};
|
|
234
503
|
}
|
|
235
504
|
|
|
236
505
|
export function collectObservationsByCoverage(
|
|
@@ -292,7 +561,7 @@ function collectObservationsPendingNextCompaction(entries: Entry[]): Observation
|
|
|
292
561
|
}
|
|
293
562
|
|
|
294
563
|
export interface MemoryState {
|
|
295
|
-
reflections:
|
|
564
|
+
reflections: MemoryReflection[];
|
|
296
565
|
committedObs: ObservationRecord[];
|
|
297
566
|
pendingObs: ObservationRecord[];
|
|
298
567
|
}
|
package/src/commands/status.ts
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
import { countByRelevance, formatRelevanceHistogram } from "../relevance.js";
|
|
9
9
|
import type { Runtime } from "../runtime.js";
|
|
10
10
|
import { estimateStringTokens } from "../tokens.js";
|
|
11
|
+
import { reflectionContent, type MemoryReflection } from "../types.js";
|
|
11
12
|
|
|
12
13
|
export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void {
|
|
13
14
|
pi.registerCommand("om-status", {
|
|
@@ -19,10 +20,11 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
|
|
|
19
20
|
const sinceCompaction = rawTokensSinceLastCompaction(entries);
|
|
20
21
|
|
|
21
22
|
const { reflections: committedRefs, committedObs, pendingObs } = getMemoryState(entries);
|
|
23
|
+
const committedRefItems = committedRefs as MemoryReflection[];
|
|
22
24
|
const committedObsTokens = committedObs.reduce((s, r) => s + estimateStringTokens(r.content), 0);
|
|
23
25
|
const committedObsCount = committedObs.length;
|
|
24
|
-
const committedRefsTokens =
|
|
25
|
-
const committedRefsCount =
|
|
26
|
+
const committedRefsTokens = committedRefItems.reduce((s, r) => s + estimateStringTokens(reflectionContent(r)), 0);
|
|
27
|
+
const committedRefsCount = committedRefItems.length;
|
|
26
28
|
|
|
27
29
|
const pendingObsTokens = pendingObs.reduce((s, r) => s + estimateStringTokens(r.content), 0);
|
|
28
30
|
const pendingObsCount = pendingObs.length;
|
|
@@ -48,7 +50,39 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
|
|
|
48
50
|
const cObsLabel = committedObsCount === 1 ? "observation" : "observations";
|
|
49
51
|
const pObsLabel = pendingObsCount === 1 ? "observation" : "observations";
|
|
50
52
|
|
|
53
|
+
const passiveLines = runtime.config.passive === true
|
|
54
|
+
? [
|
|
55
|
+
"── Mode ──",
|
|
56
|
+
"Passive: proactive observation and compaction triggers disabled; compaction hook remains active",
|
|
57
|
+
"",
|
|
58
|
+
]
|
|
59
|
+
: [];
|
|
60
|
+
|
|
61
|
+
const activityLines = runtime.config.passive === true
|
|
62
|
+
? [
|
|
63
|
+
"── Activity ──",
|
|
64
|
+
`Observation trigger: passive (~${sinceBound.toLocaleString()} / ${obsThreshold.toLocaleString()} tokens, ${obsPct}%)`,
|
|
65
|
+
" → proactive observation is disabled; manual/Pi compaction can still run sync catch-up observation",
|
|
66
|
+
`Compaction trigger: passive (~${sinceCompaction.toLocaleString()} / ${compThreshold.toLocaleString()} tokens, ${compPct}%)`,
|
|
67
|
+
" → proactive extension-triggered compaction is disabled; manual/Pi compaction still uses the custom hook",
|
|
68
|
+
`Next reflection: ~${observationPoolTokens.toLocaleString()} / ${refThreshold.toLocaleString()} tokens (${refPct}%)`,
|
|
69
|
+
` → if observations exceed ${refThreshold.toLocaleString()} tokens when compaction runs, reflections are`,
|
|
70
|
+
` distilled from them and redundant observations are pruned away`,
|
|
71
|
+
]
|
|
72
|
+
: [
|
|
73
|
+
"── Activity ──",
|
|
74
|
+
`Next observation: ~${sinceBound.toLocaleString()} / ${obsThreshold.toLocaleString()} tokens (${obsPct}%)`,
|
|
75
|
+
` → at ${obsThreshold.toLocaleString()} tokens, recent conversation is compressed into new observations`,
|
|
76
|
+
`Next compaction: ~${sinceCompaction.toLocaleString()} / ${compThreshold.toLocaleString()} tokens (${compPct}%)`,
|
|
77
|
+
` → at ${compThreshold.toLocaleString()} tokens, raw history is replaced by the updated reflections and`,
|
|
78
|
+
` observations, keeping only the last ${keepRecentTokens.toLocaleString()} tokens of conversation verbatim`,
|
|
79
|
+
`Next reflection: ~${observationPoolTokens.toLocaleString()} / ${refThreshold.toLocaleString()} tokens (${refPct}%)`,
|
|
80
|
+
` → if observations exceed ${refThreshold.toLocaleString()} tokens when compaction runs, reflections are`,
|
|
81
|
+
` distilled from them and redundant observations are pruned away`,
|
|
82
|
+
];
|
|
83
|
+
|
|
51
84
|
const lines = [
|
|
85
|
+
...passiveLines,
|
|
52
86
|
"── Memory ──",
|
|
53
87
|
`Reflections: ~${committedRefsTokens.toLocaleString()} tokens (${committedRefsCount} ${refLabel}) — durable insights`,
|
|
54
88
|
`Observations:`,
|
|
@@ -56,15 +90,7 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
|
|
|
56
90
|
` pending ~${pendingObsTokens.toLocaleString()} tokens (${pendingObsCount} ${pObsLabel}) — waiting for next compaction`,
|
|
57
91
|
` relevance ${formatRelevanceHistogram(relevanceHistogram)}`,
|
|
58
92
|
"",
|
|
59
|
-
|
|
60
|
-
`Next observation: ~${sinceBound.toLocaleString()} / ${obsThreshold.toLocaleString()} tokens (${obsPct}%)`,
|
|
61
|
-
` → at ${obsThreshold.toLocaleString()} tokens, recent conversation is compressed into new observations`,
|
|
62
|
-
`Next compaction: ~${sinceCompaction.toLocaleString()} / ${compThreshold.toLocaleString()} tokens (${compPct}%)`,
|
|
63
|
-
` → at ${compThreshold.toLocaleString()} tokens, raw history is replaced by the updated reflections and`,
|
|
64
|
-
` observations, keeping only the last ${keepRecentTokens.toLocaleString()} tokens of conversation verbatim`,
|
|
65
|
-
`Next reflection: ~${observationPoolTokens.toLocaleString()} / ${refThreshold.toLocaleString()} tokens (${refPct}%)`,
|
|
66
|
-
` → if observations exceed ${refThreshold.toLocaleString()} tokens when compaction runs, reflections are`,
|
|
67
|
-
` distilled from them and redundant observations are pruned away`,
|
|
93
|
+
...activityLines,
|
|
68
94
|
];
|
|
69
95
|
|
|
70
96
|
if (runtime.observerInFlight || runtime.compactInFlight) {
|
package/src/commands/view.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { getMemoryState } from "../branch.js";
|
|
|
3
3
|
import { countByRelevance, formatRelevanceHistogram } from "../relevance.js";
|
|
4
4
|
import type { Runtime } from "../runtime.js";
|
|
5
5
|
import { estimateStringTokens } from "../tokens.js";
|
|
6
|
-
import type
|
|
6
|
+
import { reflectionContent, reflectionToPromptLine, type MemoryReflection, type ObservationRecord } from "../types.js";
|
|
7
7
|
|
|
8
8
|
export function registerViewCommand(pi: ExtensionAPI, runtime: Runtime): void {
|
|
9
9
|
pi.registerCommand("om-view", {
|
|
@@ -12,9 +12,10 @@ export function registerViewCommand(pi: ExtensionAPI, runtime: Runtime): void {
|
|
|
12
12
|
runtime.ensureConfig(ctx.cwd);
|
|
13
13
|
const entries = ctx.sessionManager.getBranch() as Parameters<typeof getMemoryState>[0];
|
|
14
14
|
const { reflections: committedRefs, committedObs, pendingObs } = getMemoryState(entries);
|
|
15
|
+
const committedRefItems = committedRefs as MemoryReflection[];
|
|
15
16
|
|
|
16
|
-
const committedRefTokens =
|
|
17
|
-
const committedRefCount =
|
|
17
|
+
const committedRefTokens = committedRefItems.reduce((s, r) => s + estimateStringTokens(reflectionContent(r)), 0);
|
|
18
|
+
const committedRefCount = committedRefItems.length;
|
|
18
19
|
|
|
19
20
|
const committedObsTokens = committedObs.reduce((s, r) => s + estimateStringTokens(r.content), 0);
|
|
20
21
|
const committedObsCount = committedObs.length;
|
|
@@ -44,8 +45,8 @@ export function registerViewCommand(pi: ExtensionAPI, runtime: Runtime): void {
|
|
|
44
45
|
sections.push(
|
|
45
46
|
`── Reflections (${committedRefCount} ${plural(committedRefCount, "entry", "entries")}, ~${committedRefTokens.toLocaleString()} tokens) ──`,
|
|
46
47
|
);
|
|
47
|
-
if (
|
|
48
|
-
sections.push(
|
|
48
|
+
if (committedRefItems.length > 0) {
|
|
49
|
+
sections.push(committedRefItems.map(reflectionToPromptLine).join("\n\n"));
|
|
49
50
|
} else {
|
|
50
51
|
sections.push("(none)");
|
|
51
52
|
}
|