pi-blackhole 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -1
- package/example-config.json +14 -3
- package/package.json +1 -1
- package/src/commands/memory.ts +13 -5
- package/src/commands/pi-vcc.ts +21 -6
- package/src/core/unified-config.ts +29 -4
- package/src/om/agents/dropper/agent.ts +128 -10
- package/src/om/agents/dropper/coverage.ts +128 -0
- package/src/om/agents/dropper/prompts.ts +12 -12
- package/src/om/agents/observer/prompts.ts +1 -1
- package/src/om/agents/reflector/prompts.ts +8 -4
- package/src/om/consolidation.ts +160 -28
- package/src/om/ledger/projection.ts +13 -1
- package/src/om/ledger/render-summary.ts +51 -0
- package/src/om/pending.ts +34 -2
package/README.md
CHANGED
|
@@ -5,8 +5,14 @@ Algorithmic compaction + session-aware observational memory for [Pi](https://git
|
|
|
5
5
|
Combines [pi-vcc](https://github.com/sting8k/pi-vcc) and [pi-observational-memory](https://github.com/elpapi42/pi-observational-memory) with unified configuration, per-worker model fallback chains, persisted cooldowns, and a manual flush mode.
|
|
6
6
|
|
|
7
7
|
> This is a frankenmerge. I liked both extensions but they were not compatible - observational memory hooked into Pi's default compaction and prevented pi-vcc from working. So I merged them, made them share the same hook and output, and added the things both were missing: fallback chains, cooldowns, and a toggle between them.
|
|
8
|
+
> Please also see the [`CHANGELOG.md`](CHANGELOG.md)
|
|
9
|
+
|
|
10
|
+
### Lockstep with upstreams
|
|
11
|
+
|
|
12
|
+
pi-blackhole tracks both upstream repositories via a [lockstep audit system](https://github.com/k0valik/pi-blackhole/tree/lockstep/2026-05-27/.pi/skills/lockstep) that classifies every new upstream commit as safe-to-port, modified (needs review), rewritten (skip), or orphan (needs mapping). The goal is to lift bugfixes, prompts improvements, and compatible features without breaking existing users or rolling back intentional divergences. Ported changes are reviewed per-commit with human approval — nothing is blindly merged. See [SKILL.md](.pi/skills/lockstep/SKILL.md) for the full workflow. An example execution (including rationale for skipped changes) is documented in [PR #8](https://github.com/k0valik/pi-blackhole/pull/8).
|
|
13
|
+
|
|
14
|
+
#### For easy setup pass the [`llms.txt`](llms.txt) llms.txt to your agent and it will guide you through the config without needing to read all the docs if you're as lazy as me.
|
|
8
15
|
|
|
9
|
-
#### For easy setup pass the [`llms.txt`](llms.txt) llms.txt to your agent and it will guide you through the config without needing to read all the docs if you're as lazy as me
|
|
10
16
|
|
|
11
17
|
# Demo
|
|
12
18
|
|
|
@@ -131,6 +137,7 @@ The tradeoff is simplicity vs cleanliness:
|
|
|
131
137
|
|---|---|---|
|
|
132
138
|
| Workers run? | Yes | Yes |
|
|
133
139
|
| Observations go to | Conversation markers (invisible in TUI) | Disk (`pending.json`) |
|
|
140
|
+
| Observations accumulate across runs | Branch markers (replaced each cycle) | Pending batches — `/memory` shows pending counts |
|
|
134
141
|
| Auto-compact on `agent_end` | Yes | No |
|
|
135
142
|
| `/blackhole` | Optional — use it whenever you want | Required to flush + compact |
|
|
136
143
|
| Conversation history | OM marker entries between turns (they exist but do not clutter the display) | Clean — nothing between turns |
|
|
@@ -198,6 +205,7 @@ Quick start with custom models:
|
|
|
198
205
|
| `reflectAfterTokens` | `20000` | Token cadence for reflector and dropper |
|
|
199
206
|
| `compactAfterTokens` | `81000` | Auto-compaction threshold |
|
|
200
207
|
| `observerChunkMaxTokens` | `40000` | Max observer input per run (newest-first) |
|
|
208
|
+
| `observerPreambleMaxTokens` | `0` (auto) | Max preamble tokens in observer prompt for `noAutoCompact` mode (auto = 30% of chunk) |
|
|
201
209
|
| `observationsPoolMaxTokens` | `20000` | Max active observation pool before dropper prunes |
|
|
202
210
|
| `reflectorInputMaxTokens` | `80000` | Max reflector input budget |
|
|
203
211
|
| `dropperInputMaxTokens` | `80000` | Max dropper input budget |
|
|
@@ -220,6 +228,7 @@ Paste the appropriate block into your config to match your model's context size.
|
|
|
220
228
|
"reflectAfterTokens": 10000,
|
|
221
229
|
"compactAfterTokens": 30000,
|
|
222
230
|
"observerChunkMaxTokens": 15000,
|
|
231
|
+
"observerPreambleMaxTokens": 0,
|
|
223
232
|
"observationsPoolMaxTokens": 8000,
|
|
224
233
|
"reflectorInputMaxTokens": 30000,
|
|
225
234
|
"dropperInputMaxTokens": 30000
|
|
@@ -236,6 +245,7 @@ Our built-in defaults already target this tier. If you reset your config, these
|
|
|
236
245
|
"reflectAfterTokens": 20000,
|
|
237
246
|
"compactAfterTokens": 81000,
|
|
238
247
|
"observerChunkMaxTokens": 40000,
|
|
248
|
+
"observerPreambleMaxTokens": 0,
|
|
239
249
|
"observationsPoolMaxTokens": 20000,
|
|
240
250
|
"reflectorInputMaxTokens": 80000,
|
|
241
251
|
"dropperInputMaxTokens": 80000
|
|
@@ -250,6 +260,7 @@ Our built-in defaults already target this tier. If you reset your config, these
|
|
|
250
260
|
"reflectAfterTokens": 40000,
|
|
251
261
|
"compactAfterTokens": 180000,
|
|
252
262
|
"observerChunkMaxTokens": 80000,
|
|
263
|
+
"observerPreambleMaxTokens": 0,
|
|
253
264
|
"observationsPoolMaxTokens": 40000,
|
|
254
265
|
"reflectorInputMaxTokens": 160000,
|
|
255
266
|
"dropperInputMaxTokens": 160000
|
package/example-config.json
CHANGED
|
@@ -86,13 +86,14 @@
|
|
|
86
86
|
}
|
|
87
87
|
],
|
|
88
88
|
|
|
89
|
-
"observeAfterTokens":
|
|
90
|
-
"reflectAfterTokens":
|
|
89
|
+
"observeAfterTokens": 15000,
|
|
90
|
+
"reflectAfterTokens": 25000,
|
|
91
91
|
"compactAfterTokens": 81000,
|
|
92
92
|
"observationsPoolMaxTokens": 20000,
|
|
93
93
|
"reflectorInputMaxTokens": 80000,
|
|
94
94
|
"dropperInputMaxTokens": 80000,
|
|
95
95
|
"observerChunkMaxTokens": 40000,
|
|
96
|
+
"observerPreambleMaxTokens": 0,
|
|
96
97
|
"agentMaxTurns": 16,
|
|
97
98
|
|
|
98
99
|
"passive": false,
|
|
@@ -110,6 +111,16 @@
|
|
|
110
111
|
"After all candidates exhausted, the stage aborts. 30s retry gate prevents spam.",
|
|
111
112
|
"Cooldowns persist in ~/.pi/agent/pi-blackhole/pi-blackhole-cooldown.json (survives pi restarts).",
|
|
112
113
|
"Valid thinking values: off, minimal, low, medium, high, xhigh.",
|
|
113
|
-
"Env override for passive mode: PI_BLACKHOLE_PASSIVE=true."
|
|
114
|
+
"Env override for passive mode: PI_BLACKHOLE_PASSIVE=true.",
|
|
115
|
+
"",
|
|
116
|
+
"noAutoCompact mode accumulates observationBatches/reflectionBatches in",
|
|
117
|
+
"pending.json across pipeline runs. The observer preamble (CURRENT OBSERVATIONS)",
|
|
118
|
+
"is capped to observerPreambleMaxTokens (default 0 = auto 30% of observerChunkMaxTokens).",
|
|
119
|
+
"This is a soft safety valve — it only trims when accumulated observations exceed",
|
|
120
|
+
"the budget. High-relevance observations are always kept. Reflections are never trimmed.",
|
|
121
|
+
"",
|
|
122
|
+
"agentMaxTurns controls how many agent-loop turns each worker gets per chunk.",
|
|
123
|
+
"Lower-context models can reduce this (e.g., 8) so the observer doesn't waste",
|
|
124
|
+
"turns trying to extract more than it can reliably handle per run."
|
|
114
125
|
]
|
|
115
126
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-blackhole",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Unified compaction + observational memory extension for Pi — compresses conversation context while preserving durable observations and reflections",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"main": "index.ts",
|
package/src/commands/memory.ts
CHANGED
|
@@ -164,11 +164,12 @@ export function registerMemoryCommand(pi: ExtensionAPI, runtime: Runtime): void
|
|
|
164
164
|
observationLine,
|
|
165
165
|
reflectionLine,
|
|
166
166
|
"",
|
|
167
|
-
"──
|
|
168
|
-
|
|
169
|
-
`
|
|
170
|
-
`
|
|
171
|
-
`
|
|
167
|
+
"── Pipeline ──",
|
|
168
|
+
"Transcript accumulated since last run. Triggers when exceeding threshold.",
|
|
169
|
+
`Observer: ~${obsProgress.toLocaleString()} tokens (triggers at ${runtime.config.observeAfterTokens.toLocaleString()})`,
|
|
170
|
+
`Reflector: ~${reflectionProgress.toLocaleString()} tokens (triggers at ${runtime.config.reflectAfterTokens.toLocaleString()})`,
|
|
171
|
+
`Dropper: ~${dropProgress.toLocaleString()} tokens (triggers at ${runtime.config.reflectAfterTokens.toLocaleString()})`,
|
|
172
|
+
`Compaction: ~${compactionProgress.toLocaleString()} tokens` + (runtime.config.noAutoCompact ? " [auto-disabled]" : ` (triggers at ${runtime.config.compactAfterTokens.toLocaleString()})`),
|
|
172
173
|
`Obs pool: ~${visibleObservationTokens.toLocaleString()} / ${runtime.config.observationsPoolMaxTokens.toLocaleString()} tokens (${pct(visibleObservationTokens, runtime.config.observationsPoolMaxTokens)}%)`,
|
|
173
174
|
`Reflect pool: ~${visibleReflectionTokens.toLocaleString()} tokens`,
|
|
174
175
|
];
|
|
@@ -184,6 +185,13 @@ export function registerMemoryCommand(pi: ExtensionAPI, runtime: Runtime): void
|
|
|
184
185
|
if (hasObs) lines.push("Observation: waiting in pending.json");
|
|
185
186
|
if (hasRef) lines.push("Reflection: waiting in pending.json");
|
|
186
187
|
if (hasDrop) lines.push("Dropper: waiting in pending.json");
|
|
188
|
+
const preambleCap = runtime.config.observerPreambleMaxTokens > 0
|
|
189
|
+
? runtime.config.observerPreambleMaxTokens
|
|
190
|
+
: Math.round(runtime.config.observerChunkMaxTokens * 0.3);
|
|
191
|
+
const pctNote = runtime.config.observerPreambleMaxTokens > 0
|
|
192
|
+
? ""
|
|
193
|
+
: ` (30% of ${runtime.config.observerChunkMaxTokens.toLocaleString()} chunk)`;
|
|
194
|
+
lines.push(`Preamble cap: ${preambleCap.toLocaleString()} tokens for observations${pctNote}`);
|
|
187
195
|
lines.push("Run /blackhole to flush and compact.");
|
|
188
196
|
}
|
|
189
197
|
}
|
package/src/commands/pi-vcc.ts
CHANGED
|
@@ -55,14 +55,29 @@ export const registerPiVccCommand = (pi: ExtensionAPI, runtime: Runtime) => {
|
|
|
55
55
|
// before compacting so the summary includes accumulated memory.
|
|
56
56
|
if (runtime.config.noAutoCompact && hasPendingData(sessionId)) {
|
|
57
57
|
const pending = readPendingState(sessionId);
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
// Write all accumulated observation batches (or latest single batch
|
|
59
|
+
// as fallback for legacy pending.json without batch arrays).
|
|
60
|
+
const obsBatches = pending.observationBatches?.length
|
|
61
|
+
? pending.observationBatches
|
|
62
|
+
: (pending.observation ? [pending.observation] : []);
|
|
63
|
+
for (const batch of obsBatches) {
|
|
64
|
+
pi.appendEntry(OM_OBSERVATIONS_RECORDED, batch.data);
|
|
60
65
|
}
|
|
61
|
-
|
|
62
|
-
|
|
66
|
+
// Write all accumulated reflection batches (or latest single batch
|
|
67
|
+
// as fallback for legacy pending.json without batch arrays).
|
|
68
|
+
const reflBatches = pending.reflectionBatches?.length
|
|
69
|
+
? pending.reflectionBatches
|
|
70
|
+
: (pending.reflection ? [pending.reflection] : []);
|
|
71
|
+
for (const batch of reflBatches) {
|
|
72
|
+
pi.appendEntry(OM_REFLECTIONS_RECORDED, batch.data);
|
|
63
73
|
}
|
|
64
|
-
|
|
65
|
-
|
|
74
|
+
// Write all accumulated dropper batches (or latest single batch
|
|
75
|
+
// as fallback for legacy pending.json without batch arrays).
|
|
76
|
+
const dropBatches = pending.droppedBatches?.length
|
|
77
|
+
? pending.droppedBatches
|
|
78
|
+
: (pending.dropped ? [pending.dropped] : []);
|
|
79
|
+
for (const batch of dropBatches) {
|
|
80
|
+
pi.appendEntry(OM_OBSERVATIONS_DROPPED, batch.data);
|
|
66
81
|
}
|
|
67
82
|
clearPendingState(sessionId);
|
|
68
83
|
ctx.ui.notify("Observational memory: pending entries flushed", "info");
|
|
@@ -44,12 +44,27 @@ export interface UnifiedConfig {
|
|
|
44
44
|
compactAfterTokens: number;
|
|
45
45
|
/** Observation pool token pressure for full fold. */
|
|
46
46
|
observationsPoolMaxTokens: number;
|
|
47
|
+
/** Target token budget for the observation pool (dropper aims here).
|
|
48
|
+
* Optional; defaults to half of observationsPoolMaxTokens when unset.
|
|
49
|
+
* Must be less than observationsPoolMaxTokens.
|
|
50
|
+
*
|
|
51
|
+
* NOTE: Ported from upstream as forward-compat (no-op in our pool algorithm).
|
|
52
|
+
* Upstream renamed budgetTokens→targetTokens (52b5844) and uses this
|
|
53
|
+
* for their tokensOverTarget / avgTokensPerObservation drop calculation.
|
|
54
|
+
* We keep our ratio-based urgency algorithm; this knob exists so future
|
|
55
|
+
* lockstep iterations don't diverge on the config shape. */
|
|
56
|
+
observationsPoolTargetTokens: number;
|
|
47
57
|
/** Max prompt tokens for reflector model input (rolling window cap). */
|
|
48
58
|
reflectorInputMaxTokens: number;
|
|
49
59
|
/** Max prompt tokens for dropper model input (rolling window cap). */
|
|
50
60
|
dropperInputMaxTokens: number;
|
|
51
61
|
/** Max source entries tokens sent to observer per chunk. */
|
|
52
62
|
observerChunkMaxTokens: number;
|
|
63
|
+
/** Max preamble tokens (CURRENT REFLECTIONS / OBSERVATIONS) in the observer prompt.
|
|
64
|
+
* Default 0 means auto-compute from observerChunkMaxTokens (30%). Only applied in
|
|
65
|
+
* noAutoCompact mode where accumulated batch history can grow unbounded.
|
|
66
|
+
* Set to an explicit value to override the auto-computed budget. */
|
|
67
|
+
observerPreambleMaxTokens: number;
|
|
53
68
|
/** Shared turn cap for background memory agents. */
|
|
54
69
|
agentMaxTurns: number;
|
|
55
70
|
|
|
@@ -87,13 +102,15 @@ export const DEFAULTS: UnifiedConfig = {
|
|
|
87
102
|
overrideDefaultCompaction: false,
|
|
88
103
|
debug: false,
|
|
89
104
|
|
|
90
|
-
observeAfterTokens:
|
|
91
|
-
reflectAfterTokens:
|
|
105
|
+
observeAfterTokens: 15_000,
|
|
106
|
+
reflectAfterTokens: 25_000,
|
|
92
107
|
compactAfterTokens: 81_000,
|
|
93
108
|
observationsPoolMaxTokens: 20_000,
|
|
109
|
+
observationsPoolTargetTokens: 10_000,
|
|
94
110
|
reflectorInputMaxTokens: 80_000,
|
|
95
111
|
dropperInputMaxTokens: 80_000,
|
|
96
112
|
observerChunkMaxTokens: 40_000,
|
|
113
|
+
observerPreambleMaxTokens: 0,
|
|
97
114
|
agentMaxTurns: 16,
|
|
98
115
|
|
|
99
116
|
noAutoCompact: false,
|
|
@@ -154,7 +171,7 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
|
|
|
154
171
|
if (typeof raw.debugLog === "boolean") c.debugLog = raw.debugLog;
|
|
155
172
|
|
|
156
173
|
// Positive integers
|
|
157
|
-
const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "agentMaxTurns"] as const;
|
|
174
|
+
const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "observationsPoolTargetTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "observerPreambleMaxTokens", "agentMaxTurns"] as const;
|
|
158
175
|
for (const k of numKeys) {
|
|
159
176
|
const v = positiveInt(raw[k]);
|
|
160
177
|
if (v !== undefined) (c as Record<string, unknown>)[k] = v;
|
|
@@ -232,7 +249,15 @@ export function loadUnifiedConfig(cwd: string): UnifiedConfig {
|
|
|
232
249
|
else if (["0", "false", "no", "off"].includes(v)) parsed.passive = false;
|
|
233
250
|
}
|
|
234
251
|
|
|
235
|
-
|
|
252
|
+
// Merge defaults then override
|
|
253
|
+
const merged = { ...DEFAULTS, ...parsed };
|
|
254
|
+
|
|
255
|
+
// Derive observationsPoolTargetTokens if unset or invalid (must be < max)
|
|
256
|
+
if (merged.observationsPoolTargetTokens >= merged.observationsPoolMaxTokens) {
|
|
257
|
+
merged.observationsPoolTargetTokens = Math.floor(merged.observationsPoolMaxTokens / 2);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
return merged;
|
|
236
261
|
}
|
|
237
262
|
|
|
238
263
|
/**
|
|
@@ -9,9 +9,18 @@ import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } fr
|
|
|
9
9
|
import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
|
|
10
10
|
import { Type } from "@earendil-works/pi-ai";
|
|
11
11
|
import type { Static } from "typebox";
|
|
12
|
+
import { debugLog } from "../../debug-log.js";
|
|
12
13
|
import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
|
|
13
|
-
import {
|
|
14
|
+
import { reflectionToSummaryLine, type Observation, type Reflection } from "../../ledger/index.js";
|
|
14
15
|
import { DROPPER_SYSTEM } from "./prompts.js";
|
|
16
|
+
import {
|
|
17
|
+
REFLECTION_COVERAGE_DROP_RANK,
|
|
18
|
+
coverageTierForObservation,
|
|
19
|
+
observationToDropperLine,
|
|
20
|
+
reflectionCoverageMap,
|
|
21
|
+
summarizeCoverageByRelevance,
|
|
22
|
+
summarizeCoverageByRelevanceForIds,
|
|
23
|
+
} from "./coverage.js";
|
|
15
24
|
|
|
16
25
|
interface RunDropperArgs {
|
|
17
26
|
model: Model<any>;
|
|
@@ -81,6 +90,13 @@ export function maxDropCountForPool(observations: readonly Observation[], observ
|
|
|
81
90
|
return Math.max(1, Math.floor(droppableCount * dropRatio));
|
|
82
91
|
}
|
|
83
92
|
|
|
93
|
+
function relevanceCounts(observations: readonly Observation[]): Record<Observation["relevance"], number> {
|
|
94
|
+
return observations.reduce<Record<Observation["relevance"], number>>((counts, observation) => {
|
|
95
|
+
if (observation.relevance in counts) counts[observation.relevance]++;
|
|
96
|
+
return counts;
|
|
97
|
+
}, { low: 0, medium: 0, high: 0, critical: 0 });
|
|
98
|
+
}
|
|
99
|
+
|
|
84
100
|
export function normalizeDropObservationIds(
|
|
85
101
|
ids: readonly string[] | undefined,
|
|
86
102
|
observations: readonly Observation[],
|
|
@@ -92,7 +108,6 @@ export function normalizeDropObservationIds(
|
|
|
92
108
|
for (const id of ids) {
|
|
93
109
|
const observation = allowed.get(id);
|
|
94
110
|
if (!observation) continue;
|
|
95
|
-
if (observation.relevance === "critical") continue;
|
|
96
111
|
if (seen.has(id)) continue;
|
|
97
112
|
seen.add(id);
|
|
98
113
|
result.push(id);
|
|
@@ -100,14 +115,21 @@ export function normalizeDropObservationIds(
|
|
|
100
115
|
return result.length > 0 ? result : undefined;
|
|
101
116
|
}
|
|
102
117
|
|
|
118
|
+
function timestampRank(timestamp: string): number {
|
|
119
|
+
const parsed = Date.parse(timestamp);
|
|
120
|
+
return Number.isFinite(parsed) ? parsed : Number.POSITIVE_INFINITY;
|
|
121
|
+
}
|
|
122
|
+
|
|
103
123
|
export function selectDropCandidates(
|
|
104
124
|
ids: readonly string[],
|
|
105
125
|
observations: readonly Observation[],
|
|
106
126
|
maxDrops: number,
|
|
127
|
+
reflections: readonly Reflection[] = [],
|
|
107
128
|
): string[] {
|
|
108
129
|
if (maxDrops <= 0 || ids.length === 0) return [];
|
|
109
130
|
|
|
110
131
|
const byId = new Map(observations.map((observation) => [observation.id, observation]));
|
|
132
|
+
const coverageById = reflectionCoverageMap(observations, reflections);
|
|
111
133
|
const firstProposalIndex = new Map<string, number>();
|
|
112
134
|
for (let i = 0; i < ids.length; i++) {
|
|
113
135
|
const id = ids[i];
|
|
@@ -117,11 +139,16 @@ export function selectDropCandidates(
|
|
|
117
139
|
return Array.from(firstProposalIndex.entries())
|
|
118
140
|
.map(([id, index]) => ({ id, index, observation: byId.get(id) }))
|
|
119
141
|
.filter((candidate): candidate is { id: string; index: number; observation: Observation } =>
|
|
120
|
-
candidate.observation !== undefined
|
|
142
|
+
candidate.observation !== undefined
|
|
121
143
|
)
|
|
122
144
|
.sort((a, b) => {
|
|
145
|
+
const coverageDelta = REFLECTION_COVERAGE_DROP_RANK[coverageTierForObservation(a.observation, coverageById)]
|
|
146
|
+
- REFLECTION_COVERAGE_DROP_RANK[coverageTierForObservation(b.observation, coverageById)];
|
|
123
147
|
const relevanceDelta = RELEVANCE_DROP_RANK[a.observation.relevance] - RELEVANCE_DROP_RANK[b.observation.relevance];
|
|
124
|
-
|
|
148
|
+
const aAge = timestampRank(a.observation.timestamp);
|
|
149
|
+
const bAge = timestampRank(b.observation.timestamp);
|
|
150
|
+
const ageDelta = aAge === bAge ? 0 : aAge - bAge;
|
|
151
|
+
return coverageDelta || relevanceDelta || ageDelta || a.index - b.index;
|
|
125
152
|
})
|
|
126
153
|
.slice(0, maxDrops)
|
|
127
154
|
.map((candidate) => candidate.id);
|
|
@@ -135,10 +162,42 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
|
|
|
135
162
|
const fullness = observationPoolFullness(observationTokens, budgetTokens);
|
|
136
163
|
const urgency = dropUrgencyForFullness(fullness);
|
|
137
164
|
const maxDropsAllowed = maxDropCountForPool(observations, observationTokens, budgetTokens);
|
|
138
|
-
|
|
165
|
+
const coverageById = reflectionCoverageMap(observations, reflections);
|
|
166
|
+
const coverageSummaryByRelevance = summarizeCoverageByRelevance(observations, coverageById);
|
|
167
|
+
debugLog("dropper.agent_start", {
|
|
168
|
+
activeObservationCount: observations.length,
|
|
169
|
+
reflectionCount: reflections.length,
|
|
170
|
+
observationTokens,
|
|
171
|
+
budgetTokens,
|
|
172
|
+
fullness,
|
|
173
|
+
urgency,
|
|
174
|
+
maxDropsAllowed,
|
|
175
|
+
relevanceCounts: relevanceCounts(observations),
|
|
176
|
+
coverageSummaryByRelevance,
|
|
177
|
+
});
|
|
178
|
+
if (maxDropsAllowed <= 0) {
|
|
179
|
+
debugLog("dropper.result", {
|
|
180
|
+
reason: "not_over_target",
|
|
181
|
+
toolCallCount: 0,
|
|
182
|
+
rawRequestedIdsCount: 0,
|
|
183
|
+
acceptedCandidateCount: 0,
|
|
184
|
+
selectedDropsCount: 0,
|
|
185
|
+
selectedDropTokens: 0,
|
|
186
|
+
selectedCoverageSummaryByRelevance: summarizeCoverageByRelevanceForIds([], observations, coverageById),
|
|
187
|
+
maxDropsAllowed,
|
|
188
|
+
});
|
|
189
|
+
return undefined;
|
|
190
|
+
}
|
|
139
191
|
|
|
140
192
|
const proposedDropIds: string[] = [];
|
|
141
193
|
const proposed = new Set<string>();
|
|
194
|
+
const allowed = new Map(observations.map((observation) => [observation.id, observation]));
|
|
195
|
+
let toolCallCount = 0;
|
|
196
|
+
let rawRequestedIdsCount = 0;
|
|
197
|
+
let missingIdsCount = 0;
|
|
198
|
+
let criticalCandidateIdsCount = 0;
|
|
199
|
+
let duplicateInRequestCount = 0;
|
|
200
|
+
let duplicateInRunCount = 0;
|
|
142
201
|
|
|
143
202
|
const dropObservations: AgentTool<typeof DropObservationsSchema> = {
|
|
144
203
|
name: "drop_observations",
|
|
@@ -146,14 +205,51 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
|
|
|
146
205
|
description: "Propose active observation ids that are safe to remove from compacted memory.",
|
|
147
206
|
parameters: DropObservationsSchema,
|
|
148
207
|
execute: async (_id, params: DropObservationsArgs) => {
|
|
149
|
-
|
|
208
|
+
toolCallCount++;
|
|
209
|
+
rawRequestedIdsCount += params.ids.length;
|
|
210
|
+
const seenInRequest = new Set<string>();
|
|
150
211
|
let added = 0;
|
|
151
|
-
|
|
152
|
-
|
|
212
|
+
let requestMissingIds = 0;
|
|
213
|
+
let requestCriticalCandidateIds = 0;
|
|
214
|
+
let requestDuplicateIds = 0;
|
|
215
|
+
let requestDuplicateInRunIds = 0;
|
|
216
|
+
for (const id of params.ids) {
|
|
217
|
+
const observation = allowed.get(id);
|
|
218
|
+
if (!observation) {
|
|
219
|
+
missingIdsCount++;
|
|
220
|
+
requestMissingIds++;
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
if (seenInRequest.has(id)) {
|
|
224
|
+
duplicateInRequestCount++;
|
|
225
|
+
requestDuplicateIds++;
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
seenInRequest.add(id);
|
|
229
|
+
if (proposed.has(id)) {
|
|
230
|
+
duplicateInRunCount++;
|
|
231
|
+
requestDuplicateInRunIds++;
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
153
234
|
proposed.add(id);
|
|
154
235
|
proposedDropIds.push(id);
|
|
236
|
+
if (observation.relevance === "critical") {
|
|
237
|
+
criticalCandidateIdsCount++;
|
|
238
|
+
requestCriticalCandidateIds++;
|
|
239
|
+
}
|
|
155
240
|
added++;
|
|
156
241
|
}
|
|
242
|
+
debugLog("dropper.tool_call", {
|
|
243
|
+
toolCallCount,
|
|
244
|
+
rawRequestedIdsCount: params.ids.length,
|
|
245
|
+
acceptedIdsCount: added,
|
|
246
|
+
missingIdsCount: requestMissingIds,
|
|
247
|
+
criticalCandidateIdsCount: requestCriticalCandidateIds,
|
|
248
|
+
duplicateInRequestCount: requestDuplicateIds,
|
|
249
|
+
duplicateInRunCount: requestDuplicateInRunIds,
|
|
250
|
+
totalCandidates: proposedDropIds.length,
|
|
251
|
+
maxDropsAllowed,
|
|
252
|
+
});
|
|
157
253
|
return {
|
|
158
254
|
content: [{ type: "text", text: `Queued ${added} drop candidate${added === 1 ? "" : "s"}. Candidates this run: ${proposedDropIds.length}. Maximum drops allowed: ${maxDropsAllowed}.` }],
|
|
159
255
|
details: { added, totalCandidates: proposedDropIds.length, maxDropsAllowed },
|
|
@@ -166,7 +262,7 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
|
|
|
166
262
|
? `EXISTING ACTIVE OBSERVATIONS (for context only — these are NOT candidates for dropping):\n${args.existingObservationsSummary}\n\n`
|
|
167
263
|
: '';
|
|
168
264
|
|
|
169
|
-
const userText = `CURRENT REFLECTIONS:\n${joinOrEmpty(reflections.map(reflectionToSummaryLine))}\n\n${existingObservationsContext}NEW OBSERVATIONS TO EVALUATE FOR DROPPING:\n${joinOrEmpty(observations.map(
|
|
265
|
+
const userText = `CURRENT REFLECTIONS:\n${joinOrEmpty(reflections.map(reflectionToSummaryLine))}\n\n${existingObservationsContext}NEW OBSERVATIONS TO EVALUATE FOR DROPPING:\n${joinOrEmpty(observations.map((observation) => observationToDropperLine(observation, coverageTierForObservation(observation, coverageById))))}\n\nObservation pool pressure: ~${observationTokens.toLocaleString()} tokens; target budget: ~${budgetTokens.toLocaleString()} tokens; fullness: ~${fullnessPercent.toLocaleString()}%.\nDrop urgency: ${urgency}.\nMaximum drops allowed this run: ${maxDropsAllowed.toLocaleString()} observation${maxDropsAllowed === 1 ? "" : "s"}.\nThis maximum is a hard upper bound, not a target. Drop fewer or none if fewer observations are clearly safe.`;
|
|
170
266
|
const prompts: Message[] = [{ role: "user", content: [{ type: "text", text: userText }], timestamp: Date.now() }];
|
|
171
267
|
const context: AgentContext = { systemPrompt: DROPPER_SYSTEM, messages: [], tools: [dropObservations as AgentTool<any>] };
|
|
172
268
|
const reasoning = (model as { reasoning?: unknown }).reasoning;
|
|
@@ -199,6 +295,28 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
|
|
|
199
295
|
}
|
|
200
296
|
await stream.result();
|
|
201
297
|
if (agentError && proposedDropIds.length === 0) throw new Error(`Dropper API error: ${agentError}`);
|
|
202
|
-
const droppedIds = selectDropCandidates(proposedDropIds, observations, maxDropsAllowed);
|
|
298
|
+
const droppedIds = selectDropCandidates(proposedDropIds, observations, maxDropsAllowed, reflections);
|
|
299
|
+
const reason = droppedIds.length > 0
|
|
300
|
+
? "selected_nonempty"
|
|
301
|
+
: toolCallCount === 0
|
|
302
|
+
? "no_tool_call"
|
|
303
|
+
: proposedDropIds.length === 0
|
|
304
|
+
? "all_filtered"
|
|
305
|
+
: "selected_empty";
|
|
306
|
+
const selectedDropTokens = droppedIds.reduce((sum, id) => sum + (allowed.get(id)?.tokenCount ?? 0), 0);
|
|
307
|
+
debugLog("dropper.result", {
|
|
308
|
+
reason,
|
|
309
|
+
toolCallCount,
|
|
310
|
+
rawRequestedIdsCount,
|
|
311
|
+
missingIdsCount,
|
|
312
|
+
criticalCandidateIdsCount,
|
|
313
|
+
duplicateInRequestCount,
|
|
314
|
+
duplicateInRunCount,
|
|
315
|
+
acceptedCandidateCount: proposedDropIds.length,
|
|
316
|
+
selectedDropsCount: droppedIds.length,
|
|
317
|
+
selectedDropTokens,
|
|
318
|
+
selectedCoverageSummaryByRelevance: summarizeCoverageByRelevanceForIds(droppedIds, observations, coverageById),
|
|
319
|
+
maxDropsAllowed,
|
|
320
|
+
});
|
|
203
321
|
return droppedIds.length > 0 ? droppedIds : undefined;
|
|
204
322
|
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import type { Observation, Reflection } from "../../ledger/index.js";
|
|
2
|
+
|
|
3
|
+
export const REFLECTION_COVERAGE_TIERS = ["none", "partial", "strong"] as const;
|
|
4
|
+
export type ReflectionCoverageTier = typeof REFLECTION_COVERAGE_TIERS[number];
|
|
5
|
+
|
|
6
|
+
type Relevance = Observation["relevance"];
|
|
7
|
+
|
|
8
|
+
type CoverageBucket = Record<ReflectionCoverageTier, { count: number; tokens: number }>;
|
|
9
|
+
export type CoverageSummaryByRelevance = Record<Relevance, CoverageBucket>;
|
|
10
|
+
export type CoverageTransitionSummaryByRelevance = Record<Relevance, Record<string, { count: number; tokens: number }>>;
|
|
11
|
+
|
|
12
|
+
export const REFLECTION_COVERAGE_DROP_RANK: Record<ReflectionCoverageTier, number> = {
|
|
13
|
+
strong: 0,
|
|
14
|
+
partial: 1,
|
|
15
|
+
none: 2,
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export function reflectionSupportCounts(reflections: readonly Reflection[]): Map<string, number> {
|
|
19
|
+
const counts = new Map<string, number>();
|
|
20
|
+
for (const reflection of reflections) {
|
|
21
|
+
const uniqueIds = new Set(reflection.supportingObservationIds);
|
|
22
|
+
for (const id of uniqueIds) counts.set(id, (counts.get(id) ?? 0) + 1);
|
|
23
|
+
}
|
|
24
|
+
return counts;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function reflectionCoverageTierForCount(count: number): ReflectionCoverageTier {
|
|
28
|
+
if (count <= 0) return "none";
|
|
29
|
+
if (count === 1) return "partial";
|
|
30
|
+
return "strong";
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function reflectionCoverageMap(
|
|
34
|
+
observations: readonly Observation[],
|
|
35
|
+
reflections: readonly Reflection[],
|
|
36
|
+
): Map<string, ReflectionCoverageTier> {
|
|
37
|
+
const counts = reflectionSupportCounts(reflections);
|
|
38
|
+
return new Map(observations.map((observation) => [
|
|
39
|
+
observation.id,
|
|
40
|
+
reflectionCoverageTierForCount(counts.get(observation.id) ?? 0),
|
|
41
|
+
]));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function emptyCoverageBucket(): CoverageBucket {
|
|
45
|
+
return {
|
|
46
|
+
none: { count: 0, tokens: 0 },
|
|
47
|
+
partial: { count: 0, tokens: 0 },
|
|
48
|
+
strong: { count: 0, tokens: 0 },
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function emptyCoverageSummaryByRelevance(): CoverageSummaryByRelevance {
|
|
53
|
+
return {
|
|
54
|
+
low: emptyCoverageBucket(),
|
|
55
|
+
medium: emptyCoverageBucket(),
|
|
56
|
+
high: emptyCoverageBucket(),
|
|
57
|
+
critical: emptyCoverageBucket(),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function summarizeCoverageByRelevance(
|
|
62
|
+
observations: readonly Observation[],
|
|
63
|
+
coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
|
|
64
|
+
): CoverageSummaryByRelevance {
|
|
65
|
+
const summary = emptyCoverageSummaryByRelevance();
|
|
66
|
+
for (const observation of observations) {
|
|
67
|
+
const tier = coverageById.get(observation.id) ?? "none";
|
|
68
|
+
const bucket = summary[observation.relevance][tier];
|
|
69
|
+
bucket.count++;
|
|
70
|
+
bucket.tokens += observation.tokenCount;
|
|
71
|
+
}
|
|
72
|
+
return summary;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function summarizeCoverageByRelevanceForIds(
|
|
76
|
+
ids: readonly string[],
|
|
77
|
+
observations: readonly Observation[],
|
|
78
|
+
coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
|
|
79
|
+
): CoverageSummaryByRelevance {
|
|
80
|
+
const byId = new Map(observations.map((observation) => [observation.id, observation]));
|
|
81
|
+
const selected = ids.flatMap((id) => {
|
|
82
|
+
const observation = byId.get(id);
|
|
83
|
+
return observation ? [observation] : [];
|
|
84
|
+
});
|
|
85
|
+
return summarizeCoverageByRelevance(selected, coverageById);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function emptyCoverageTransitionSummaryByRelevance(): CoverageTransitionSummaryByRelevance {
|
|
89
|
+
return {
|
|
90
|
+
low: {},
|
|
91
|
+
medium: {},
|
|
92
|
+
high: {},
|
|
93
|
+
critical: {},
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function summarizeCoverageTransitionsByRelevance(
|
|
98
|
+
observations: readonly Observation[],
|
|
99
|
+
beforeCoverageById: ReadonlyMap<string, ReflectionCoverageTier>,
|
|
100
|
+
afterCoverageById: ReadonlyMap<string, ReflectionCoverageTier>,
|
|
101
|
+
): CoverageTransitionSummaryByRelevance {
|
|
102
|
+
const summary = emptyCoverageTransitionSummaryByRelevance();
|
|
103
|
+
for (const observation of observations) {
|
|
104
|
+
const before = beforeCoverageById.get(observation.id) ?? "none";
|
|
105
|
+
const after = afterCoverageById.get(observation.id) ?? "none";
|
|
106
|
+
if (before === after) continue;
|
|
107
|
+
const key = `${before}->${after}`;
|
|
108
|
+
const bucket = summary[observation.relevance][key] ?? { count: 0, tokens: 0 };
|
|
109
|
+
bucket.count++;
|
|
110
|
+
bucket.tokens += observation.tokenCount;
|
|
111
|
+
summary[observation.relevance][key] = bucket;
|
|
112
|
+
}
|
|
113
|
+
return summary;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function observationToDropperLine(
|
|
117
|
+
observation: Observation,
|
|
118
|
+
coverage: ReflectionCoverageTier,
|
|
119
|
+
): string {
|
|
120
|
+
return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] [coverage: ${coverage}] ${observation.content}`;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function coverageTierForObservation(
|
|
124
|
+
observation: Observation,
|
|
125
|
+
coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
|
|
126
|
+
): ReflectionCoverageTier {
|
|
127
|
+
return coverageById.get(observation.id) ?? "none";
|
|
128
|
+
}
|
|
@@ -6,30 +6,30 @@ Your job is to identify only the safest active observations to remove from compa
|
|
|
6
6
|
|
|
7
7
|
Active-memory framing. Dropping an observation removes it from active compacted memory; it does not erase the ledger history or source evidence. Still, future compressed context will no longer show the observation, so only drop it when its durable meaning is safely captured elsewhere or it is genuinely low-signal and carries no unique future value.
|
|
8
8
|
|
|
9
|
-
The user message includes
|
|
10
|
-
|
|
11
|
-
Urgency guidance:
|
|
12
|
-
- low urgency: only propose trivially safe drops, usually low-signal observations with no unique detail.
|
|
13
|
-
- medium urgency: perform conservative cleanup; prefer low observations and clearly redundant medium observations.
|
|
14
|
-
- high urgency: cleanup is more useful, but preservation rules do not weaken and load-bearing memory must still be kept.
|
|
9
|
+
The user message includes the active observation pool target and "Maximum drops allowed this run". The maximum is a hard upper bound sized to move the pool toward the target if every proposed drop is clearly safe. It is not a target. Do not try to fill it. Drop fewer or none when fewer observations are safely removable. When the active pool is far over target, make a thorough pass over safe candidates rather than stopping after a few obvious examples.
|
|
15
10
|
|
|
16
11
|
What to drop, in priority order:
|
|
17
12
|
- Redundant observations whose durable meaning is already captured by current reflections with equivalent fidelity.
|
|
18
13
|
- Superseded observations where a later observation clearly replaces the older state.
|
|
19
14
|
- Repeated routine tool acknowledgements or low-signal progress updates that do not carry decisions, constraints, exact errors, or user-specific facts.
|
|
20
|
-
- Older
|
|
15
|
+
- Older observations that no longer carry working context and are covered by a reflection or a newer observation.
|
|
16
|
+
|
|
17
|
+
Age-gradient rule. Recent observations carry working context the assistant may still need; older observations have usually been summarized elsewhere or are no longer load-bearing. Prefer older safe drops before newer working context, but age alone is not enough to drop important or uniquely load-bearing observations.
|
|
21
18
|
|
|
22
|
-
|
|
19
|
+
Reflection coverage guidance. Each observation line includes [coverage: none|partial|strong]. Coverage is evidence, not an automatic decision:
|
|
20
|
+
- none: no current reflection cites this observation id. Be cautious, especially for high or critical observations.
|
|
21
|
+
- partial: one current reflection cites this observation id. Compare the observation to the reflection before dropping.
|
|
22
|
+
- strong: two or more current reflections cite this observation id. This is stronger evidence that the durable meaning is preserved, but you must still keep uniquely load-bearing or uncertain observations.
|
|
23
23
|
|
|
24
|
-
Relevance guidance:
|
|
24
|
+
Relevance guidance. Relevance is importance/resistance, not an absolute keep/drop lock:
|
|
25
25
|
- low: consider first, but drop only when it carries no unique detail, decision, state, error, identifier, or user-specific fact.
|
|
26
26
|
- medium: drop when redundant with reflections or other observations, or when the work state is clearly obsolete.
|
|
27
27
|
- high: drop only when clearly superseded or already captured by a reflection with equivalent fidelity.
|
|
28
|
-
- critical:
|
|
28
|
+
- critical: highest importance and strongest resistance. Do not drop fresh or uniquely load-bearing critical observations. Critical observations may be dropped only with strong semantic evidence such as age plus partial/strong reflection coverage, supersession by newer memory, redundancy, or clear obsolescence.
|
|
29
29
|
|
|
30
|
-
User assertions and concrete completions
|
|
30
|
+
User assertions and concrete completions must be preserved unless a current reflection or newer observation preserves the exact assertion/completion and its important details with equivalent fidelity.
|
|
31
31
|
|
|
32
|
-
Preservation floor. Regardless of relevance label,
|
|
32
|
+
Preservation floor. Regardless of relevance label, budget pressure, coverage, or age, do not drop observations that uniquely carry any of the following:
|
|
33
33
|
- User preferences, constraints, corrections, or identity/role facts.
|
|
34
34
|
- Concrete completions that future runs must not redo.
|
|
35
35
|
- Named identifiers, file paths, function names, package names, tickets, commit SHAs, handles, or exact commands.
|
|
@@ -101,7 +101,7 @@ If a detail is non-obvious from the code or git history, it belongs in the obser
|
|
|
101
101
|
|
|
102
102
|
Relevance levels (pick one per observation; this field drives future dropping):
|
|
103
103
|
|
|
104
|
-
- critical: user assertions about identity, role, or persistent preferences; explicit corrections ("no, don't do X"); concrete completions that future runs MUST NOT redo. These are load-bearing and
|
|
104
|
+
- critical: user assertions about identity, role, or persistent preferences; explicit corrections ("no, don't do X"); concrete completions that future runs MUST NOT redo. These are highest-resistance, load-bearing observations and require the strongest evidence before leaving active memory. Why this matters: if a "critical" item is lost, the assistant may redo finished work, contradict a correction, or misrepresent who the user is.
|
|
105
105
|
- high: non-trivial technical decisions, architectural direction, unresolved blockers, key constraints. Worth keeping across many compactions.
|
|
106
106
|
- medium: task-level context that helps within the current work but isn't durable. The default when you are unsure between medium and high.
|
|
107
107
|
- low: routine tool-call acks, repetitive status updates, content trivially re-derivable from recent messages. The dropper will drop these first.
|
|
@@ -6,7 +6,8 @@ Your task is different from the observer's: you are not recording events, you ar
|
|
|
6
6
|
|
|
7
7
|
You receive:
|
|
8
8
|
- Current reflections: durable facts already crystallized.
|
|
9
|
-
- Current observations: active timestamped evidence lines, each shown as "[id] YYYY-MM-DD HH:MM [relevance] content".
|
|
9
|
+
- Current observations: active timestamped evidence lines, each shown as "[id] YYYY-MM-DD HH:MM [relevance] [coverage: none|partial|strong] content".
|
|
10
|
+
- Coverage tiers are review context: none means no current reflection supports the observation id, partial means exactly one current reflection supports it, and strong means two or more current reflections support it. Coverage is not a quota, target, priority score, or instruction to emit reflections.
|
|
10
11
|
|
|
11
12
|
What to emit:
|
|
12
13
|
- Emit only new durable reflections not already present in current reflections.
|
|
@@ -39,13 +40,16 @@ Focus on:
|
|
|
39
40
|
- Completed outcomes future runs must not redo.
|
|
40
41
|
- Durable blockers, invariants, and open decisions that should survive compaction.
|
|
41
42
|
|
|
42
|
-
Support ids:
|
|
43
|
+
Support ids and coverage stewardship:
|
|
43
44
|
- Every reflection must include supportingObservationIds from the current observations list.
|
|
44
|
-
-
|
|
45
|
-
- supportingObservationIds are
|
|
45
|
+
- First decide whether the reflection content passes the durable-value bar. Then audit support ids for that already-worthy reflection.
|
|
46
|
+
- supportingObservationIds are a coverage/provenance set and downstream dropper coverage evidence: include all current observation ids whose durable meaning is preserved by the reflection with equivalent fidelity and can later be treated as redundant active-memory detail.
|
|
47
|
+
- supportingObservationIds are not a checklist to cover every observation. Do not add ids merely to improve coverage counts, maximize support ids, maximize strong coverage, or unlock the dropper.
|
|
48
|
+
- False or inflated support ids can cause unsafe downstream dropper pruning, including removal of high-resistance active observations whose meaning was not actually preserved.
|
|
46
49
|
- Include additional observation ids only when the reflection preserves their durable meaning with equivalent fidelity.
|
|
47
50
|
- Leave observations unsupported when their details are still active working state, too specific to compress safely, or not yet durable enough.
|
|
48
51
|
- Do not include observations whose unique exact detail, current task state, user correction, user constraint, or concrete completion is not captured by the reflection.
|
|
52
|
+
- If no candidate reflection passes the durable-value bar, emit zero reflections even when observations have coverage: none.
|
|
49
53
|
- Never invent observation ids. Proposals with missing, empty, or invalid supportingObservationIds are rejected.
|
|
50
54
|
|
|
51
55
|
User assertions are authoritative. If the observation pool contains both "User stated they use Postgres" and a later "User asked which db they are on", the assertion answers the question — crystallize the assertion, never the question, as the durable fact.
|
package/src/om/consolidation.ts
CHANGED
|
@@ -49,7 +49,10 @@ import {
|
|
|
49
49
|
rawTokensSinceReflectionCoverage,
|
|
50
50
|
reflectionToSummaryLine,
|
|
51
51
|
reflectionsCreatedAfterIndex,
|
|
52
|
+
scoreObservation,
|
|
53
|
+
selectPriorObservations,
|
|
52
54
|
type Entry,
|
|
55
|
+
type Observation,
|
|
53
56
|
type Reflection,
|
|
54
57
|
} from "./ledger/index.js";
|
|
55
58
|
|
|
@@ -124,6 +127,35 @@ function mergeReflections(existing: Reflection[], additional: Reflection[]): Ref
|
|
|
124
127
|
return merged;
|
|
125
128
|
}
|
|
126
129
|
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Extract all pending observations from accumulated batches that were recorded
|
|
134
|
+
* after a given coverage ID (e.g., the last reflection or drop coverage ID).
|
|
135
|
+
* This is needed in noAutoCompact mode because the reflector/dropper may skip
|
|
136
|
+
* a pipeline cycle, leaving unprocessed batches in observationBatches that
|
|
137
|
+
* should still be served as "new" on subsequent runs.
|
|
138
|
+
*/
|
|
139
|
+
function pendingObservationsCreatedAfter(
|
|
140
|
+
pending: any,
|
|
141
|
+
entries: Entry[],
|
|
142
|
+
afterCoversUpToId: string | undefined,
|
|
143
|
+
): Observation[] {
|
|
144
|
+
const batches = pending.observationBatches ?? [];
|
|
145
|
+
if (!afterCoversUpToId || entryIndexForId(entries, afterCoversUpToId) < 0) {
|
|
146
|
+
return batches.flatMap((b: any) => (b.data as any)?.observations ?? []);
|
|
147
|
+
}
|
|
148
|
+
const afterIdx = entryIndexForId(entries, afterCoversUpToId);
|
|
149
|
+
const newObs: Observation[] = [];
|
|
150
|
+
for (const batch of batches) {
|
|
151
|
+
const batchIdx = entryIndexForId(entries, batch.coversUpToId);
|
|
152
|
+
if (batchIdx >= 0 && batchIdx > afterIdx) {
|
|
153
|
+
newObs.push(...((batch.data as any)?.observations ?? []));
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return newObs;
|
|
157
|
+
}
|
|
158
|
+
|
|
127
159
|
function anyStageDue(entries: Entry[], runtime: Runtime): boolean {
|
|
128
160
|
return rawTokensSinceObservationCoverage(entries) >= runtime.config.observeAfterTokens
|
|
129
161
|
|| rawTokensSinceReflectionCoverage(entries) >= runtime.config.reflectAfterTokens
|
|
@@ -272,12 +304,40 @@ async function runObserverStage(
|
|
|
272
304
|
if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
|
|
273
305
|
const chunkTokens = Math.ceil(chunk.length / 4);
|
|
274
306
|
|
|
307
|
+
const sessionId = ctx.sessionManager.getSessionId();
|
|
308
|
+
|
|
275
309
|
const memory = fullProjection(entries);
|
|
276
|
-
|
|
277
|
-
|
|
310
|
+
let priorReflections = memory.reflections.map(reflectionToSummaryLine);
|
|
311
|
+
let priorObservations = memory.observations.map(observationToSummaryLine);
|
|
312
|
+
|
|
313
|
+
// In noAutoCompact, append accumulated batch history to whatever
|
|
314
|
+
// fullProjection found in the branch (preserving pre-switch markers
|
|
315
|
+
// when transitioning from autoCompact to noAutoCompact mid-session).
|
|
316
|
+
// The preamble is capped via observerPreambleMaxTokens so accumulated
|
|
317
|
+
// observations don't grow unbounded across turns.
|
|
318
|
+
if (runtime.config.noAutoCompact) {
|
|
319
|
+
const pendingCtx = readPendingState(sessionId);
|
|
320
|
+
const accumulatedReflections = (pendingCtx.reflectionBatches ?? [])
|
|
321
|
+
.flatMap(b => (b.data as any).reflections ?? []);
|
|
322
|
+
const accumulatedObservations = (pendingCtx.observationBatches ?? [])
|
|
323
|
+
.flatMap(b => (b.data as any).observations ?? []);
|
|
324
|
+
|
|
325
|
+
// Capped preamble: high always kept, medium/low scored by relevance + recency
|
|
326
|
+
const preambleMaxTokens = runtime.config.observerPreambleMaxTokens > 0
|
|
327
|
+
? runtime.config.observerPreambleMaxTokens
|
|
328
|
+
: Math.round(runtime.config.observerChunkMaxTokens * 0.3);
|
|
329
|
+
const allObservations = [...memory.observations, ...accumulatedObservations];
|
|
330
|
+
priorObservations = selectPriorObservations(allObservations, preambleMaxTokens)
|
|
331
|
+
.map(observationToSummaryLine);
|
|
332
|
+
|
|
333
|
+
// Reflections are never trimmed — rare and always kept
|
|
334
|
+
priorReflections = [
|
|
335
|
+
...priorReflections,
|
|
336
|
+
...accumulatedReflections.map(reflectionToSummaryLine),
|
|
337
|
+
];
|
|
338
|
+
}
|
|
278
339
|
|
|
279
340
|
// If noAutoCompact: skip if this exact chunk was already processed
|
|
280
|
-
const sessionId = ctx.sessionManager.getSessionId();
|
|
281
341
|
if (runtime.config.noAutoCompact && isObservationChunkPending(sessionId, coversUpToId)) {
|
|
282
342
|
debugLog("observer.pending_skip", { coversUpToId, sessionId });
|
|
283
343
|
return "continue";
|
|
@@ -379,11 +439,33 @@ async function runReflectorStage(
|
|
|
379
439
|
): Promise<ReflectorStageResult> {
|
|
380
440
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
381
441
|
const entries = ctx.sessionManager.getBranch() as Entry[];
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
442
|
+
let reflectionTokens: number;
|
|
443
|
+
let observationCoverageId: string | undefined;
|
|
444
|
+
if (runtime.config.noAutoCompact) {
|
|
445
|
+
const pending = readPendingState(sessionId);
|
|
446
|
+
// Check any accumulated batch for unprocessed observations, not just the latest
|
|
447
|
+
const hasPendingObs = (pending.observationBatches ?? []).some((b: any) => (b.data as any)?.observations?.length);
|
|
448
|
+
if (!hasPendingObs) return { outcome: "continue", sameRunReflections: [] };
|
|
449
|
+
observationCoverageId = pending.observation?.coversUpToId;
|
|
450
|
+
if (pending.reflection?.coversUpToId) {
|
|
451
|
+
const obsIdx = entryIndexForId(entries, pending.observation?.coversUpToId ?? "");
|
|
452
|
+
const refIdx = entryIndexForId(entries, pending.reflection.coversUpToId);
|
|
453
|
+
if (obsIdx >= 0 && refIdx >= 0 && obsIdx <= refIdx) return { outcome: "continue", sameRunReflections: [] };
|
|
454
|
+
if (refIdx >= 0) {
|
|
455
|
+
reflectionTokens = rawTokensAfterIndex(entries, refIdx);
|
|
456
|
+
if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
|
|
457
|
+
} else {
|
|
458
|
+
reflectionTokens = rawTokensSinceObservationCoverage(entries);
|
|
459
|
+
}
|
|
460
|
+
} else {
|
|
461
|
+
reflectionTokens = rawTokensSinceObservationCoverage(entries);
|
|
462
|
+
}
|
|
463
|
+
} else {
|
|
464
|
+
reflectionTokens = rawTokensSinceReflectionCoverage(entries);
|
|
465
|
+
if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
|
|
466
|
+
observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
|
|
467
|
+
if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
|
|
468
|
+
}
|
|
387
469
|
|
|
388
470
|
for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
|
|
389
471
|
const resolved = await resolveModel("reflector");
|
|
@@ -391,12 +473,15 @@ async function runReflectorStage(
|
|
|
391
473
|
|
|
392
474
|
// Compute ahead for an accurate notification
|
|
393
475
|
const folded = foldLedger(entries);
|
|
394
|
-
const
|
|
395
|
-
const
|
|
396
|
-
const
|
|
476
|
+
const pending = runtime.config.noAutoCompact ? readPendingState(sessionId) : undefined;
|
|
477
|
+
const lastReflectionIdx = pending ? -1 : latestCoverageIndex(entries, OM_REFLECTIONS_RECORDED);
|
|
478
|
+
const newObservations = pending
|
|
479
|
+
? pendingObservationsCreatedAfter(pending, entries, pending.reflection?.coversUpToId)
|
|
480
|
+
: observationsCreatedAfterIndex(entries, lastReflectionIdx);
|
|
481
|
+
const newReflections = pending ? [] : reflectionsCreatedAfterIndex(entries, lastReflectionIdx);
|
|
397
482
|
const newItemsTokens = Math.ceil(
|
|
398
|
-
(newObservations.reduce((s, o) => s + o.content.length, 0) +
|
|
399
|
-
newReflections.reduce((s, r) => s + r.content.length, 0)) / 4
|
|
483
|
+
(newObservations.reduce((s: number, o: any) => s + o.content.length, 0) +
|
|
484
|
+
newReflections.reduce((s: number, r: any) => s + r.content.length, 0)) / 4
|
|
400
485
|
);
|
|
401
486
|
const summaryBudget = Math.floor(runtime.config.reflectorInputMaxTokens * 0.15) * 2;
|
|
402
487
|
const reflectorInputTokens = Math.min(newItemsTokens + summaryBudget, runtime.config.reflectorInputMaxTokens);
|
|
@@ -414,13 +499,21 @@ async function runReflectorStage(
|
|
|
414
499
|
// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
|
|
415
500
|
const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "reflector"), stageFallbacks: stageFallbackModels(runtime, "reflector") });
|
|
416
501
|
try {
|
|
417
|
-
// Existing memory summaries for context (capped)
|
|
502
|
+
// Existing memory summaries for context (capped).
|
|
503
|
+
// In noAutoCompact, merge accumulated pending batches with
|
|
504
|
+
// branch data (preserving pre-switch markers).
|
|
505
|
+
const sourceReflections = pending
|
|
506
|
+
? [...folded.reflections, ...(pending.reflectionBatches ?? []).flatMap((b: any) => (b.data as any)?.reflections ?? [])]
|
|
507
|
+
: folded.reflections;
|
|
508
|
+
const sourceObservations = pending
|
|
509
|
+
? [...folded.activeObservations, ...(pending.observationBatches ?? []).flatMap((b: any) => (b.data as any)?.observations ?? [])]
|
|
510
|
+
: folded.activeObservations;
|
|
418
511
|
const existingReflectionsSummary = buildExistingReflectionsSummary(
|
|
419
|
-
|
|
512
|
+
sourceReflections,
|
|
420
513
|
Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
|
|
421
514
|
);
|
|
422
515
|
const existingObservationsSummary = buildExistingObservationsSummary(
|
|
423
|
-
|
|
516
|
+
sourceObservations.filter((o: any) => !newObservations.some((no: any) => no.id === o.id)),
|
|
424
517
|
Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
|
|
425
518
|
);
|
|
426
519
|
|
|
@@ -437,6 +530,7 @@ async function runReflectorStage(
|
|
|
437
530
|
});
|
|
438
531
|
|
|
439
532
|
if (!reflections || reflections.length === 0) return { outcome: "continue", sameRunReflections: [] };
|
|
533
|
+
if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
|
|
440
534
|
|
|
441
535
|
const data = buildReflectionsRecordedData(reflections, observationCoverageId);
|
|
442
536
|
if (!data) return { outcome: "continue", sameRunReflections: [] };
|
|
@@ -474,11 +568,33 @@ async function runDropperStage(
|
|
|
474
568
|
): Promise<StageOutcome> {
|
|
475
569
|
const sessionId = ctx.sessionManager.getSessionId();
|
|
476
570
|
const entries = ctx.sessionManager.getBranch() as Entry[];
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
571
|
+
let dropTokens: number;
|
|
572
|
+
let observationCoverageId: string | undefined;
|
|
573
|
+
if (runtime.config.noAutoCompact) {
|
|
574
|
+
const pending = readPendingState(sessionId);
|
|
575
|
+
// Check any accumulated batch for unprocessed observations, not just the latest
|
|
576
|
+
const hasPendingObs = (pending.observationBatches ?? []).some((b: any) => (b.data as any)?.observations?.length);
|
|
577
|
+
if (!hasPendingObs) return "continue";
|
|
578
|
+
observationCoverageId = pending.observation?.coversUpToId;
|
|
579
|
+
if (pending.dropped?.coversUpToId) {
|
|
580
|
+
const obsIdx = entryIndexForId(entries, pending.observation?.coversUpToId ?? "");
|
|
581
|
+
const dropIdx = entryIndexForId(entries, pending.dropped.coversUpToId);
|
|
582
|
+
if (obsIdx >= 0 && dropIdx >= 0 && obsIdx <= dropIdx) return "continue";
|
|
583
|
+
if (dropIdx >= 0) {
|
|
584
|
+
dropTokens = rawTokensAfterIndex(entries, dropIdx);
|
|
585
|
+
if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
|
|
586
|
+
} else {
|
|
587
|
+
dropTokens = rawTokensSinceDropCoverage(entries);
|
|
588
|
+
}
|
|
589
|
+
} else {
|
|
590
|
+
dropTokens = rawTokensSinceDropCoverage(entries);
|
|
591
|
+
}
|
|
592
|
+
} else {
|
|
593
|
+
dropTokens = rawTokensSinceDropCoverage(entries);
|
|
594
|
+
if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
|
|
595
|
+
observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
|
|
596
|
+
if (!observationCoverageId) return "continue";
|
|
597
|
+
}
|
|
482
598
|
|
|
483
599
|
for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
|
|
484
600
|
const resolved = await resolveModel("dropper");
|
|
@@ -486,10 +602,13 @@ async function runDropperStage(
|
|
|
486
602
|
|
|
487
603
|
// Compute ahead for an accurate notification
|
|
488
604
|
const folded = foldLedger(entries);
|
|
489
|
-
const
|
|
490
|
-
const
|
|
605
|
+
const pending = runtime.config.noAutoCompact ? readPendingState(sessionId) : undefined;
|
|
606
|
+
const lastDropIdx = pending ? -1 : latestCoverageIndex(entries, OM_OBSERVATIONS_DROPPED);
|
|
607
|
+
const newObservations = pending
|
|
608
|
+
? pendingObservationsCreatedAfter(pending, entries, pending.dropped?.coversUpToId)
|
|
609
|
+
: observationsCreatedAfterIndex(entries, lastDropIdx);
|
|
491
610
|
const dropperNewObsTokens = Math.ceil(
|
|
492
|
-
newObservations.reduce((s, o) => s + o.content.length, 0) / 4
|
|
611
|
+
newObservations.reduce((s: number, o: any) => s + o.content.length, 0) / 4
|
|
493
612
|
);
|
|
494
613
|
const dropperSummaryBudget = Math.floor(runtime.config.dropperInputMaxTokens * 0.2);
|
|
495
614
|
const dropperInputTokens = Math.min(dropperNewObsTokens + dropperSummaryBudget, runtime.config.dropperInputMaxTokens);
|
|
@@ -505,12 +624,23 @@ async function runDropperStage(
|
|
|
505
624
|
if (ctx.hasUI) ctx.ui?.notify(`Observational memory: dropper running (~${effectiveDropTokens.toLocaleString()} tokens accumulated, ~${dropperInputTokens.toLocaleString()}-token input)`, "info");
|
|
506
625
|
|
|
507
626
|
try {
|
|
508
|
-
// Existing active observations summary for context (capped)
|
|
627
|
+
// Existing active observations summary for context (capped).
|
|
628
|
+
// In noAutoCompact, merge accumulated pending batches with
|
|
629
|
+
// branch data (preserving pre-switch markers).
|
|
630
|
+
const sourceObsForDropper = pending
|
|
631
|
+
? [...folded.activeObservations, ...(pending.observationBatches ?? []).flatMap((b: any) => (b.data as any)?.observations ?? [])]
|
|
632
|
+
: folded.activeObservations;
|
|
509
633
|
const existingObservationsSummary = buildExistingObservationsSummary(
|
|
510
|
-
|
|
634
|
+
sourceObsForDropper.filter((o: any) => !newObservations.some((no: any) => no.id === o.id)),
|
|
511
635
|
Math.floor(runtime.config.dropperInputMaxTokens * 0.2),
|
|
512
636
|
);
|
|
513
|
-
|
|
637
|
+
// In noAutoCompact, merge accumulated reflection batches with
|
|
638
|
+
// branch data (preserving pre-switch markers), matching the
|
|
639
|
+
// dropper's full autoCompact context.
|
|
640
|
+
const pendingReflections = pending
|
|
641
|
+
? [...folded.reflections, ...(pending.reflectionBatches ?? []).flatMap((b: any) => (b.data as any)?.reflections ?? [])]
|
|
642
|
+
: folded.reflections;
|
|
643
|
+
const reflectionsForDropper = mergeReflections(pendingReflections, sameRunReflections);
|
|
514
644
|
|
|
515
645
|
// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
|
|
516
646
|
const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "dropper"), stageFallbacks: stageFallbackModels(runtime, "dropper") });
|
|
@@ -525,7 +655,9 @@ async function runDropperStage(
|
|
|
525
655
|
maxTurns: runtime.config.agentMaxTurns,
|
|
526
656
|
thinkingLevel: stageThinkingLevel(runtime, "dropper", stageModelForThinking),
|
|
527
657
|
});
|
|
528
|
-
const latestReflectionCoverageId =
|
|
658
|
+
const latestReflectionCoverageId = runtime.config.noAutoCompact
|
|
659
|
+
? pending?.reflection?.coversUpToId
|
|
660
|
+
: latestCoverageMarkerId(entries, OM_REFLECTIONS_RECORDED);
|
|
529
661
|
const effectiveReflectionCoverageId = sameRunReflectionCoverageId ?? latestReflectionCoverageId;
|
|
530
662
|
const coversUpToId = earlierCoverageMarkerId(entries, observationCoverageId, effectiveReflectionCoverageId);
|
|
531
663
|
const data = coversUpToId && droppedIds ? buildObservationsDroppedData(droppedIds, coversUpToId) : undefined;
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/projection.ts)
|
|
5
5
|
* Unmodified.
|
|
6
6
|
*/
|
|
7
|
+
import { selectPriorObservations } from "./render-summary.js";
|
|
7
8
|
import {
|
|
8
9
|
OM_FOLDED,
|
|
9
10
|
isMemoryDetails,
|
|
@@ -207,10 +208,21 @@ export function buildCompactionProjection(
|
|
|
207
208
|
0,
|
|
208
209
|
);
|
|
209
210
|
const fullFold = observationTokens >= config.observationsPoolMaxTokens;
|
|
210
|
-
|
|
211
|
+
let projection = fullFold
|
|
211
212
|
? fullProjection(entries, firstKeptEntryId)
|
|
212
213
|
: normalProjection;
|
|
213
214
|
|
|
215
|
+
// Cap observations to budget using relevance-tiered + recency scoring.
|
|
216
|
+
// Even if the dropper determined some old observations are worth keeping,
|
|
217
|
+
// this safety valve ensures the compaction output never exceeds the pool
|
|
218
|
+
// token budget. Observations survive in the branch regardless.
|
|
219
|
+
if (config.observationsPoolMaxTokens > 0 && observationTokens >= config.observationsPoolMaxTokens) {
|
|
220
|
+
projection = {
|
|
221
|
+
observations: selectPriorObservations(projection.observations, config.observationsPoolMaxTokens),
|
|
222
|
+
reflections: projection.reflections,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
214
226
|
const details: MemoryDetails = {
|
|
215
227
|
type: OM_FOLDED,
|
|
216
228
|
version: 1,
|
|
@@ -25,6 +25,57 @@ export function observationToSummaryLine(observation: Observation): string {
|
|
|
25
25
|
return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] ${observation.content}`;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
/** Score an observation for cap/trim selection.
|
|
29
|
+
* Relevance tier dominates: medium (5+) always outranks low (max 2).
|
|
30
|
+
* Recency is based on position in the flat-mapped array (0 = oldest, N-1 = newest),
|
|
31
|
+
* avoiding wall-clock dependency that punishes sessions spanning days or weeks. */
|
|
32
|
+
export function scoreObservation(obs: Observation, index: number, total: number): number {
|
|
33
|
+
const base = obs.relevance === "high" || obs.relevance === "critical" ? 10
|
|
34
|
+
: obs.relevance === "medium" ? 5 : 1;
|
|
35
|
+
const recency = total > 1 ? index / (total - 1) : 1;
|
|
36
|
+
return base + recency;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Select observations up to a token budget, keeping all high-relevance items
|
|
40
|
+
* unconditionally and filling the remaining budget with the best-scoring
|
|
41
|
+
* medium and low observations (relevance-tiered + recency).
|
|
42
|
+
*
|
|
43
|
+
* Reflections are never trimmed — they are inherently rare and always stay.
|
|
44
|
+
* Observations stay in the branch either way; this only caps what is rendered
|
|
45
|
+
* in the compaction summary output. */
|
|
46
|
+
export function selectPriorObservations(observations: Observation[], maxTokens: number): Observation[] {
|
|
47
|
+
// Track original indices so we can restore chronological order after scoring
|
|
48
|
+
const indexed = observations.map((obs, i) => ({ obs, originalIndex: i }));
|
|
49
|
+
const high = indexed.filter(item => item.obs.relevance === "high" || item.obs.relevance === "critical");
|
|
50
|
+
const rest = indexed.filter(item => item.obs.relevance !== "high" && item.obs.relevance !== "critical");
|
|
51
|
+
|
|
52
|
+
// High always kept — consume budget first
|
|
53
|
+
let budget = maxTokens;
|
|
54
|
+
const selected = new Set<{ obs: Observation; originalIndex: number }>();
|
|
55
|
+
for (const item of high) {
|
|
56
|
+
const lineTokens = Math.ceil(observationToSummaryLine(item.obs).length / 4);
|
|
57
|
+
selected.add(item);
|
|
58
|
+
budget -= lineTokens;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Score medium + low and select best within remaining budget
|
|
62
|
+
if (rest.length > 0 && budget > 0) {
|
|
63
|
+
const scored = rest.map((item, i) => ({ item, score: scoreObservation(item.obs, i, rest.length) }));
|
|
64
|
+
scored.sort((a, b) => b.score - a.score); // highest score first
|
|
65
|
+
for (const { item } of scored) {
|
|
66
|
+
const lineTokens = Math.ceil(observationToSummaryLine(item.obs).length / 4);
|
|
67
|
+
if (budget - lineTokens < 0) break;
|
|
68
|
+
selected.add(item);
|
|
69
|
+
budget -= lineTokens;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Restore original chronological order before returning
|
|
74
|
+
return Array.from(selected)
|
|
75
|
+
.sort((a, b) => a.originalIndex - b.originalIndex)
|
|
76
|
+
.map(item => item.obs);
|
|
77
|
+
}
|
|
78
|
+
|
|
28
79
|
export function reflectionToSummaryLine(reflection: Reflection): string {
|
|
29
80
|
return `[${reflection.id}] ${reflection.content}`;
|
|
30
81
|
}
|
package/src/om/pending.ts
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
17
17
|
import { dirname, join } from "node:path";
|
|
18
18
|
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
19
|
+
import type { Observation, Reflection } from "./ledger/types.js";
|
|
19
20
|
|
|
20
21
|
// ── Types ───────────────────────────────────────────────────────────────────
|
|
21
22
|
|
|
@@ -41,6 +42,25 @@ export interface PendingOMState {
|
|
|
41
42
|
reflection?: PendingReflection;
|
|
42
43
|
/** Latest dropper run (replaced each time, not accumulated). */
|
|
43
44
|
dropped?: PendingDropped;
|
|
45
|
+
/**
|
|
46
|
+
* All observation batches accumulated across noAutoCompact pipeline runs.
|
|
47
|
+
* Each batch preserves per-run coverage (coversUpToId) matching the normal
|
|
48
|
+
* branch-marker pattern. Used for LLM context and /blackhole flush.
|
|
49
|
+
*/
|
|
50
|
+
observationBatches?: PendingObservation[];
|
|
51
|
+
/**
|
|
52
|
+
* All reflection batches accumulated across noAutoCompact pipeline runs.
|
|
53
|
+
* Preserves per-run coverage for LLM context and /blackhole flush.
|
|
54
|
+
*/
|
|
55
|
+
reflectionBatches?: PendingReflection[];
|
|
56
|
+
/**
|
|
57
|
+
* All dropper batches accumulated across noAutoCompact pipeline runs.
|
|
58
|
+
* Each batch preserves which observations were dropped in that run.
|
|
59
|
+
* Without accumulation, earlier drops are lost when the next dropper
|
|
60
|
+
* run overwrites pending.dropped, causing them to be "un-dropped" on
|
|
61
|
+
* /blackhole flush.
|
|
62
|
+
*/
|
|
63
|
+
droppedBatches?: PendingDropped[];
|
|
44
64
|
}
|
|
45
65
|
|
|
46
66
|
// ── Persistence ─────────────────────────────────────────────────────────────
|
|
@@ -70,7 +90,10 @@ function defaultState(): PendingOMState {
|
|
|
70
90
|
}
|
|
71
91
|
|
|
72
92
|
function isEmptyState(s: PendingOMState): boolean {
|
|
73
|
-
return !s.observation && !s.reflection && !s.dropped
|
|
93
|
+
return !s.observation && !s.reflection && !s.dropped
|
|
94
|
+
&& (!s.observationBatches || s.observationBatches.length === 0)
|
|
95
|
+
&& (!s.reflectionBatches || s.reflectionBatches.length === 0)
|
|
96
|
+
&& (!s.droppedBatches || s.droppedBatches.length === 0);
|
|
74
97
|
}
|
|
75
98
|
|
|
76
99
|
// ── Per-session file read/write ─────────────────────────────────────────────
|
|
@@ -145,6 +168,10 @@ function isPendingOMState(value: unknown): value is PendingOMState {
|
|
|
145
168
|
export function savePendingObservation(sessionId: string, entry: PendingObservation): void {
|
|
146
169
|
const state = readSessionState(sessionId);
|
|
147
170
|
state.observation = entry;
|
|
171
|
+
// Append to accumulated batches for LLM context and /blackhole flush.
|
|
172
|
+
// Each batch preserves per-run coverage (coversUpToId) matching the
|
|
173
|
+
// normal branch-marker pattern.
|
|
174
|
+
state.observationBatches = [...(state.observationBatches ?? []), entry];
|
|
148
175
|
writeSessionState(sessionId, state);
|
|
149
176
|
}
|
|
150
177
|
|
|
@@ -154,15 +181,20 @@ export function savePendingObservation(sessionId: string, entry: PendingObservat
|
|
|
154
181
|
export function savePendingReflection(sessionId: string, entry: PendingReflection): void {
|
|
155
182
|
const state = readSessionState(sessionId);
|
|
156
183
|
state.reflection = entry;
|
|
184
|
+
// Append to accumulated batches for LLM context and /blackhole flush.
|
|
185
|
+
state.reflectionBatches = [...(state.reflectionBatches ?? []), entry];
|
|
157
186
|
writeSessionState(sessionId, state);
|
|
158
187
|
}
|
|
159
188
|
|
|
160
189
|
/**
|
|
161
|
-
* Save (replace) the latest dropper result for a session
|
|
190
|
+
* Save (replace) the latest dropper result for a session and
|
|
191
|
+
* append to droppedBatches so no drops are lost across cycles
|
|
192
|
+
* before /blackhole flush.
|
|
162
193
|
*/
|
|
163
194
|
export function savePendingDropped(sessionId: string, entry: PendingDropped): void {
|
|
164
195
|
const state = readSessionState(sessionId);
|
|
165
196
|
state.dropped = entry;
|
|
197
|
+
state.droppedBatches = [...(state.droppedBatches ?? []), entry];
|
|
166
198
|
writeSessionState(sessionId, state);
|
|
167
199
|
}
|
|
168
200
|
|