pi-blackhole 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -5,8 +5,14 @@ Algorithmic compaction + session-aware observational memory for [Pi](https://git
5
5
  Combines [pi-vcc](https://github.com/sting8k/pi-vcc) and [pi-observational-memory](https://github.com/elpapi42/pi-observational-memory) with unified configuration, per-worker model fallback chains, persisted cooldowns, and a manual flush mode.
6
6
 
7
7
  > This is a frankenmerge. I liked both extensions but they were not compatible - observational memory hooked into Pi's default compaction and prevented pi-vcc from working. So I merged them, made them share the same hook and output, and added the things both were missing: fallback chains, cooldowns, and a toggle between them.
8
+ > Please also see the [`CHANGELOG.md`](CHANGELOG.md)
9
+
10
+ ### Lockstep with upstreams
11
+
12
+ pi-blackhole tracks both upstream repositories via a [lockstep audit system](https://github.com/k0valik/pi-blackhole/tree/lockstep/2026-05-27/.pi/skills/lockstep) that classifies every new upstream commit as safe-to-port, modified (needs review), rewritten (skip), or orphan (needs mapping). The goal is to lift bugfixes, prompts improvements, and compatible features without breaking existing users or rolling back intentional divergences. Ported changes are reviewed per-commit with human approval — nothing is blindly merged. See [SKILL.md](.pi/skills/lockstep/SKILL.md) for the full workflow. An example execution (including rationale for skipped changes) is documented in [PR #8](https://github.com/k0valik/pi-blackhole/pull/8).
13
+
14
+ #### For easy setup pass the [`llms.txt`](llms.txt) llms.txt to your agent and it will guide you through the config without needing to read all the docs if you're as lazy as me.
8
15
 
9
- #### For easy setup pass the [`llms.txt`](llms.txt) llms.txt to your agent and it will guide you through the config without needing to read all the docs if you're as lazy as me
10
16
 
11
17
  # Demo
12
18
 
@@ -131,6 +137,7 @@ The tradeoff is simplicity vs cleanliness:
131
137
  |---|---|---|
132
138
  | Workers run? | Yes | Yes |
133
139
  | Observations go to | Conversation markers (invisible in TUI) | Disk (`pending.json`) |
140
+ | Observations accumulate across runs | Branch markers (replaced each cycle) | Pending batches — `/memory` shows pending counts |
134
141
  | Auto-compact on `agent_end` | Yes | No |
135
142
  | `/blackhole` | Optional — use it whenever you want | Required to flush + compact |
136
143
  | Conversation history | OM marker entries between turns (they exist but do not clutter the display) | Clean — nothing between turns |
@@ -198,6 +205,7 @@ Quick start with custom models:
198
205
  | `reflectAfterTokens` | `20000` | Token cadence for reflector and dropper |
199
206
  | `compactAfterTokens` | `81000` | Auto-compaction threshold |
200
207
  | `observerChunkMaxTokens` | `40000` | Max observer input per run (newest-first) |
208
+ | `observerPreambleMaxTokens` | `0` (auto) | Max preamble tokens in observer prompt for `noAutoCompact` mode (auto = 30% of chunk) |
201
209
  | `observationsPoolMaxTokens` | `20000` | Max active observation pool before dropper prunes |
202
210
  | `reflectorInputMaxTokens` | `80000` | Max reflector input budget |
203
211
  | `dropperInputMaxTokens` | `80000` | Max dropper input budget |
@@ -220,6 +228,7 @@ Paste the appropriate block into your config to match your model's context size.
220
228
  "reflectAfterTokens": 10000,
221
229
  "compactAfterTokens": 30000,
222
230
  "observerChunkMaxTokens": 15000,
231
+ "observerPreambleMaxTokens": 0,
223
232
  "observationsPoolMaxTokens": 8000,
224
233
  "reflectorInputMaxTokens": 30000,
225
234
  "dropperInputMaxTokens": 30000
@@ -236,6 +245,7 @@ Our built-in defaults already target this tier. If you reset your config, these
236
245
  "reflectAfterTokens": 20000,
237
246
  "compactAfterTokens": 81000,
238
247
  "observerChunkMaxTokens": 40000,
248
+ "observerPreambleMaxTokens": 0,
239
249
  "observationsPoolMaxTokens": 20000,
240
250
  "reflectorInputMaxTokens": 80000,
241
251
  "dropperInputMaxTokens": 80000
@@ -250,6 +260,7 @@ Our built-in defaults already target this tier. If you reset your config, these
250
260
  "reflectAfterTokens": 40000,
251
261
  "compactAfterTokens": 180000,
252
262
  "observerChunkMaxTokens": 80000,
263
+ "observerPreambleMaxTokens": 0,
253
264
  "observationsPoolMaxTokens": 40000,
254
265
  "reflectorInputMaxTokens": 160000,
255
266
  "dropperInputMaxTokens": 160000
@@ -86,13 +86,14 @@
86
86
  }
87
87
  ],
88
88
 
89
- "observeAfterTokens": 10000,
90
- "reflectAfterTokens": 20000,
89
+ "observeAfterTokens": 15000,
90
+ "reflectAfterTokens": 25000,
91
91
  "compactAfterTokens": 81000,
92
92
  "observationsPoolMaxTokens": 20000,
93
93
  "reflectorInputMaxTokens": 80000,
94
94
  "dropperInputMaxTokens": 80000,
95
95
  "observerChunkMaxTokens": 40000,
96
+ "observerPreambleMaxTokens": 0,
96
97
  "agentMaxTurns": 16,
97
98
 
98
99
  "passive": false,
@@ -110,6 +111,16 @@
110
111
  "After all candidates exhausted, the stage aborts. 30s retry gate prevents spam.",
111
112
  "Cooldowns persist in ~/.pi/agent/pi-blackhole/pi-blackhole-cooldown.json (survives pi restarts).",
112
113
  "Valid thinking values: off, minimal, low, medium, high, xhigh.",
113
- "Env override for passive mode: PI_BLACKHOLE_PASSIVE=true."
114
+ "Env override for passive mode: PI_BLACKHOLE_PASSIVE=true.",
115
+ "",
116
+ "noAutoCompact mode accumulates observationBatches/reflectionBatches in",
117
+ "pending.json across pipeline runs. The observer preamble (CURRENT OBSERVATIONS)",
118
+ "is capped to observerPreambleMaxTokens (default 0 = auto 30% of observerChunkMaxTokens).",
119
+ "This is a soft safety valve — it only trims when accumulated observations exceed",
120
+ "the budget. High-relevance observations are always kept. Reflections are never trimmed.",
121
+ "",
122
+ "agentMaxTurns controls how many agent-loop turns each worker gets per chunk.",
123
+ "Lower-context models can reduce this (e.g., 8) so the observer doesn't waste",
124
+ "turns trying to extract more than it can reliably handle per run."
114
125
  ]
115
126
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-blackhole",
3
- "version": "0.2.1",
3
+ "version": "0.2.3",
4
4
  "description": "Unified compaction + observational memory extension for Pi — compresses conversation context while preserving durable observations and reflections",
5
5
  "license": "MIT",
6
6
  "main": "index.ts",
@@ -164,11 +164,12 @@ export function registerMemoryCommand(pi: ExtensionAPI, runtime: Runtime): void
164
164
  observationLine,
165
165
  reflectionLine,
166
166
  "",
167
- "── Activity ──",
168
- `Observer: ~${obsProgress.toLocaleString()} / ${runtime.config.observeAfterTokens.toLocaleString()} tokens (${pct(obsProgress, runtime.config.observeAfterTokens)}%)`,
169
- `Reflector: ~${reflectionProgress.toLocaleString()} / ${runtime.config.reflectAfterTokens.toLocaleString()} tokens (${pct(reflectionProgress, runtime.config.reflectAfterTokens)}%)`,
170
- `Dropper: ~${dropProgress.toLocaleString()} / ${runtime.config.reflectAfterTokens.toLocaleString()} tokens (${pct(dropProgress, runtime.config.reflectAfterTokens)}%)`,
171
- `Compaction: ~${compactionProgress.toLocaleString()} / ${runtime.config.compactAfterTokens.toLocaleString()} tokens (${pct(compactionProgress, runtime.config.compactAfterTokens)}%)`,
167
+ "── Pipeline ──",
168
+ "Transcript accumulated since last run. Triggers when exceeding threshold.",
169
+ `Observer: ~${obsProgress.toLocaleString()} tokens (triggers at ${runtime.config.observeAfterTokens.toLocaleString()})`,
170
+ `Reflector: ~${reflectionProgress.toLocaleString()} tokens (triggers at ${runtime.config.reflectAfterTokens.toLocaleString()})`,
171
+ `Dropper: ~${dropProgress.toLocaleString()} tokens (triggers at ${runtime.config.reflectAfterTokens.toLocaleString()})`,
172
+ `Compaction: ~${compactionProgress.toLocaleString()} tokens` + (runtime.config.noAutoCompact ? " [auto-disabled]" : ` (triggers at ${runtime.config.compactAfterTokens.toLocaleString()})`),
172
173
  `Obs pool: ~${visibleObservationTokens.toLocaleString()} / ${runtime.config.observationsPoolMaxTokens.toLocaleString()} tokens (${pct(visibleObservationTokens, runtime.config.observationsPoolMaxTokens)}%)`,
173
174
  `Reflect pool: ~${visibleReflectionTokens.toLocaleString()} tokens`,
174
175
  ];
@@ -184,6 +185,13 @@ export function registerMemoryCommand(pi: ExtensionAPI, runtime: Runtime): void
184
185
  if (hasObs) lines.push("Observation: waiting in pending.json");
185
186
  if (hasRef) lines.push("Reflection: waiting in pending.json");
186
187
  if (hasDrop) lines.push("Dropper: waiting in pending.json");
188
+ const preambleCap = runtime.config.observerPreambleMaxTokens > 0
189
+ ? runtime.config.observerPreambleMaxTokens
190
+ : Math.round(runtime.config.observerChunkMaxTokens * 0.3);
191
+ const pctNote = runtime.config.observerPreambleMaxTokens > 0
192
+ ? ""
193
+ : ` (30% of ${runtime.config.observerChunkMaxTokens.toLocaleString()} chunk)`;
194
+ lines.push(`Preamble cap: ${preambleCap.toLocaleString()} tokens for observations${pctNote}`);
187
195
  lines.push("Run /blackhole to flush and compact.");
188
196
  }
189
197
  }
@@ -55,14 +55,29 @@ export const registerPiVccCommand = (pi: ExtensionAPI, runtime: Runtime) => {
55
55
  // before compacting so the summary includes accumulated memory.
56
56
  if (runtime.config.noAutoCompact && hasPendingData(sessionId)) {
57
57
  const pending = readPendingState(sessionId);
58
- if (pending.observation) {
59
- pi.appendEntry(OM_OBSERVATIONS_RECORDED, pending.observation.data);
58
+ // Write all accumulated observation batches (or latest single batch
59
+ // as fallback for legacy pending.json without batch arrays).
60
+ const obsBatches = pending.observationBatches?.length
61
+ ? pending.observationBatches
62
+ : (pending.observation ? [pending.observation] : []);
63
+ for (const batch of obsBatches) {
64
+ pi.appendEntry(OM_OBSERVATIONS_RECORDED, batch.data);
60
65
  }
61
- if (pending.reflection) {
62
- pi.appendEntry(OM_REFLECTIONS_RECORDED, pending.reflection.data);
66
+ // Write all accumulated reflection batches (or latest single batch
67
+ // as fallback for legacy pending.json without batch arrays).
68
+ const reflBatches = pending.reflectionBatches?.length
69
+ ? pending.reflectionBatches
70
+ : (pending.reflection ? [pending.reflection] : []);
71
+ for (const batch of reflBatches) {
72
+ pi.appendEntry(OM_REFLECTIONS_RECORDED, batch.data);
63
73
  }
64
- if (pending.dropped) {
65
- pi.appendEntry(OM_OBSERVATIONS_DROPPED, pending.dropped.data);
74
+ // Write all accumulated dropper batches (or latest single batch
75
+ // as fallback for legacy pending.json without batch arrays).
76
+ const dropBatches = pending.droppedBatches?.length
77
+ ? pending.droppedBatches
78
+ : (pending.dropped ? [pending.dropped] : []);
79
+ for (const batch of dropBatches) {
80
+ pi.appendEntry(OM_OBSERVATIONS_DROPPED, batch.data);
66
81
  }
67
82
  clearPendingState(sessionId);
68
83
  ctx.ui.notify("Observational memory: pending entries flushed", "info");
@@ -44,12 +44,27 @@ export interface UnifiedConfig {
44
44
  compactAfterTokens: number;
45
45
  /** Observation pool token pressure for full fold. */
46
46
  observationsPoolMaxTokens: number;
47
+ /** Target token budget for the observation pool (dropper aims here).
48
+ * Optional; defaults to half of observationsPoolMaxTokens when unset.
49
+ * Must be less than observationsPoolMaxTokens.
50
+ *
51
+ * NOTE: Ported from upstream as forward-compat (no-op in our pool algorithm).
52
+ * Upstream renamed budgetTokens→targetTokens (52b5844) and uses this
53
+ * for their tokensOverTarget / avgTokensPerObservation drop calculation.
54
+ * We keep our ratio-based urgency algorithm; this knob exists so future
55
+ * lockstep iterations don't diverge on the config shape. */
56
+ observationsPoolTargetTokens: number;
47
57
  /** Max prompt tokens for reflector model input (rolling window cap). */
48
58
  reflectorInputMaxTokens: number;
49
59
  /** Max prompt tokens for dropper model input (rolling window cap). */
50
60
  dropperInputMaxTokens: number;
51
61
  /** Max source entries tokens sent to observer per chunk. */
52
62
  observerChunkMaxTokens: number;
63
+ /** Max preamble tokens (CURRENT REFLECTIONS / OBSERVATIONS) in the observer prompt.
64
+ * Default 0 means auto-compute from observerChunkMaxTokens (30%). Only applied in
65
+ * noAutoCompact mode where accumulated batch history can grow unbounded.
66
+ * Set to an explicit value to override the auto-computed budget. */
67
+ observerPreambleMaxTokens: number;
53
68
  /** Shared turn cap for background memory agents. */
54
69
  agentMaxTurns: number;
55
70
 
@@ -87,13 +102,15 @@ export const DEFAULTS: UnifiedConfig = {
87
102
  overrideDefaultCompaction: false,
88
103
  debug: false,
89
104
 
90
- observeAfterTokens: 10_000,
91
- reflectAfterTokens: 20_000,
105
+ observeAfterTokens: 15_000,
106
+ reflectAfterTokens: 25_000,
92
107
  compactAfterTokens: 81_000,
93
108
  observationsPoolMaxTokens: 20_000,
109
+ observationsPoolTargetTokens: 10_000,
94
110
  reflectorInputMaxTokens: 80_000,
95
111
  dropperInputMaxTokens: 80_000,
96
112
  observerChunkMaxTokens: 40_000,
113
+ observerPreambleMaxTokens: 0,
97
114
  agentMaxTurns: 16,
98
115
 
99
116
  noAutoCompact: false,
@@ -154,7 +171,7 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
154
171
  if (typeof raw.debugLog === "boolean") c.debugLog = raw.debugLog;
155
172
 
156
173
  // Positive integers
157
- const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "agentMaxTurns"] as const;
174
+ const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "observationsPoolTargetTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "observerPreambleMaxTokens", "agentMaxTurns"] as const;
158
175
  for (const k of numKeys) {
159
176
  const v = positiveInt(raw[k]);
160
177
  if (v !== undefined) (c as Record<string, unknown>)[k] = v;
@@ -232,7 +249,15 @@ export function loadUnifiedConfig(cwd: string): UnifiedConfig {
232
249
  else if (["0", "false", "no", "off"].includes(v)) parsed.passive = false;
233
250
  }
234
251
 
235
- return { ...DEFAULTS, ...parsed };
252
+ // Merge defaults then override
253
+ const merged = { ...DEFAULTS, ...parsed };
254
+
255
+ // Derive observationsPoolTargetTokens if unset or invalid (must be < max)
256
+ if (merged.observationsPoolTargetTokens >= merged.observationsPoolMaxTokens) {
257
+ merged.observationsPoolTargetTokens = Math.floor(merged.observationsPoolMaxTokens / 2);
258
+ }
259
+
260
+ return merged;
236
261
  }
237
262
 
238
263
  /**
@@ -9,9 +9,18 @@ import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } fr
9
9
  import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
10
10
  import { Type } from "@earendil-works/pi-ai";
11
11
  import type { Static } from "typebox";
12
+ import { debugLog } from "../../debug-log.js";
12
13
  import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
13
- import { observationToSummaryLine, reflectionToSummaryLine, type Observation, type Reflection } from "../../ledger/index.js";
14
+ import { reflectionToSummaryLine, type Observation, type Reflection } from "../../ledger/index.js";
14
15
  import { DROPPER_SYSTEM } from "./prompts.js";
16
+ import {
17
+ REFLECTION_COVERAGE_DROP_RANK,
18
+ coverageTierForObservation,
19
+ observationToDropperLine,
20
+ reflectionCoverageMap,
21
+ summarizeCoverageByRelevance,
22
+ summarizeCoverageByRelevanceForIds,
23
+ } from "./coverage.js";
15
24
 
16
25
  interface RunDropperArgs {
17
26
  model: Model<any>;
@@ -81,6 +90,13 @@ export function maxDropCountForPool(observations: readonly Observation[], observ
81
90
  return Math.max(1, Math.floor(droppableCount * dropRatio));
82
91
  }
83
92
 
93
+ function relevanceCounts(observations: readonly Observation[]): Record<Observation["relevance"], number> {
94
+ return observations.reduce<Record<Observation["relevance"], number>>((counts, observation) => {
95
+ if (observation.relevance in counts) counts[observation.relevance]++;
96
+ return counts;
97
+ }, { low: 0, medium: 0, high: 0, critical: 0 });
98
+ }
99
+
84
100
  export function normalizeDropObservationIds(
85
101
  ids: readonly string[] | undefined,
86
102
  observations: readonly Observation[],
@@ -92,7 +108,6 @@ export function normalizeDropObservationIds(
92
108
  for (const id of ids) {
93
109
  const observation = allowed.get(id);
94
110
  if (!observation) continue;
95
- if (observation.relevance === "critical") continue;
96
111
  if (seen.has(id)) continue;
97
112
  seen.add(id);
98
113
  result.push(id);
@@ -100,14 +115,21 @@ export function normalizeDropObservationIds(
100
115
  return result.length > 0 ? result : undefined;
101
116
  }
102
117
 
118
+ function timestampRank(timestamp: string): number {
119
+ const parsed = Date.parse(timestamp);
120
+ return Number.isFinite(parsed) ? parsed : Number.POSITIVE_INFINITY;
121
+ }
122
+
103
123
  export function selectDropCandidates(
104
124
  ids: readonly string[],
105
125
  observations: readonly Observation[],
106
126
  maxDrops: number,
127
+ reflections: readonly Reflection[] = [],
107
128
  ): string[] {
108
129
  if (maxDrops <= 0 || ids.length === 0) return [];
109
130
 
110
131
  const byId = new Map(observations.map((observation) => [observation.id, observation]));
132
+ const coverageById = reflectionCoverageMap(observations, reflections);
111
133
  const firstProposalIndex = new Map<string, number>();
112
134
  for (let i = 0; i < ids.length; i++) {
113
135
  const id = ids[i];
@@ -117,11 +139,16 @@ export function selectDropCandidates(
117
139
  return Array.from(firstProposalIndex.entries())
118
140
  .map(([id, index]) => ({ id, index, observation: byId.get(id) }))
119
141
  .filter((candidate): candidate is { id: string; index: number; observation: Observation } =>
120
- candidate.observation !== undefined && candidate.observation.relevance !== "critical"
142
+ candidate.observation !== undefined
121
143
  )
122
144
  .sort((a, b) => {
145
+ const coverageDelta = REFLECTION_COVERAGE_DROP_RANK[coverageTierForObservation(a.observation, coverageById)]
146
+ - REFLECTION_COVERAGE_DROP_RANK[coverageTierForObservation(b.observation, coverageById)];
123
147
  const relevanceDelta = RELEVANCE_DROP_RANK[a.observation.relevance] - RELEVANCE_DROP_RANK[b.observation.relevance];
124
- return relevanceDelta || a.index - b.index;
148
+ const aAge = timestampRank(a.observation.timestamp);
149
+ const bAge = timestampRank(b.observation.timestamp);
150
+ const ageDelta = aAge === bAge ? 0 : aAge - bAge;
151
+ return coverageDelta || relevanceDelta || ageDelta || a.index - b.index;
125
152
  })
126
153
  .slice(0, maxDrops)
127
154
  .map((candidate) => candidate.id);
@@ -135,10 +162,42 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
135
162
  const fullness = observationPoolFullness(observationTokens, budgetTokens);
136
163
  const urgency = dropUrgencyForFullness(fullness);
137
164
  const maxDropsAllowed = maxDropCountForPool(observations, observationTokens, budgetTokens);
138
- if (maxDropsAllowed <= 0) return undefined;
165
+ const coverageById = reflectionCoverageMap(observations, reflections);
166
+ const coverageSummaryByRelevance = summarizeCoverageByRelevance(observations, coverageById);
167
+ debugLog("dropper.agent_start", {
168
+ activeObservationCount: observations.length,
169
+ reflectionCount: reflections.length,
170
+ observationTokens,
171
+ budgetTokens,
172
+ fullness,
173
+ urgency,
174
+ maxDropsAllowed,
175
+ relevanceCounts: relevanceCounts(observations),
176
+ coverageSummaryByRelevance,
177
+ });
178
+ if (maxDropsAllowed <= 0) {
179
+ debugLog("dropper.result", {
180
+ reason: "not_over_target",
181
+ toolCallCount: 0,
182
+ rawRequestedIdsCount: 0,
183
+ acceptedCandidateCount: 0,
184
+ selectedDropsCount: 0,
185
+ selectedDropTokens: 0,
186
+ selectedCoverageSummaryByRelevance: summarizeCoverageByRelevanceForIds([], observations, coverageById),
187
+ maxDropsAllowed,
188
+ });
189
+ return undefined;
190
+ }
139
191
 
140
192
  const proposedDropIds: string[] = [];
141
193
  const proposed = new Set<string>();
194
+ const allowed = new Map(observations.map((observation) => [observation.id, observation]));
195
+ let toolCallCount = 0;
196
+ let rawRequestedIdsCount = 0;
197
+ let missingIdsCount = 0;
198
+ let criticalCandidateIdsCount = 0;
199
+ let duplicateInRequestCount = 0;
200
+ let duplicateInRunCount = 0;
142
201
 
143
202
  const dropObservations: AgentTool<typeof DropObservationsSchema> = {
144
203
  name: "drop_observations",
@@ -146,14 +205,51 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
146
205
  description: "Propose active observation ids that are safe to remove from compacted memory.",
147
206
  parameters: DropObservationsSchema,
148
207
  execute: async (_id, params: DropObservationsArgs) => {
149
- const normalized = normalizeDropObservationIds(params.ids, observations) ?? [];
208
+ toolCallCount++;
209
+ rawRequestedIdsCount += params.ids.length;
210
+ const seenInRequest = new Set<string>();
150
211
  let added = 0;
151
- for (const id of normalized) {
152
- if (proposed.has(id)) continue;
212
+ let requestMissingIds = 0;
213
+ let requestCriticalCandidateIds = 0;
214
+ let requestDuplicateIds = 0;
215
+ let requestDuplicateInRunIds = 0;
216
+ for (const id of params.ids) {
217
+ const observation = allowed.get(id);
218
+ if (!observation) {
219
+ missingIdsCount++;
220
+ requestMissingIds++;
221
+ continue;
222
+ }
223
+ if (seenInRequest.has(id)) {
224
+ duplicateInRequestCount++;
225
+ requestDuplicateIds++;
226
+ continue;
227
+ }
228
+ seenInRequest.add(id);
229
+ if (proposed.has(id)) {
230
+ duplicateInRunCount++;
231
+ requestDuplicateInRunIds++;
232
+ continue;
233
+ }
153
234
  proposed.add(id);
154
235
  proposedDropIds.push(id);
236
+ if (observation.relevance === "critical") {
237
+ criticalCandidateIdsCount++;
238
+ requestCriticalCandidateIds++;
239
+ }
155
240
  added++;
156
241
  }
242
+ debugLog("dropper.tool_call", {
243
+ toolCallCount,
244
+ rawRequestedIdsCount: params.ids.length,
245
+ acceptedIdsCount: added,
246
+ missingIdsCount: requestMissingIds,
247
+ criticalCandidateIdsCount: requestCriticalCandidateIds,
248
+ duplicateInRequestCount: requestDuplicateIds,
249
+ duplicateInRunCount: requestDuplicateInRunIds,
250
+ totalCandidates: proposedDropIds.length,
251
+ maxDropsAllowed,
252
+ });
157
253
  return {
158
254
  content: [{ type: "text", text: `Queued ${added} drop candidate${added === 1 ? "" : "s"}. Candidates this run: ${proposedDropIds.length}. Maximum drops allowed: ${maxDropsAllowed}.` }],
159
255
  details: { added, totalCandidates: proposedDropIds.length, maxDropsAllowed },
@@ -166,7 +262,7 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
166
262
  ? `EXISTING ACTIVE OBSERVATIONS (for context only — these are NOT candidates for dropping):\n${args.existingObservationsSummary}\n\n`
167
263
  : '';
168
264
 
169
- const userText = `CURRENT REFLECTIONS:\n${joinOrEmpty(reflections.map(reflectionToSummaryLine))}\n\n${existingObservationsContext}NEW OBSERVATIONS TO EVALUATE FOR DROPPING:\n${joinOrEmpty(observations.map(observationToSummaryLine))}\n\nObservation pool pressure: ~${observationTokens.toLocaleString()} tokens; target budget: ~${budgetTokens.toLocaleString()} tokens; fullness: ~${fullnessPercent.toLocaleString()}%.\nDrop urgency: ${urgency}.\nMaximum drops allowed this run: ${maxDropsAllowed.toLocaleString()} observation${maxDropsAllowed === 1 ? "" : "s"}.\nThis maximum is a hard upper bound, not a target. Drop fewer or none if fewer observations are clearly safe.`;
265
+ const userText = `CURRENT REFLECTIONS:\n${joinOrEmpty(reflections.map(reflectionToSummaryLine))}\n\n${existingObservationsContext}NEW OBSERVATIONS TO EVALUATE FOR DROPPING:\n${joinOrEmpty(observations.map((observation) => observationToDropperLine(observation, coverageTierForObservation(observation, coverageById))))}\n\nObservation pool pressure: ~${observationTokens.toLocaleString()} tokens; target budget: ~${budgetTokens.toLocaleString()} tokens; fullness: ~${fullnessPercent.toLocaleString()}%.\nDrop urgency: ${urgency}.\nMaximum drops allowed this run: ${maxDropsAllowed.toLocaleString()} observation${maxDropsAllowed === 1 ? "" : "s"}.\nThis maximum is a hard upper bound, not a target. Drop fewer or none if fewer observations are clearly safe.`;
170
266
  const prompts: Message[] = [{ role: "user", content: [{ type: "text", text: userText }], timestamp: Date.now() }];
171
267
  const context: AgentContext = { systemPrompt: DROPPER_SYSTEM, messages: [], tools: [dropObservations as AgentTool<any>] };
172
268
  const reasoning = (model as { reasoning?: unknown }).reasoning;
@@ -199,6 +295,28 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
199
295
  }
200
296
  await stream.result();
201
297
  if (agentError && proposedDropIds.length === 0) throw new Error(`Dropper API error: ${agentError}`);
202
- const droppedIds = selectDropCandidates(proposedDropIds, observations, maxDropsAllowed);
298
+ const droppedIds = selectDropCandidates(proposedDropIds, observations, maxDropsAllowed, reflections);
299
+ const reason = droppedIds.length > 0
300
+ ? "selected_nonempty"
301
+ : toolCallCount === 0
302
+ ? "no_tool_call"
303
+ : proposedDropIds.length === 0
304
+ ? "all_filtered"
305
+ : "selected_empty";
306
+ const selectedDropTokens = droppedIds.reduce((sum, id) => sum + (allowed.get(id)?.tokenCount ?? 0), 0);
307
+ debugLog("dropper.result", {
308
+ reason,
309
+ toolCallCount,
310
+ rawRequestedIdsCount,
311
+ missingIdsCount,
312
+ criticalCandidateIdsCount,
313
+ duplicateInRequestCount,
314
+ duplicateInRunCount,
315
+ acceptedCandidateCount: proposedDropIds.length,
316
+ selectedDropsCount: droppedIds.length,
317
+ selectedDropTokens,
318
+ selectedCoverageSummaryByRelevance: summarizeCoverageByRelevanceForIds(droppedIds, observations, coverageById),
319
+ maxDropsAllowed,
320
+ });
203
321
  return droppedIds.length > 0 ? droppedIds : undefined;
204
322
  }
@@ -0,0 +1,128 @@
1
+ import type { Observation, Reflection } from "../../ledger/index.js";
2
+
3
+ export const REFLECTION_COVERAGE_TIERS = ["none", "partial", "strong"] as const;
4
+ export type ReflectionCoverageTier = typeof REFLECTION_COVERAGE_TIERS[number];
5
+
6
+ type Relevance = Observation["relevance"];
7
+
8
+ type CoverageBucket = Record<ReflectionCoverageTier, { count: number; tokens: number }>;
9
+ export type CoverageSummaryByRelevance = Record<Relevance, CoverageBucket>;
10
+ export type CoverageTransitionSummaryByRelevance = Record<Relevance, Record<string, { count: number; tokens: number }>>;
11
+
12
+ export const REFLECTION_COVERAGE_DROP_RANK: Record<ReflectionCoverageTier, number> = {
13
+ strong: 0,
14
+ partial: 1,
15
+ none: 2,
16
+ };
17
+
18
+ export function reflectionSupportCounts(reflections: readonly Reflection[]): Map<string, number> {
19
+ const counts = new Map<string, number>();
20
+ for (const reflection of reflections) {
21
+ const uniqueIds = new Set(reflection.supportingObservationIds);
22
+ for (const id of uniqueIds) counts.set(id, (counts.get(id) ?? 0) + 1);
23
+ }
24
+ return counts;
25
+ }
26
+
27
+ export function reflectionCoverageTierForCount(count: number): ReflectionCoverageTier {
28
+ if (count <= 0) return "none";
29
+ if (count === 1) return "partial";
30
+ return "strong";
31
+ }
32
+
33
+ export function reflectionCoverageMap(
34
+ observations: readonly Observation[],
35
+ reflections: readonly Reflection[],
36
+ ): Map<string, ReflectionCoverageTier> {
37
+ const counts = reflectionSupportCounts(reflections);
38
+ return new Map(observations.map((observation) => [
39
+ observation.id,
40
+ reflectionCoverageTierForCount(counts.get(observation.id) ?? 0),
41
+ ]));
42
+ }
43
+
44
+ function emptyCoverageBucket(): CoverageBucket {
45
+ return {
46
+ none: { count: 0, tokens: 0 },
47
+ partial: { count: 0, tokens: 0 },
48
+ strong: { count: 0, tokens: 0 },
49
+ };
50
+ }
51
+
52
+ export function emptyCoverageSummaryByRelevance(): CoverageSummaryByRelevance {
53
+ return {
54
+ low: emptyCoverageBucket(),
55
+ medium: emptyCoverageBucket(),
56
+ high: emptyCoverageBucket(),
57
+ critical: emptyCoverageBucket(),
58
+ };
59
+ }
60
+
61
+ export function summarizeCoverageByRelevance(
62
+ observations: readonly Observation[],
63
+ coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
64
+ ): CoverageSummaryByRelevance {
65
+ const summary = emptyCoverageSummaryByRelevance();
66
+ for (const observation of observations) {
67
+ const tier = coverageById.get(observation.id) ?? "none";
68
+ const bucket = summary[observation.relevance][tier];
69
+ bucket.count++;
70
+ bucket.tokens += observation.tokenCount;
71
+ }
72
+ return summary;
73
+ }
74
+
75
+ export function summarizeCoverageByRelevanceForIds(
76
+ ids: readonly string[],
77
+ observations: readonly Observation[],
78
+ coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
79
+ ): CoverageSummaryByRelevance {
80
+ const byId = new Map(observations.map((observation) => [observation.id, observation]));
81
+ const selected = ids.flatMap((id) => {
82
+ const observation = byId.get(id);
83
+ return observation ? [observation] : [];
84
+ });
85
+ return summarizeCoverageByRelevance(selected, coverageById);
86
+ }
87
+
88
+ export function emptyCoverageTransitionSummaryByRelevance(): CoverageTransitionSummaryByRelevance {
89
+ return {
90
+ low: {},
91
+ medium: {},
92
+ high: {},
93
+ critical: {},
94
+ };
95
+ }
96
+
97
+ export function summarizeCoverageTransitionsByRelevance(
98
+ observations: readonly Observation[],
99
+ beforeCoverageById: ReadonlyMap<string, ReflectionCoverageTier>,
100
+ afterCoverageById: ReadonlyMap<string, ReflectionCoverageTier>,
101
+ ): CoverageTransitionSummaryByRelevance {
102
+ const summary = emptyCoverageTransitionSummaryByRelevance();
103
+ for (const observation of observations) {
104
+ const before = beforeCoverageById.get(observation.id) ?? "none";
105
+ const after = afterCoverageById.get(observation.id) ?? "none";
106
+ if (before === after) continue;
107
+ const key = `${before}->${after}`;
108
+ const bucket = summary[observation.relevance][key] ?? { count: 0, tokens: 0 };
109
+ bucket.count++;
110
+ bucket.tokens += observation.tokenCount;
111
+ summary[observation.relevance][key] = bucket;
112
+ }
113
+ return summary;
114
+ }
115
+
116
+ export function observationToDropperLine(
117
+ observation: Observation,
118
+ coverage: ReflectionCoverageTier,
119
+ ): string {
120
+ return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] [coverage: ${coverage}] ${observation.content}`;
121
+ }
122
+
123
+ export function coverageTierForObservation(
124
+ observation: Observation,
125
+ coverageById: ReadonlyMap<string, ReflectionCoverageTier>,
126
+ ): ReflectionCoverageTier {
127
+ return coverageById.get(observation.id) ?? "none";
128
+ }
@@ -6,30 +6,30 @@ Your job is to identify only the safest active observations to remove from compa
6
6
 
7
7
  Active-memory framing. Dropping an observation removes it from active compacted memory; it does not erase the ledger history or source evidence. Still, future compressed context will no longer show the observation, so only drop it when its durable meaning is safely captured elsewhere or it is genuinely low-signal and carries no unique future value.
8
8
 
9
- The user message includes drop urgency and "Maximum drops allowed this run". The maximum is a hard upper bound, not a target. Never try to hit it. Drop fewer or none when fewer observations are safely removable.
10
-
11
- Urgency guidance:
12
- - low urgency: only propose trivially safe drops, usually low-signal observations with no unique detail.
13
- - medium urgency: perform conservative cleanup; prefer low observations and clearly redundant medium observations.
14
- - high urgency: cleanup is more useful, but preservation rules do not weaken and load-bearing memory must still be kept.
9
+ The user message includes the active observation pool target and "Maximum drops allowed this run". The maximum is a hard upper bound sized to move the pool toward the target if every proposed drop is clearly safe. It is not a target. Do not try to fill it. Drop fewer or none when fewer observations are safely removable. When the active pool is far over target, make a thorough pass over safe candidates rather than stopping after a few obvious examples.
15
10
 
16
11
  What to drop, in priority order:
17
12
  - Redundant observations whose durable meaning is already captured by current reflections with equivalent fidelity.
18
13
  - Superseded observations where a later observation clearly replaces the older state.
19
14
  - Repeated routine tool acknowledgements or low-signal progress updates that do not carry decisions, constraints, exact errors, or user-specific facts.
20
- - Older medium observations that no longer carry working context and are covered by a reflection or a newer observation.
15
+ - Older observations that no longer carry working context and are covered by a reflection or a newer observation.
16
+
17
+ Age-gradient rule. Recent observations carry working context the assistant may still need; older observations have usually been summarized elsewhere or are no longer load-bearing. Prefer older safe drops before newer working context, but age alone is not enough to drop important or uniquely load-bearing observations.
21
18
 
22
- Age-gradient rule. Recent observations carry working context the assistant may still need; older observations have usually been summarized elsewhere or are no longer load-bearing. Prefer older safe drops before newer working context.
19
+ Reflection coverage guidance. Each observation line includes [coverage: none|partial|strong]. Coverage is evidence, not an automatic decision:
20
+ - none: no current reflection cites this observation id. Be cautious, especially for high or critical observations.
21
+ - partial: one current reflection cites this observation id. Compare the observation to the reflection before dropping.
22
+ - strong: two or more current reflections cite this observation id. This is stronger evidence that the durable meaning is preserved, but you must still keep uniquely load-bearing or uncertain observations.
23
23
 
24
- Relevance guidance:
24
+ Relevance guidance. Relevance is importance/resistance, not an absolute keep/drop lock:
25
25
  - low: consider first, but drop only when it carries no unique detail, decision, state, error, identifier, or user-specific fact.
26
26
  - medium: drop when redundant with reflections or other observations, or when the work state is clearly obsolete.
27
27
  - high: drop only when clearly superseded or already captured by a reflection with equivalent fidelity.
28
- - critical: NEVER drop. Code also rejects critical ids, but you must avoid proposing them.
28
+ - critical: highest importance and strongest resistance. Do not drop fresh or uniquely load-bearing critical observations. Critical observations may be dropped only with strong semantic evidence such as age plus partial/strong reflection coverage, supersession by newer memory, redundancy, or clear obsolescence.
29
29
 
30
- User assertions and concrete completions are never droppable, even at non-critical relevance, unless a current reflection preserves the exact assertion/completion and its important details with equivalent fidelity.
30
+ User assertions and concrete completions must be preserved unless a current reflection or newer observation preserves the exact assertion/completion and its important details with equivalent fidelity.
31
31
 
32
- Preservation floor. Regardless of relevance label, urgency, budget pressure, or age, do not drop observations that uniquely carry any of the following:
32
+ Preservation floor. Regardless of relevance label, budget pressure, coverage, or age, do not drop observations that uniquely carry any of the following:
33
33
  - User preferences, constraints, corrections, or identity/role facts.
34
34
  - Concrete completions that future runs must not redo.
35
35
  - Named identifiers, file paths, function names, package names, tickets, commit SHAs, handles, or exact commands.
@@ -101,7 +101,7 @@ If a detail is non-obvious from the code or git history, it belongs in the obser
101
101
 
102
102
  Relevance levels (pick one per observation; this field drives future dropping):
103
103
 
104
- - critical: user assertions about identity, role, or persistent preferences; explicit corrections ("no, don't do X"); concrete completions that future runs MUST NOT redo. These are load-bearing and will NEVER be dropped. Why this matters: if a "critical" item is lost, the assistant may redo finished work, contradict a correction, or misrepresent who the user is.
104
+ - critical: user assertions about identity, role, or persistent preferences; explicit corrections ("no, don't do X"); concrete completions that future runs MUST NOT redo. These are highest-resistance, load-bearing observations and require the strongest evidence before leaving active memory. Why this matters: if a "critical" item is lost, the assistant may redo finished work, contradict a correction, or misrepresent who the user is.
105
105
  - high: non-trivial technical decisions, architectural direction, unresolved blockers, key constraints. Worth keeping across many compactions.
106
106
  - medium: task-level context that helps within the current work but isn't durable. The default when you are unsure between medium and high.
107
107
  - low: routine tool-call acks, repetitive status updates, content trivially re-derivable from recent messages. The dropper will drop these first.
@@ -6,7 +6,8 @@ Your task is different from the observer's: you are not recording events, you ar
6
6
 
7
7
  You receive:
8
8
  - Current reflections: durable facts already crystallized.
9
- - Current observations: active timestamped evidence lines, each shown as "[id] YYYY-MM-DD HH:MM [relevance] content".
9
+ - Current observations: active timestamped evidence lines, each shown as "[id] YYYY-MM-DD HH:MM [relevance] [coverage: none|partial|strong] content".
10
+ - Coverage tiers are review context: none means no current reflection supports the observation id, partial means exactly one current reflection supports it, and strong means two or more current reflections support it. Coverage is not a quota, target, priority score, or instruction to emit reflections.
10
11
 
11
12
  What to emit:
12
13
  - Emit only new durable reflections not already present in current reflections.
@@ -39,13 +40,16 @@ Focus on:
39
40
  - Completed outcomes future runs must not redo.
40
41
  - Durable blockers, invariants, and open decisions that should survive compaction.
41
42
 
42
- Support ids:
43
+ Support ids and coverage stewardship:
43
44
  - Every reflection must include supportingObservationIds from the current observations list.
44
- - supportingObservationIds are a coverage/provenance set: include current observation ids whose durable meaning is preserved by the reflection and can later be treated as redundant active-memory detail.
45
- - supportingObservationIds are not a checklist to cover every observation. They are evidence for a reflection that already passed the durable-value bar.
45
+ - First decide whether the reflection content passes the durable-value bar. Then audit support ids for that already-worthy reflection.
46
+ - supportingObservationIds are a coverage/provenance set and downstream dropper coverage evidence: include all current observation ids whose durable meaning is preserved by the reflection with equivalent fidelity and can later be treated as redundant active-memory detail.
47
+ - supportingObservationIds are not a checklist to cover every observation. Do not add ids merely to improve coverage counts, maximize support ids, maximize strong coverage, or unlock the dropper.
48
+ - False or inflated support ids can cause unsafe downstream dropper pruning, including removal of high-resistance active observations whose meaning was not actually preserved.
46
49
  - Include additional observation ids only when the reflection preserves their durable meaning with equivalent fidelity.
47
50
  - Leave observations unsupported when their details are still active working state, too specific to compress safely, or not yet durable enough.
48
51
  - Do not include observations whose unique exact detail, current task state, user correction, user constraint, or concrete completion is not captured by the reflection.
52
+ - If no candidate reflection passes the durable-value bar, emit zero reflections even when observations have coverage: none.
49
53
  - Never invent observation ids. Proposals with missing, empty, or invalid supportingObservationIds are rejected.
50
54
 
51
55
  User assertions are authoritative. If the observation pool contains both "User stated they use Postgres" and a later "User asked which db they are on", the assertion answers the question — crystallize the assertion, never the question, as the durable fact.
@@ -49,7 +49,10 @@ import {
49
49
  rawTokensSinceReflectionCoverage,
50
50
  reflectionToSummaryLine,
51
51
  reflectionsCreatedAfterIndex,
52
+ scoreObservation,
53
+ selectPriorObservations,
52
54
  type Entry,
55
+ type Observation,
53
56
  type Reflection,
54
57
  } from "./ledger/index.js";
55
58
 
@@ -124,6 +127,35 @@ function mergeReflections(existing: Reflection[], additional: Reflection[]): Ref
124
127
  return merged;
125
128
  }
126
129
 
130
+
131
+
132
+ /**
133
+ * Extract all pending observations from accumulated batches that were recorded
134
+ * after a given coverage ID (e.g., the last reflection or drop coverage ID).
135
+ * This is needed in noAutoCompact mode because the reflector/dropper may skip
136
+ * a pipeline cycle, leaving unprocessed batches in observationBatches that
137
+ * should still be served as "new" on subsequent runs.
138
+ */
139
+ function pendingObservationsCreatedAfter(
140
+ pending: any,
141
+ entries: Entry[],
142
+ afterCoversUpToId: string | undefined,
143
+ ): Observation[] {
144
+ const batches = pending.observationBatches ?? [];
145
+ if (!afterCoversUpToId || entryIndexForId(entries, afterCoversUpToId) < 0) {
146
+ return batches.flatMap((b: any) => (b.data as any)?.observations ?? []);
147
+ }
148
+ const afterIdx = entryIndexForId(entries, afterCoversUpToId);
149
+ const newObs: Observation[] = [];
150
+ for (const batch of batches) {
151
+ const batchIdx = entryIndexForId(entries, batch.coversUpToId);
152
+ if (batchIdx >= 0 && batchIdx > afterIdx) {
153
+ newObs.push(...((batch.data as any)?.observations ?? []));
154
+ }
155
+ }
156
+ return newObs;
157
+ }
158
+
127
159
  function anyStageDue(entries: Entry[], runtime: Runtime): boolean {
128
160
  return rawTokensSinceObservationCoverage(entries) >= runtime.config.observeAfterTokens
129
161
  || rawTokensSinceReflectionCoverage(entries) >= runtime.config.reflectAfterTokens
@@ -272,12 +304,40 @@ async function runObserverStage(
272
304
  if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
273
305
  const chunkTokens = Math.ceil(chunk.length / 4);
274
306
 
307
+ const sessionId = ctx.sessionManager.getSessionId();
308
+
275
309
  const memory = fullProjection(entries);
276
- const priorReflections = memory.reflections.map(reflectionToSummaryLine);
277
- const priorObservations = memory.observations.map(observationToSummaryLine);
310
+ let priorReflections = memory.reflections.map(reflectionToSummaryLine);
311
+ let priorObservations = memory.observations.map(observationToSummaryLine);
312
+
313
+ // In noAutoCompact, append accumulated batch history to whatever
314
+ // fullProjection found in the branch (preserving pre-switch markers
315
+ // when transitioning from autoCompact to noAutoCompact mid-session).
316
+ // The preamble is capped via observerPreambleMaxTokens so accumulated
317
+ // observations don't grow unbounded across turns.
318
+ if (runtime.config.noAutoCompact) {
319
+ const pendingCtx = readPendingState(sessionId);
320
+ const accumulatedReflections = (pendingCtx.reflectionBatches ?? [])
321
+ .flatMap(b => (b.data as any).reflections ?? []);
322
+ const accumulatedObservations = (pendingCtx.observationBatches ?? [])
323
+ .flatMap(b => (b.data as any).observations ?? []);
324
+
325
+ // Capped preamble: high always kept, medium/low scored by relevance + recency
326
+ const preambleMaxTokens = runtime.config.observerPreambleMaxTokens > 0
327
+ ? runtime.config.observerPreambleMaxTokens
328
+ : Math.round(runtime.config.observerChunkMaxTokens * 0.3);
329
+ const allObservations = [...memory.observations, ...accumulatedObservations];
330
+ priorObservations = selectPriorObservations(allObservations, preambleMaxTokens)
331
+ .map(observationToSummaryLine);
332
+
333
+ // Reflections are never trimmed — rare and always kept
334
+ priorReflections = [
335
+ ...priorReflections,
336
+ ...accumulatedReflections.map(reflectionToSummaryLine),
337
+ ];
338
+ }
278
339
 
279
340
  // If noAutoCompact: skip if this exact chunk was already processed
280
- const sessionId = ctx.sessionManager.getSessionId();
281
341
  if (runtime.config.noAutoCompact && isObservationChunkPending(sessionId, coversUpToId)) {
282
342
  debugLog("observer.pending_skip", { coversUpToId, sessionId });
283
343
  return "continue";
@@ -379,11 +439,33 @@ async function runReflectorStage(
379
439
  ): Promise<ReflectorStageResult> {
380
440
  const sessionId = ctx.sessionManager.getSessionId();
381
441
  const entries = ctx.sessionManager.getBranch() as Entry[];
382
- const reflectionTokens = rawTokensSinceReflectionCoverage(entries);
383
- if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
384
-
385
- const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
386
- if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
442
+ let reflectionTokens: number;
443
+ let observationCoverageId: string | undefined;
444
+ if (runtime.config.noAutoCompact) {
445
+ const pending = readPendingState(sessionId);
446
+ // Check any accumulated batch for unprocessed observations, not just the latest
447
+ const hasPendingObs = (pending.observationBatches ?? []).some((b: any) => (b.data as any)?.observations?.length);
448
+ if (!hasPendingObs) return { outcome: "continue", sameRunReflections: [] };
449
+ observationCoverageId = pending.observation?.coversUpToId;
450
+ if (pending.reflection?.coversUpToId) {
451
+ const obsIdx = entryIndexForId(entries, pending.observation?.coversUpToId ?? "");
452
+ const refIdx = entryIndexForId(entries, pending.reflection.coversUpToId);
453
+ if (obsIdx >= 0 && refIdx >= 0 && obsIdx <= refIdx) return { outcome: "continue", sameRunReflections: [] };
454
+ if (refIdx >= 0) {
455
+ reflectionTokens = rawTokensAfterIndex(entries, refIdx);
456
+ if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
457
+ } else {
458
+ reflectionTokens = rawTokensSinceObservationCoverage(entries);
459
+ }
460
+ } else {
461
+ reflectionTokens = rawTokensSinceObservationCoverage(entries);
462
+ }
463
+ } else {
464
+ reflectionTokens = rawTokensSinceReflectionCoverage(entries);
465
+ if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
466
+ observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
467
+ if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
468
+ }
387
469
 
388
470
  for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
389
471
  const resolved = await resolveModel("reflector");
@@ -391,12 +473,15 @@ async function runReflectorStage(
391
473
 
392
474
  // Compute ahead for an accurate notification
393
475
  const folded = foldLedger(entries);
394
- const lastReflectionIdx = latestCoverageIndex(entries, OM_REFLECTIONS_RECORDED);
395
- const newObservations = observationsCreatedAfterIndex(entries, lastReflectionIdx);
396
- const newReflections = reflectionsCreatedAfterIndex(entries, lastReflectionIdx);
476
+ const pending = runtime.config.noAutoCompact ? readPendingState(sessionId) : undefined;
477
+ const lastReflectionIdx = pending ? -1 : latestCoverageIndex(entries, OM_REFLECTIONS_RECORDED);
478
+ const newObservations = pending
479
+ ? pendingObservationsCreatedAfter(pending, entries, pending.reflection?.coversUpToId)
480
+ : observationsCreatedAfterIndex(entries, lastReflectionIdx);
481
+ const newReflections = pending ? [] : reflectionsCreatedAfterIndex(entries, lastReflectionIdx);
397
482
  const newItemsTokens = Math.ceil(
398
- (newObservations.reduce((s, o) => s + o.content.length, 0) +
399
- newReflections.reduce((s, r) => s + r.content.length, 0)) / 4
483
+ (newObservations.reduce((s: number, o: any) => s + o.content.length, 0) +
484
+ newReflections.reduce((s: number, r: any) => s + r.content.length, 0)) / 4
400
485
  );
401
486
  const summaryBudget = Math.floor(runtime.config.reflectorInputMaxTokens * 0.15) * 2;
402
487
  const reflectorInputTokens = Math.min(newItemsTokens + summaryBudget, runtime.config.reflectorInputMaxTokens);
@@ -414,13 +499,21 @@ async function runReflectorStage(
414
499
  // Resolve thinking level for the specific model (fallbacks may have their own thinking config)
415
500
  const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "reflector"), stageFallbacks: stageFallbackModels(runtime, "reflector") });
416
501
  try {
417
- // Existing memory summaries for context (capped)
502
+ // Existing memory summaries for context (capped).
503
+ // In noAutoCompact, merge accumulated pending batches with
504
+ // branch data (preserving pre-switch markers).
505
+ const sourceReflections = pending
506
+ ? [...folded.reflections, ...(pending.reflectionBatches ?? []).flatMap((b: any) => (b.data as any)?.reflections ?? [])]
507
+ : folded.reflections;
508
+ const sourceObservations = pending
509
+ ? [...folded.activeObservations, ...(pending.observationBatches ?? []).flatMap((b: any) => (b.data as any)?.observations ?? [])]
510
+ : folded.activeObservations;
418
511
  const existingReflectionsSummary = buildExistingReflectionsSummary(
419
- folded.reflections,
512
+ sourceReflections,
420
513
  Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
421
514
  );
422
515
  const existingObservationsSummary = buildExistingObservationsSummary(
423
- folded.activeObservations.filter(o => !newObservations.some(no => no.id === o.id)),
516
+ sourceObservations.filter((o: any) => !newObservations.some((no: any) => no.id === o.id)),
424
517
  Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
425
518
  );
426
519
 
@@ -437,6 +530,7 @@ async function runReflectorStage(
437
530
  });
438
531
 
439
532
  if (!reflections || reflections.length === 0) return { outcome: "continue", sameRunReflections: [] };
533
+ if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
440
534
 
441
535
  const data = buildReflectionsRecordedData(reflections, observationCoverageId);
442
536
  if (!data) return { outcome: "continue", sameRunReflections: [] };
@@ -474,11 +568,33 @@ async function runDropperStage(
474
568
  ): Promise<StageOutcome> {
475
569
  const sessionId = ctx.sessionManager.getSessionId();
476
570
  const entries = ctx.sessionManager.getBranch() as Entry[];
477
- const dropTokens = rawTokensSinceDropCoverage(entries);
478
- if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
479
-
480
- const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
481
- if (!observationCoverageId) return "continue";
571
+ let dropTokens: number;
572
+ let observationCoverageId: string | undefined;
573
+ if (runtime.config.noAutoCompact) {
574
+ const pending = readPendingState(sessionId);
575
+ // Check any accumulated batch for unprocessed observations, not just the latest
576
+ const hasPendingObs = (pending.observationBatches ?? []).some((b: any) => (b.data as any)?.observations?.length);
577
+ if (!hasPendingObs) return "continue";
578
+ observationCoverageId = pending.observation?.coversUpToId;
579
+ if (pending.dropped?.coversUpToId) {
580
+ const obsIdx = entryIndexForId(entries, pending.observation?.coversUpToId ?? "");
581
+ const dropIdx = entryIndexForId(entries, pending.dropped.coversUpToId);
582
+ if (obsIdx >= 0 && dropIdx >= 0 && obsIdx <= dropIdx) return "continue";
583
+ if (dropIdx >= 0) {
584
+ dropTokens = rawTokensAfterIndex(entries, dropIdx);
585
+ if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
586
+ } else {
587
+ dropTokens = rawTokensSinceDropCoverage(entries);
588
+ }
589
+ } else {
590
+ dropTokens = rawTokensSinceDropCoverage(entries);
591
+ }
592
+ } else {
593
+ dropTokens = rawTokensSinceDropCoverage(entries);
594
+ if (dropTokens < runtime.config.reflectAfterTokens) return "continue";
595
+ observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
596
+ if (!observationCoverageId) return "continue";
597
+ }
482
598
 
483
599
  for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
484
600
  const resolved = await resolveModel("dropper");
@@ -486,10 +602,13 @@ async function runDropperStage(
486
602
 
487
603
  // Compute ahead for an accurate notification
488
604
  const folded = foldLedger(entries);
489
- const lastDropIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_DROPPED);
490
- const newObservations = observationsCreatedAfterIndex(entries, lastDropIdx);
605
+ const pending = runtime.config.noAutoCompact ? readPendingState(sessionId) : undefined;
606
+ const lastDropIdx = pending ? -1 : latestCoverageIndex(entries, OM_OBSERVATIONS_DROPPED);
607
+ const newObservations = pending
608
+ ? pendingObservationsCreatedAfter(pending, entries, pending.dropped?.coversUpToId)
609
+ : observationsCreatedAfterIndex(entries, lastDropIdx);
491
610
  const dropperNewObsTokens = Math.ceil(
492
- newObservations.reduce((s, o) => s + o.content.length, 0) / 4
611
+ newObservations.reduce((s: number, o: any) => s + o.content.length, 0) / 4
493
612
  );
494
613
  const dropperSummaryBudget = Math.floor(runtime.config.dropperInputMaxTokens * 0.2);
495
614
  const dropperInputTokens = Math.min(dropperNewObsTokens + dropperSummaryBudget, runtime.config.dropperInputMaxTokens);
@@ -505,12 +624,23 @@ async function runDropperStage(
505
624
  if (ctx.hasUI) ctx.ui?.notify(`Observational memory: dropper running (~${effectiveDropTokens.toLocaleString()} tokens accumulated, ~${dropperInputTokens.toLocaleString()}-token input)`, "info");
506
625
 
507
626
  try {
508
- // Existing active observations summary for context (capped)
627
+ // Existing active observations summary for context (capped).
628
+ // In noAutoCompact, merge accumulated pending batches with
629
+ // branch data (preserving pre-switch markers).
630
+ const sourceObsForDropper = pending
631
+ ? [...folded.activeObservations, ...(pending.observationBatches ?? []).flatMap((b: any) => (b.data as any)?.observations ?? [])]
632
+ : folded.activeObservations;
509
633
  const existingObservationsSummary = buildExistingObservationsSummary(
510
- folded.activeObservations.filter(o => !newObservations.some(no => no.id === o.id)),
634
+ sourceObsForDropper.filter((o: any) => !newObservations.some((no: any) => no.id === o.id)),
511
635
  Math.floor(runtime.config.dropperInputMaxTokens * 0.2),
512
636
  );
513
- const reflectionsForDropper = mergeReflections(folded.reflections, sameRunReflections);
637
+ // In noAutoCompact, merge accumulated reflection batches with
638
+ // branch data (preserving pre-switch markers), matching the
639
+ // dropper's full autoCompact context.
640
+ const pendingReflections = pending
641
+ ? [...folded.reflections, ...(pending.reflectionBatches ?? []).flatMap((b: any) => (b.data as any)?.reflections ?? [])]
642
+ : folded.reflections;
643
+ const reflectionsForDropper = mergeReflections(pendingReflections, sameRunReflections);
514
644
 
515
645
  // Resolve thinking level for the specific model (fallbacks may have their own thinking config)
516
646
  const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "dropper"), stageFallbacks: stageFallbackModels(runtime, "dropper") });
@@ -525,7 +655,9 @@ async function runDropperStage(
525
655
  maxTurns: runtime.config.agentMaxTurns,
526
656
  thinkingLevel: stageThinkingLevel(runtime, "dropper", stageModelForThinking),
527
657
  });
528
- const latestReflectionCoverageId = latestCoverageMarkerId(entries, OM_REFLECTIONS_RECORDED);
658
+ const latestReflectionCoverageId = runtime.config.noAutoCompact
659
+ ? pending?.reflection?.coversUpToId
660
+ : latestCoverageMarkerId(entries, OM_REFLECTIONS_RECORDED);
529
661
  const effectiveReflectionCoverageId = sameRunReflectionCoverageId ?? latestReflectionCoverageId;
530
662
  const coversUpToId = earlierCoverageMarkerId(entries, observationCoverageId, effectiveReflectionCoverageId);
531
663
  const data = coversUpToId && droppedIds ? buildObservationsDroppedData(droppedIds, coversUpToId) : undefined;
@@ -4,6 +4,7 @@
4
4
  * Upstream: https://github.com/elpapi42/pi-observational-memory (src/session-ledger/projection.ts)
5
5
  * Unmodified.
6
6
  */
7
+ import { selectPriorObservations } from "./render-summary.js";
7
8
  import {
8
9
  OM_FOLDED,
9
10
  isMemoryDetails,
@@ -207,10 +208,21 @@ export function buildCompactionProjection(
207
208
  0,
208
209
  );
209
210
  const fullFold = observationTokens >= config.observationsPoolMaxTokens;
210
- const projection = fullFold
211
+ let projection = fullFold
211
212
  ? fullProjection(entries, firstKeptEntryId)
212
213
  : normalProjection;
213
214
 
215
+ // Cap observations to budget using relevance-tiered + recency scoring.
216
+ // Even if the dropper determined some old observations are worth keeping,
217
+ // this safety valve ensures the compaction output never exceeds the pool
218
+ // token budget. Observations survive in the branch regardless.
219
+ if (config.observationsPoolMaxTokens > 0 && observationTokens >= config.observationsPoolMaxTokens) {
220
+ projection = {
221
+ observations: selectPriorObservations(projection.observations, config.observationsPoolMaxTokens),
222
+ reflections: projection.reflections,
223
+ };
224
+ }
225
+
214
226
  const details: MemoryDetails = {
215
227
  type: OM_FOLDED,
216
228
  version: 1,
@@ -25,6 +25,57 @@ export function observationToSummaryLine(observation: Observation): string {
25
25
  return `[${observation.id}] ${observation.timestamp} [${observation.relevance}] ${observation.content}`;
26
26
  }
27
27
 
28
+ /** Score an observation for cap/trim selection.
29
+ * Relevance tier dominates: medium (5+) always outranks low (max 2).
30
+ * Recency is based on position in the flat-mapped array (0 = oldest, N-1 = newest),
31
+ * avoiding wall-clock dependency that punishes sessions spanning days or weeks. */
32
+ export function scoreObservation(obs: Observation, index: number, total: number): number {
33
+ const base = obs.relevance === "high" || obs.relevance === "critical" ? 10
34
+ : obs.relevance === "medium" ? 5 : 1;
35
+ const recency = total > 1 ? index / (total - 1) : 1;
36
+ return base + recency;
37
+ }
38
+
39
+ /** Select observations up to a token budget, keeping all high-relevance items
40
+ * unconditionally and filling the remaining budget with the best-scoring
41
+ * medium and low observations (relevance-tiered + recency).
42
+ *
43
+ * Reflections are never trimmed — they are inherently rare and always stay.
44
+ * Observations stay in the branch either way; this only caps what is rendered
45
+ * in the compaction summary output. */
46
+ export function selectPriorObservations(observations: Observation[], maxTokens: number): Observation[] {
47
+ // Track original indices so we can restore chronological order after scoring
48
+ const indexed = observations.map((obs, i) => ({ obs, originalIndex: i }));
49
+ const high = indexed.filter(item => item.obs.relevance === "high" || item.obs.relevance === "critical");
50
+ const rest = indexed.filter(item => item.obs.relevance !== "high" && item.obs.relevance !== "critical");
51
+
52
+ // High always kept — consume budget first
53
+ let budget = maxTokens;
54
+ const selected = new Set<{ obs: Observation; originalIndex: number }>();
55
+ for (const item of high) {
56
+ const lineTokens = Math.ceil(observationToSummaryLine(item.obs).length / 4);
57
+ selected.add(item);
58
+ budget -= lineTokens;
59
+ }
60
+
61
+ // Score medium + low and select best within remaining budget
62
+ if (rest.length > 0 && budget > 0) {
63
+ const scored = rest.map((item, i) => ({ item, score: scoreObservation(item.obs, i, rest.length) }));
64
+ scored.sort((a, b) => b.score - a.score); // highest score first
65
+ for (const { item } of scored) {
66
+ const lineTokens = Math.ceil(observationToSummaryLine(item.obs).length / 4);
67
+ if (budget - lineTokens < 0) break;
68
+ selected.add(item);
69
+ budget -= lineTokens;
70
+ }
71
+ }
72
+
73
+ // Restore original chronological order before returning
74
+ return Array.from(selected)
75
+ .sort((a, b) => a.originalIndex - b.originalIndex)
76
+ .map(item => item.obs);
77
+ }
78
+
28
79
  export function reflectionToSummaryLine(reflection: Reflection): string {
29
80
  return `[${reflection.id}] ${reflection.content}`;
30
81
  }
package/src/om/pending.ts CHANGED
@@ -16,6 +16,7 @@
16
16
  import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
17
17
  import { dirname, join } from "node:path";
18
18
  import { getAgentDir } from "@earendil-works/pi-coding-agent";
19
+ import type { Observation, Reflection } from "./ledger/types.js";
19
20
 
20
21
  // ── Types ───────────────────────────────────────────────────────────────────
21
22
 
@@ -41,6 +42,25 @@ export interface PendingOMState {
41
42
  reflection?: PendingReflection;
42
43
  /** Latest dropper run (replaced each time, not accumulated). */
43
44
  dropped?: PendingDropped;
45
+ /**
46
+ * All observation batches accumulated across noAutoCompact pipeline runs.
47
+ * Each batch preserves per-run coverage (coversUpToId) matching the normal
48
+ * branch-marker pattern. Used for LLM context and /blackhole flush.
49
+ */
50
+ observationBatches?: PendingObservation[];
51
+ /**
52
+ * All reflection batches accumulated across noAutoCompact pipeline runs.
53
+ * Preserves per-run coverage for LLM context and /blackhole flush.
54
+ */
55
+ reflectionBatches?: PendingReflection[];
56
+ /**
57
+ * All dropper batches accumulated across noAutoCompact pipeline runs.
58
+ * Each batch preserves which observations were dropped in that run.
59
+ * Without accumulation, earlier drops are lost when the next dropper
60
+ * run overwrites pending.dropped, causing them to be "un-dropped" on
61
+ * /blackhole flush.
62
+ */
63
+ droppedBatches?: PendingDropped[];
44
64
  }
45
65
 
46
66
  // ── Persistence ─────────────────────────────────────────────────────────────
@@ -70,7 +90,10 @@ function defaultState(): PendingOMState {
70
90
  }
71
91
 
72
92
  function isEmptyState(s: PendingOMState): boolean {
73
- return !s.observation && !s.reflection && !s.dropped;
93
+ return !s.observation && !s.reflection && !s.dropped
94
+ && (!s.observationBatches || s.observationBatches.length === 0)
95
+ && (!s.reflectionBatches || s.reflectionBatches.length === 0)
96
+ && (!s.droppedBatches || s.droppedBatches.length === 0);
74
97
  }
75
98
 
76
99
  // ── Per-session file read/write ─────────────────────────────────────────────
@@ -145,6 +168,10 @@ function isPendingOMState(value: unknown): value is PendingOMState {
145
168
  export function savePendingObservation(sessionId: string, entry: PendingObservation): void {
146
169
  const state = readSessionState(sessionId);
147
170
  state.observation = entry;
171
+ // Append to accumulated batches for LLM context and /blackhole flush.
172
+ // Each batch preserves per-run coverage (coversUpToId) matching the
173
+ // normal branch-marker pattern.
174
+ state.observationBatches = [...(state.observationBatches ?? []), entry];
148
175
  writeSessionState(sessionId, state);
149
176
  }
150
177
 
@@ -154,15 +181,20 @@ export function savePendingObservation(sessionId: string, entry: PendingObservat
154
181
  export function savePendingReflection(sessionId: string, entry: PendingReflection): void {
155
182
  const state = readSessionState(sessionId);
156
183
  state.reflection = entry;
184
+ // Append to accumulated batches for LLM context and /blackhole flush.
185
+ state.reflectionBatches = [...(state.reflectionBatches ?? []), entry];
157
186
  writeSessionState(sessionId, state);
158
187
  }
159
188
 
160
189
  /**
161
- * Save (replace) the latest dropper result for a session.
190
+ * Save (replace) the latest dropper result for a session and
191
+ * append to droppedBatches so no drops are lost across cycles
192
+ * before /blackhole flush.
162
193
  */
163
194
  export function savePendingDropped(sessionId: string, entry: PendingDropped): void {
164
195
  const state = readSessionState(sessionId);
165
196
  state.dropped = entry;
197
+ state.droppedBatches = [...(state.droppedBatches ?? []), entry];
166
198
  writeSessionState(sessionId, state);
167
199
  }
168
200