pi-blackhole 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/index.js +11660 -0
- package/dist/index.js.map +1 -0
- package/example-config.json +1 -1
- package/index.ts +37 -63
- package/package.json +21 -9
- package/src/commands/cleanup.ts +279 -240
- package/src/commands/memory.ts +236 -184
- package/src/commands/pi-vcc.ts +202 -152
- package/src/commands/vcc-recall.ts +126 -95
- package/src/core/brief.ts +167 -33
- package/src/core/build-sections.ts +8 -2
- package/src/core/config-env.ts +117 -0
- package/src/core/content.ts +31 -7
- package/src/core/drill-down.ts +41 -11
- package/src/core/filter-noise.ts +9 -3
- package/src/core/format-recall.ts +15 -6
- package/src/core/format.ts +14 -4
- package/src/core/lineage.ts +9 -3
- package/src/core/load-messages.ts +24 -5
- package/src/core/normalize.ts +38 -14
- package/src/core/recall-scope.ts +11 -3
- package/src/core/render-entries.ts +22 -6
- package/src/core/sanitize.ts +5 -1
- package/src/core/search-entries.ts +111 -19
- package/src/core/settings.ts +1 -3
- package/src/core/summarize.ts +42 -21
- package/src/core/unified-config.ts +549 -411
- package/src/extract/commits.ts +4 -2
- package/src/extract/files.ts +10 -5
- package/src/extract/goals.ts +7 -2
- package/src/hooks/before-compact.ts +210 -88
- package/src/om/agents/dropper/agent.ts +380 -265
- package/src/om/agents/dropper/coverage.ts +102 -82
- package/src/om/agents/observer/agent.ts +242 -206
- package/src/om/agents/reflector/agent.ts +212 -153
- package/src/om/cleanup.ts +239 -218
- package/src/om/clipboard.ts +59 -51
- package/src/om/compaction-trigger.ts +448 -333
- package/src/om/config.ts +13 -6
- package/src/om/configure-overlay.ts +518 -355
- package/src/om/consolidation.ts +1460 -953
- package/src/om/cooldown.ts +75 -65
- package/src/om/debug-log.ts +86 -68
- package/src/om/ids.ts +1 -1
- package/src/om/ledger/fold.ts +89 -78
- package/src/om/ledger/progress.ts +181 -153
- package/src/om/ledger/projection.ts +248 -185
- package/src/om/ledger/recall.ts +247 -196
- package/src/om/ledger/render-summary.ts +79 -50
- package/src/om/ledger/types.ts +146 -117
- package/src/om/model-budget.ts +23 -13
- package/src/om/pending.ts +243 -179
- package/src/om/provider-stream.ts +52 -7
- package/src/om/retryable-error.ts +12 -16
- package/src/om/reverse-recall.ts +97 -91
- package/src/om/runtime.ts +474 -375
- package/src/om/serialize.ts +190 -166
- package/src/om/status-overlay.ts +246 -195
- package/src/om/tokens.ts +28 -21
- package/src/pi-base/blackhole-settings.ts +437 -0
- package/src/pi-base/config-manager.ts +440 -0
- package/src/pi-base/config.ts +469 -0
- package/src/pi-base/env.ts +43 -0
- package/src/pi-base/paths.ts +47 -0
- package/src/pi-base/settings/body.ts +1648 -0
- package/src/pi-base/settings/fields/action.ts +43 -0
- package/src/pi-base/settings/fields/boolean.ts +47 -0
- package/src/pi-base/settings/fields/custom.ts +72 -0
- package/src/pi-base/settings/fields/enum.ts +310 -0
- package/src/pi-base/settings/fields/index.ts +46 -0
- package/src/pi-base/settings/fields/model.ts +452 -0
- package/src/pi-base/settings/fields/string.ts +527 -0
- package/src/pi-base/settings/fields/text.ts +115 -0
- package/src/pi-base/settings/frame.ts +197 -0
- package/src/pi-base/settings/index.ts +77 -0
- package/src/pi-base/settings/inline-edit.ts +313 -0
- package/src/pi-base/settings/modal.ts +152 -0
- package/src/pi-base/settings/types.ts +500 -0
- package/src/pi-base/settings/validate-field.ts +113 -0
- package/src/pi-base/shell.ts +117 -0
- package/src/pi-base/types.ts +6 -0
- package/src/pi-base/ui.ts +32 -0
- package/src/tools/recall.ts +347 -225
- package/src/types.ts +20 -3
- package/tsup.config.ts +23 -0
- package/vitest.config.ts +15 -15
package/src/om/consolidation.ts
CHANGED
|
@@ -15,68 +15,74 @@ import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
|
|
|
15
15
|
import type { ConfiguredModel } from "./config.js";
|
|
16
16
|
import { debugLog, withDebugLogContext } from "./debug-log.js";
|
|
17
17
|
import { type ResolveResult, type Runtime } from "./runtime.js";
|
|
18
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
isRetryableError,
|
|
20
|
+
isStaleExtensionContextError,
|
|
21
|
+
} from "./retryable-error.js";
|
|
19
22
|
import { effectiveContextWindow } from "./model-budget.js";
|
|
20
23
|
import { serializeSourceAddressedBranchEntries } from "./serialize.js";
|
|
21
24
|
|
|
22
25
|
/** Fixed overhead for system prompt, tool definitions, and turn scaffold in context window pre-check. */
|
|
23
26
|
const AGENT_LOOP_RESERVE = 8_000;
|
|
24
27
|
import {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
readPendingState,
|
|
29
|
+
savePendingObservation,
|
|
30
|
+
savePendingReflection,
|
|
31
|
+
savePendingDropped,
|
|
32
|
+
isObservationChunkPending,
|
|
33
|
+
PendingOMState,
|
|
31
34
|
} from "./pending.js";
|
|
35
|
+
import { isManualMode } from "../core/unified-config.js";
|
|
32
36
|
import {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
37
|
+
OM_OBSERVATIONS_DROPPED,
|
|
38
|
+
OM_OBSERVATIONS_RECORDED,
|
|
39
|
+
OM_REFLECTIONS_RECORDED,
|
|
40
|
+
buildExistingObservationsSummary,
|
|
41
|
+
buildExistingReflectionsSummary,
|
|
42
|
+
buildObservationsDroppedData,
|
|
43
|
+
buildObservationsRecordedData,
|
|
44
|
+
buildReflectionsRecordedData,
|
|
45
|
+
earlierCoverageMarkerId,
|
|
46
|
+
entryIndexForId,
|
|
47
|
+
foldLedger,
|
|
48
|
+
findLastCompactionIndex,
|
|
49
|
+
fullProjection,
|
|
50
|
+
isSourceEntry,
|
|
51
|
+
latestCoverageIndex,
|
|
52
|
+
latestCoverageMarkerId,
|
|
53
|
+
observationsCreatedAfterIndex,
|
|
54
|
+
observationToSummaryLine,
|
|
55
|
+
rawTokensAfterIndex,
|
|
56
|
+
rawTokensSinceDropCoverage,
|
|
57
|
+
rawTokensSinceObservationCoverage,
|
|
58
|
+
rawTokensSinceReflectionCoverage,
|
|
59
|
+
reflectionToSummaryLine,
|
|
60
|
+
reflectionsCreatedAfterIndex,
|
|
61
|
+
selectPriorObservations,
|
|
62
|
+
type Entry,
|
|
63
|
+
type Observation,
|
|
64
|
+
type Reflection,
|
|
61
65
|
} from "./ledger/index.js";
|
|
62
66
|
|
|
63
67
|
export type ResolvedModel = Extract<ResolveResult, { ok: true }>;
|
|
64
68
|
|
|
65
69
|
export type ConsolidationCtx = {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
cwd: string;
|
|
71
|
+
hasUI: boolean;
|
|
72
|
+
ui?: {
|
|
73
|
+
notify: (message: string, type?: "warning" | "info" | "error") => void;
|
|
74
|
+
};
|
|
75
|
+
model: unknown;
|
|
76
|
+
modelRegistry: any;
|
|
77
|
+
sessionManager: { getBranch: () => unknown; getSessionId: () => string };
|
|
72
78
|
};
|
|
73
79
|
|
|
74
80
|
type StageOutcome = "continue" | "abort";
|
|
75
81
|
|
|
76
82
|
type ReflectorStageResult = {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
83
|
+
outcome: StageOutcome;
|
|
84
|
+
sameRunReflections: Reflection[];
|
|
85
|
+
effectiveReflectionCoverageId?: string;
|
|
80
86
|
};
|
|
81
87
|
|
|
82
88
|
// Max attempts per stage (primary + all fallbacks the runtime will try internally).
|
|
@@ -85,7 +91,7 @@ type ReflectorStageResult = {
|
|
|
85
91
|
const MAX_STAGE_ATTEMPTS = 10;
|
|
86
92
|
|
|
87
93
|
function sourceEntriesAfter(entries: Entry[], index: number): Entry[] {
|
|
88
|
-
|
|
94
|
+
return entries.slice(index + 1).filter(isSourceEntry);
|
|
89
95
|
}
|
|
90
96
|
|
|
91
97
|
/**
|
|
@@ -93,89 +99,102 @@ function sourceEntriesAfter(entries: Entry[], index: number): Entry[] {
|
|
|
93
99
|
* walking backwards until the token budget is exceeded.
|
|
94
100
|
* Uses a conservative chars/4 heuristic for token estimation.
|
|
95
101
|
*/
|
|
96
|
-
function capSourceEntriesToTokens(
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
102
|
+
function capSourceEntriesToTokens(
|
|
103
|
+
entries: Entry[],
|
|
104
|
+
maxTokens: number,
|
|
105
|
+
): Entry[] {
|
|
106
|
+
let totalTokens = 0;
|
|
107
|
+
const kept: Entry[] = [];
|
|
108
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
109
|
+
const entry = entries[i];
|
|
110
|
+
let chars = 0;
|
|
111
|
+
// Tokenize all entry types, not just "message": custom_message and
|
|
112
|
+
// branch_summary entries also consume observer context window.
|
|
113
|
+
if (entry.type === "message" && entry.message) {
|
|
114
|
+
const msg = entry.message as any;
|
|
115
|
+
if (typeof msg.content === "string") chars = msg.content.length;
|
|
116
|
+
else if (Array.isArray(msg.content)) {
|
|
117
|
+
for (const block of msg.content) {
|
|
118
|
+
if (block.text) chars += block.text.length;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
} else if (
|
|
122
|
+
entry.type === "custom" &&
|
|
123
|
+
(entry.customType === OM_OBSERVATIONS_RECORDED ||
|
|
124
|
+
entry.customType === OM_REFLECTIONS_RECORDED ||
|
|
125
|
+
entry.customType === OM_OBSERVATIONS_DROPPED)
|
|
126
|
+
) {
|
|
127
|
+
// Custom entries carry structured data — estimate from JSON serialization
|
|
128
|
+
chars = String(JSON.stringify(entry.data ?? {})).length;
|
|
129
|
+
} else if (entry.summary) {
|
|
130
|
+
chars = String(entry.summary).length;
|
|
131
|
+
}
|
|
132
|
+
const estTokens = Math.ceil(chars / 4);
|
|
133
|
+
if (totalTokens + estTokens > maxTokens && kept.length > 0) break;
|
|
134
|
+
// Remove the `kept.length > 0` guard? No — keep the guard but allow
|
|
135
|
+
// the first entry to be dropped only if it exceeds maxTokens alone.
|
|
136
|
+
// (The guard against empty kept list prevents dropping the first entry
|
|
137
|
+
// when later entries are small; but a single oversized entry should
|
|
138
|
+
// still be included to avoid losing the newest data entirely.)
|
|
139
|
+
if (totalTokens + estTokens > maxTokens && kept.length === 0) {
|
|
140
|
+
// First (newest) entry exceeds maxTokens alone — include it anyway
|
|
141
|
+
// to avoid data loss, but don't add more.
|
|
142
|
+
kept.unshift(entry);
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
kept.unshift(entry);
|
|
146
|
+
totalTokens += estTokens;
|
|
147
|
+
}
|
|
148
|
+
return kept;
|
|
135
149
|
}
|
|
136
150
|
|
|
137
|
-
function appendEntry(
|
|
138
|
-
|
|
151
|
+
function appendEntry(
|
|
152
|
+
pi: ExtensionAPI,
|
|
153
|
+
customType: string,
|
|
154
|
+
data: unknown,
|
|
155
|
+
): void {
|
|
156
|
+
pi.appendEntry(customType, data);
|
|
139
157
|
}
|
|
140
158
|
|
|
141
|
-
function mergeReflections(
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
159
|
+
function mergeReflections(
|
|
160
|
+
existing: Reflection[],
|
|
161
|
+
additional: Reflection[],
|
|
162
|
+
): Reflection[] {
|
|
163
|
+
const seen = new Set(existing.map((reflection) => reflection.id));
|
|
164
|
+
const merged = [...existing];
|
|
165
|
+
for (const reflection of additional) {
|
|
166
|
+
if (seen.has(reflection.id)) continue;
|
|
167
|
+
seen.add(reflection.id);
|
|
168
|
+
merged.push(reflection);
|
|
169
|
+
}
|
|
170
|
+
return merged;
|
|
150
171
|
}
|
|
151
172
|
|
|
152
|
-
|
|
153
|
-
|
|
154
173
|
/**
|
|
155
174
|
* Extract all pending observations from accumulated batches that were recorded
|
|
156
175
|
* after a given coverage ID (e.g., the last reflection or drop coverage ID).
|
|
157
|
-
* This is needed in
|
|
176
|
+
* This is needed in manual mode (pending-based) because the reflector/dropper may skip
|
|
158
177
|
* a pipeline cycle, leaving unprocessed batches in observationBatches that
|
|
159
178
|
* should still be served as "new" on subsequent runs.
|
|
160
179
|
*/
|
|
161
180
|
function pendingObservationsCreatedAfter(
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
181
|
+
pending: PendingOMState,
|
|
182
|
+
entries: Entry[],
|
|
183
|
+
afterCoversUpToId: string | undefined,
|
|
165
184
|
): Observation[] {
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
185
|
+
const batches = pending.observationBatches ?? [];
|
|
186
|
+
if (!afterCoversUpToId || entryIndexForId(entries, afterCoversUpToId) < 0) {
|
|
187
|
+
return batches.flatMap((b: any) => (b.data as any)?.observations ?? []);
|
|
188
|
+
}
|
|
189
|
+
const afterIdx = entryIndexForId(entries, afterCoversUpToId);
|
|
190
|
+
const newObs: Observation[] = [];
|
|
191
|
+
for (const batch of batches) {
|
|
192
|
+
const batchIdx = entryIndexForId(entries, batch.coversUpToId);
|
|
193
|
+
if (batchIdx >= 0 && batchIdx > afterIdx) {
|
|
194
|
+
newObs.push(...((batch.data as any)?.observations ?? []));
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return newObs;
|
|
179
198
|
}
|
|
180
199
|
|
|
181
200
|
/** Cursor-aware stage-due check. Uses cursors when available; falls back to
|
|
@@ -184,887 +203,1375 @@ function pendingObservationsCreatedAfter(
|
|
|
184
203
|
* In compaction: "manual" mode, the branch has no OM markers — observations
|
|
185
204
|
* live in the per‑session pending file. `pending` provides the pool fullness
|
|
186
205
|
* and new‑data visibility that the reflector/dropper checks need. */
|
|
187
|
-
export function anyStageDue(
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
206
|
+
export function anyStageDue(
|
|
207
|
+
entries: Entry[],
|
|
208
|
+
runtime: Runtime,
|
|
209
|
+
pending?: PendingOMState,
|
|
210
|
+
): boolean {
|
|
211
|
+
const config = runtime.config;
|
|
212
|
+
const cursors = runtime.cursors ?? {};
|
|
213
|
+
|
|
214
|
+
// ── Observer ──────────────────────────────────────────────────────────
|
|
215
|
+
const observerDue = (() => {
|
|
216
|
+
const cursor = cursors.observer;
|
|
217
|
+
if (!cursor) {
|
|
218
|
+
return (
|
|
219
|
+
rawTokensSinceObservationCoverage(entries) >= config.observeAfterTokens
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const idx = entryIndexForId(entries, cursor.entryId);
|
|
223
|
+
const tokensSince =
|
|
224
|
+
idx >= 0
|
|
225
|
+
? rawTokensAfterIndex(entries, idx)
|
|
226
|
+
: rawTokensSinceObservationCoverage(entries);
|
|
227
|
+
return tokensSince >= config.observeAfterTokens;
|
|
228
|
+
})();
|
|
229
|
+
|
|
230
|
+
// ── Reflector ─────────────────────────────────────────────────────────
|
|
231
|
+
const reflectorDue = (() => {
|
|
232
|
+
const cursor = cursors.reflector;
|
|
233
|
+
if (!cursor) {
|
|
234
|
+
return (
|
|
235
|
+
rawTokensSinceReflectionCoverage(entries) >= config.reflectAfterTokens
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
const idx = entryIndexForId(entries, cursor.entryId);
|
|
239
|
+
if (idx < 0) {
|
|
240
|
+
return (
|
|
241
|
+
rawTokensSinceReflectionCoverage(entries) >= config.reflectAfterTokens
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
// Must have enough accumulated tokens before considering reflector
|
|
245
|
+
const tokensSince = rawTokensAfterIndex(entries, idx);
|
|
246
|
+
if (tokensSince < config.reflectAfterTokens) {
|
|
247
|
+
return false;
|
|
248
|
+
}
|
|
249
|
+
// Check for new observation batches after the cursor
|
|
250
|
+
for (let i = idx + 1; i < entries.length; i++) {
|
|
251
|
+
const e = entries[i];
|
|
252
|
+
if (e.type === "custom" && e.customType === OM_OBSERVATIONS_RECORDED) {
|
|
253
|
+
// Skip if this marker's coversUpToId is at or before the cursor
|
|
254
|
+
// — data it covers was already processed.
|
|
255
|
+
const markerCoversUpTo: string | undefined = (e as any).data
|
|
256
|
+
?.coversUpToId;
|
|
257
|
+
if (markerCoversUpTo) {
|
|
258
|
+
const markerCoversIdx = entryIndexForId(entries, markerCoversUpTo);
|
|
259
|
+
if (markerCoversIdx >= 0 && markerCoversIdx <= idx) continue;
|
|
260
|
+
}
|
|
261
|
+
return true;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
// In manual mode, also check pending observation batches that arrived
|
|
265
|
+
// after the cursor (since branch has no OM markers).
|
|
266
|
+
if (pending) {
|
|
267
|
+
const pendingBatches = pending.observationBatches ?? [];
|
|
268
|
+
for (const batch of pendingBatches) {
|
|
269
|
+
if (batch.coversUpToId) {
|
|
270
|
+
const batchIdx = entryIndexForId(entries, batch.coversUpToId);
|
|
271
|
+
if (batchIdx >= 0 && batchIdx > idx) return true;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return false;
|
|
276
|
+
})();
|
|
277
|
+
|
|
278
|
+
// ── Dropper ───────────────────────────────────────────────────────────
|
|
279
|
+
// Short‑circuit: only compute dropperDue when observer and reflector are
|
|
280
|
+
// both not due — if either is due, the pipeline launches anyway.
|
|
281
|
+
const dropperDue =
|
|
282
|
+
observerDue || reflectorDue
|
|
283
|
+
? false
|
|
284
|
+
: (() => {
|
|
285
|
+
// Compute active observation pool tokens (branch + pending in manual mode)
|
|
286
|
+
const folded = foldLedger(entries);
|
|
287
|
+
let poolTokens = folded.activeObservations.reduce(
|
|
288
|
+
(s: number, o: Observation) => s + (o.tokenCount ?? 0),
|
|
289
|
+
0,
|
|
290
|
+
);
|
|
291
|
+
// In manual mode, include pending observation batches
|
|
292
|
+
if (pending) {
|
|
293
|
+
const pendingBatches = pending.observationBatches ?? [];
|
|
294
|
+
for (const batch of pendingBatches) {
|
|
295
|
+
poolTokens += ((batch.data as any)?.observations ?? []).reduce(
|
|
296
|
+
(s: number, o: any) => s + (o.tokenCount ?? 0),
|
|
297
|
+
0,
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const fullnessVsPool =
|
|
302
|
+
config.observationsPoolMaxTokens > 0
|
|
303
|
+
? poolTokens / config.observationsPoolMaxTokens
|
|
304
|
+
: 0;
|
|
305
|
+
|
|
306
|
+
// Must have at least dropperPoolFullnessThreshold fullness to consider dropper
|
|
307
|
+
if (fullnessVsPool < (config.dropperPoolFullnessThreshold ?? 0.1))
|
|
308
|
+
return false;
|
|
309
|
+
|
|
310
|
+
// Pressure check: pool ≥ threshold × reflectorInputMaxTokens
|
|
311
|
+
const pressure =
|
|
312
|
+
poolTokens >=
|
|
313
|
+
config.dropperPressureThreshold * config.reflectorInputMaxTokens;
|
|
314
|
+
if (pressure) return true;
|
|
315
|
+
|
|
316
|
+
// New data check: new obs or ref batches after dropper cursor
|
|
317
|
+
const cursor = cursors.dropper;
|
|
318
|
+
if (!cursor) {
|
|
319
|
+
// In manual mode, pending batches are the only source of new‑data
|
|
320
|
+
// visibility (branch has no OM markers).
|
|
321
|
+
const hasPendingNewData = pending
|
|
322
|
+
? (pending.observationBatches?.length ?? 0) > 0 ||
|
|
323
|
+
(pending.reflectionBatches?.length ?? 0) > 0
|
|
324
|
+
: false;
|
|
325
|
+
if (hasPendingNewData) return true;
|
|
326
|
+
return (
|
|
327
|
+
rawTokensSinceDropCoverage(entries) >= config.reflectAfterTokens
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
const idx = entryIndexForId(entries, cursor.entryId);
|
|
331
|
+
if (idx < 0) {
|
|
332
|
+
return (
|
|
333
|
+
rawTokensSinceDropCoverage(entries) >= config.reflectAfterTokens
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
// Must have enough accumulated tokens before considering dropper
|
|
337
|
+
const tokensSince = rawTokensAfterIndex(entries, idx);
|
|
338
|
+
if (tokensSince < config.reflectAfterTokens) {
|
|
339
|
+
return false;
|
|
340
|
+
}
|
|
341
|
+
for (let i = idx + 1; i < entries.length; i++) {
|
|
342
|
+
const e = entries[i];
|
|
343
|
+
if (
|
|
344
|
+
e.type === "custom" &&
|
|
345
|
+
(e.customType === OM_OBSERVATIONS_RECORDED ||
|
|
346
|
+
e.customType === OM_REFLECTIONS_RECORDED)
|
|
347
|
+
) {
|
|
348
|
+
const markerCoversUpTo: string | undefined = (e as any).data
|
|
349
|
+
?.coversUpToId;
|
|
350
|
+
if (markerCoversUpTo) {
|
|
351
|
+
const markerCoversIdx = entryIndexForId(
|
|
352
|
+
entries,
|
|
353
|
+
markerCoversUpTo,
|
|
354
|
+
);
|
|
355
|
+
if (markerCoversIdx >= 0 && markerCoversIdx <= idx) continue;
|
|
356
|
+
}
|
|
357
|
+
return true;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
// In manual mode, also check pending batches after the cursor
|
|
361
|
+
if (pending) {
|
|
362
|
+
const pendingObs = pending.observationBatches ?? [];
|
|
363
|
+
const pendingRef = pending.reflectionBatches ?? [];
|
|
364
|
+
for (const batch of [...pendingObs, ...pendingRef]) {
|
|
365
|
+
if (batch.coversUpToId) {
|
|
366
|
+
const batchIdx = entryIndexForId(entries, batch.coversUpToId);
|
|
367
|
+
if (batchIdx >= 0 && batchIdx > idx) return true;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
return false;
|
|
372
|
+
})();
|
|
373
|
+
|
|
374
|
+
return observerDue || reflectorDue || dropperDue;
|
|
323
375
|
}
|
|
324
376
|
|
|
325
|
-
function stageModelConfig(
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
377
|
+
function stageModelConfig(
|
|
378
|
+
runtime: Runtime,
|
|
379
|
+
stage: "observer" | "reflector" | "dropper",
|
|
380
|
+
): ConfiguredModel | undefined {
|
|
381
|
+
if (stage === "observer") return runtime.config.observerModel;
|
|
382
|
+
if (stage === "reflector") return runtime.config.reflectorModel;
|
|
383
|
+
return runtime.config.dropperModel;
|
|
329
384
|
}
|
|
330
385
|
|
|
331
|
-
function stageFallbackModels(
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
386
|
+
function stageFallbackModels(
|
|
387
|
+
runtime: Runtime,
|
|
388
|
+
stage: "observer" | "reflector" | "dropper",
|
|
389
|
+
): ConfiguredModel[] {
|
|
390
|
+
if (stage === "observer") return runtime.config.observerFallbackModels ?? [];
|
|
391
|
+
if (stage === "reflector")
|
|
392
|
+
return runtime.config.reflectorFallbackModels ?? [];
|
|
393
|
+
return runtime.config.dropperFallbackModels ?? [];
|
|
335
394
|
}
|
|
336
395
|
|
|
337
|
-
function stageThinkingLevel(
|
|
338
|
-
|
|
339
|
-
|
|
396
|
+
function stageThinkingLevel(
|
|
397
|
+
runtime: Runtime,
|
|
398
|
+
stage: "observer" | "reflector" | "dropper",
|
|
399
|
+
modelConfig?: ConfiguredModel,
|
|
400
|
+
): ModelThinkingLevel {
|
|
401
|
+
const stageModel = modelConfig ?? stageModelConfig(runtime, stage);
|
|
402
|
+
return stageModel?.thinking ?? runtime.config.model?.thinking ?? "low";
|
|
340
403
|
}
|
|
341
404
|
|
|
342
|
-
export function makeModelResolver(
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
405
|
+
export function makeModelResolver(
|
|
406
|
+
runtime: Runtime,
|
|
407
|
+
ctx: ConsolidationCtx,
|
|
408
|
+
): (
|
|
409
|
+
stage: "observer" | "reflector" | "dropper",
|
|
410
|
+
) => Promise<ResolvedModel | undefined> {
|
|
411
|
+
return async (stage) => {
|
|
412
|
+
const stageFallbacks = stageFallbackModels(runtime, stage);
|
|
413
|
+
const resolved = await runtime.resolveModel({
|
|
414
|
+
model: ctx.model,
|
|
415
|
+
modelRegistry: ctx.modelRegistry,
|
|
416
|
+
hasUI: ctx.hasUI,
|
|
417
|
+
ui: ctx.ui,
|
|
418
|
+
stageModel: stageModelConfig(runtime, stage),
|
|
419
|
+
stageFallbacks,
|
|
420
|
+
});
|
|
421
|
+
if (resolved.ok) {
|
|
422
|
+
runtime.resolveFailureNotified = false;
|
|
423
|
+
return resolved;
|
|
424
|
+
}
|
|
425
|
+
debugLog(`${stage}.model_unavailable`, { reason: resolved.reason });
|
|
426
|
+
if (!runtime.resolveFailureNotified && ctx.hasUI && ctx.ui) {
|
|
427
|
+
if (
|
|
428
|
+
runtime.failedInCycle.size > 0 &&
|
|
429
|
+
resolved.reason.includes("all candidates exhausted")
|
|
430
|
+
) {
|
|
431
|
+
const fallbackMsg =
|
|
432
|
+
stageFallbacks.length === 0
|
|
433
|
+
? "no fallbacks configured"
|
|
434
|
+
: "no available fallbacks";
|
|
435
|
+
runtime.tryEmitInfo(
|
|
436
|
+
true,
|
|
437
|
+
ctx.ui,
|
|
438
|
+
`Observational memory: ${stage} skipped — model unavailable (cooldown set to 0, ${fallbackMsg}, will retry next run)`,
|
|
439
|
+
);
|
|
440
|
+
} else {
|
|
441
|
+
ctx.ui.notify(
|
|
442
|
+
`Observational memory: ${stage} skipped — ${resolved.reason}`,
|
|
443
|
+
"warning",
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
runtime.resolveFailureNotified = true;
|
|
447
|
+
}
|
|
448
|
+
return undefined;
|
|
449
|
+
};
|
|
372
450
|
}
|
|
373
451
|
|
|
374
452
|
// ── Trigger registration ────────────────────────────────────────────────────
|
|
375
453
|
|
|
376
|
-
export function registerConsolidationTrigger(
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
454
|
+
export function registerConsolidationTrigger(
|
|
455
|
+
pi: ExtensionAPI,
|
|
456
|
+
runtime: Runtime,
|
|
457
|
+
): void {
|
|
458
|
+
const launch = (_event: unknown, ctx: ConsolidationCtx) => {
|
|
459
|
+
maybeLaunchConsolidation(pi, runtime, ctx);
|
|
460
|
+
};
|
|
461
|
+
pi.on("agent_start", launch);
|
|
462
|
+
pi.on("turn_end", launch);
|
|
382
463
|
}
|
|
383
464
|
|
|
384
465
|
/** Validate cursors against the current branch. If a cursor's entry ID no longer
|
|
385
466
|
* exists in the branch (fork, navigation, compaction), fall back to the best
|
|
386
467
|
* available coverage marker for that stage. */
|
|
387
468
|
function validateCursors(entries: Entry[], runtime: Runtime): void {
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
469
|
+
const cursors = runtime.cursors ?? {};
|
|
470
|
+
|
|
471
|
+
// Observer: fall back to latest OM_OBSERVATIONS_RECORDED marker
|
|
472
|
+
if (
|
|
473
|
+
cursors.observer &&
|
|
474
|
+
entryIndexForId(entries, cursors.observer.entryId) < 0
|
|
475
|
+
) {
|
|
476
|
+
const markerId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
|
|
477
|
+
if (markerId) {
|
|
478
|
+
cursors.observer = { entryId: markerId, state: "initial" };
|
|
479
|
+
} else {
|
|
480
|
+
delete cursors.observer;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
// Reflector: fall back to latest OM_REFLECTIONS_RECORDED marker
|
|
485
|
+
if (
|
|
486
|
+
cursors.reflector &&
|
|
487
|
+
entryIndexForId(entries, cursors.reflector.entryId) < 0
|
|
488
|
+
) {
|
|
489
|
+
const markerId = latestCoverageMarkerId(entries, OM_REFLECTIONS_RECORDED);
|
|
490
|
+
if (markerId) {
|
|
491
|
+
cursors.reflector = { entryId: markerId, state: "initial" };
|
|
492
|
+
} else {
|
|
493
|
+
delete cursors.reflector;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// Dropper: fall back to latest OM_OBSERVATIONS_DROPPED marker
|
|
498
|
+
if (
|
|
499
|
+
cursors.dropper &&
|
|
500
|
+
entryIndexForId(entries, cursors.dropper.entryId) < 0
|
|
501
|
+
) {
|
|
502
|
+
const markerId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_DROPPED);
|
|
503
|
+
if (markerId) {
|
|
504
|
+
cursors.dropper = { entryId: markerId, state: "initial" };
|
|
505
|
+
} else {
|
|
506
|
+
delete cursors.dropper;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
419
509
|
}
|
|
420
510
|
|
|
421
|
-
function maybeLaunchConsolidation(
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
511
|
+
function maybeLaunchConsolidation(
|
|
512
|
+
pi: ExtensionAPI,
|
|
513
|
+
runtime: Runtime,
|
|
514
|
+
ctx: ConsolidationCtx,
|
|
515
|
+
): void {
|
|
516
|
+
runtime.ensureConfig(ctx.cwd, (msg) => ctx.ui?.notify?.(msg, "warning"));
|
|
517
|
+
if (runtime.config.memory === false) return;
|
|
518
|
+
|
|
519
|
+
// LEGACY: passive check — only applies when new keys are absent (unmigrated config)
|
|
520
|
+
if (
|
|
521
|
+
runtime.config.compaction === undefined &&
|
|
522
|
+
runtime.config.compactionEngine === undefined
|
|
523
|
+
) {
|
|
524
|
+
if (runtime.config.passive === true) return;
|
|
525
|
+
}
|
|
526
|
+
if (runtime.consolidationInFlight) return;
|
|
527
|
+
if (runtime.isConsolidationRetryGated()) return;
|
|
528
|
+
|
|
529
|
+
// Load and validate cursors from pending file (once per session; re-load on fork)
|
|
530
|
+
let sessionId: string;
|
|
531
|
+
try {
|
|
532
|
+
sessionId = ctx.sessionManager.getSessionId();
|
|
533
|
+
} catch (error) {
|
|
534
|
+
if (isStaleExtensionContextError(error)) return;
|
|
535
|
+
throw error;
|
|
536
|
+
}
|
|
537
|
+
if (runtime.cursorsLoadedSessionId !== sessionId) {
|
|
538
|
+
if (typeof runtime.loadCursorsFromPending === "function") {
|
|
539
|
+
runtime.loadCursorsFromPending(sessionId);
|
|
540
|
+
}
|
|
541
|
+
let entries: Entry[];
|
|
542
|
+
try {
|
|
543
|
+
entries = ctx.sessionManager.getBranch() as Entry[];
|
|
544
|
+
} catch (error) {
|
|
545
|
+
if (isStaleExtensionContextError(error)) return;
|
|
546
|
+
throw error;
|
|
547
|
+
}
|
|
548
|
+
validateCursors(entries, runtime);
|
|
549
|
+
runtime.cursorsLoadedSessionId = sessionId;
|
|
550
|
+
const c = runtime.cursors ?? {};
|
|
551
|
+
debugLog("cursor.loaded", {
|
|
552
|
+
observer: c.observer ?? null,
|
|
553
|
+
reflector: c.reflector ?? null,
|
|
554
|
+
dropper: c.dropper ?? null,
|
|
555
|
+
});
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
let entries: Entry[];
|
|
559
|
+
try {
|
|
560
|
+
entries = ctx.sessionManager.getBranch() as Entry[];
|
|
561
|
+
} catch (error) {
|
|
562
|
+
if (isStaleExtensionContextError(error)) return;
|
|
563
|
+
throw error;
|
|
564
|
+
}
|
|
565
|
+
// In manual mode, the branch has no OM markers — pending state provides
|
|
566
|
+
// pool fullness and new‑data visibility for reflector/dropper checks.
|
|
567
|
+
const pending = isManualMode(runtime.config)
|
|
568
|
+
? readPendingState(sessionId)
|
|
569
|
+
: undefined;
|
|
570
|
+
if (!anyStageDue(entries, runtime, pending)) return;
|
|
571
|
+
|
|
572
|
+
const runId = `consolidation-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
|
|
573
|
+
const consolidationCtx: ConsolidationCtx = {
|
|
574
|
+
cwd: ctx.cwd,
|
|
575
|
+
hasUI: ctx.hasUI,
|
|
576
|
+
ui: ctx.ui,
|
|
577
|
+
model: ctx.model,
|
|
578
|
+
modelRegistry: ctx.modelRegistry,
|
|
579
|
+
sessionManager: ctx.sessionManager,
|
|
580
|
+
};
|
|
581
|
+
|
|
582
|
+
void runtime.launchConsolidationTask(ctx, async () =>
|
|
583
|
+
withDebugLogContext(
|
|
584
|
+
{ enabled: runtime.config.debugLog === true, cwd: ctx.cwd, runId },
|
|
585
|
+
async () => {
|
|
586
|
+
await runConsolidationPipeline(pi, runtime, consolidationCtx);
|
|
587
|
+
},
|
|
588
|
+
),
|
|
589
|
+
);
|
|
486
590
|
}
|
|
487
591
|
|
|
488
592
|
// ── Pipeline ─────────────────────────────────────────────────────────────────
|
|
489
593
|
|
|
490
594
|
export async function runConsolidationPipeline(
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
595
|
+
pi: ExtensionAPI,
|
|
596
|
+
runtime: Runtime,
|
|
597
|
+
ctx: ConsolidationCtx,
|
|
494
598
|
): Promise<void> {
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
599
|
+
const resolveModel = makeModelResolver(runtime, ctx);
|
|
600
|
+
|
|
601
|
+
runtime.consolidationPhase = "observer";
|
|
602
|
+
runtime.failedInCycle.clear();
|
|
603
|
+
runtime.resolveFailureNotified = false;
|
|
604
|
+
try {
|
|
605
|
+
const observerOutcome = await runObserverStage(
|
|
606
|
+
pi,
|
|
607
|
+
runtime,
|
|
608
|
+
ctx,
|
|
609
|
+
resolveModel,
|
|
610
|
+
);
|
|
611
|
+
if (observerOutcome === "abort") return;
|
|
612
|
+
} catch (error) {
|
|
613
|
+
debugLog("observer.error", {
|
|
614
|
+
errorMessage: runtime.recordConsolidationStageError(
|
|
615
|
+
ctx,
|
|
616
|
+
"observer",
|
|
617
|
+
error,
|
|
618
|
+
),
|
|
619
|
+
});
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
runtime.consolidationPhase = "reflector";
|
|
624
|
+
runtime.failedInCycle.clear();
|
|
625
|
+
runtime.resolveFailureNotified = false;
|
|
626
|
+
let reflectorResult: ReflectorStageResult;
|
|
627
|
+
try {
|
|
628
|
+
reflectorResult = await runReflectorStage(pi, runtime, ctx, resolveModel);
|
|
629
|
+
if (reflectorResult.outcome === "abort") return;
|
|
630
|
+
} catch (error) {
|
|
631
|
+
debugLog("reflector.error", {
|
|
632
|
+
errorMessage: runtime.recordConsolidationStageError(
|
|
633
|
+
ctx,
|
|
634
|
+
"reflector",
|
|
635
|
+
error,
|
|
636
|
+
),
|
|
637
|
+
});
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
runtime.consolidationPhase = "dropper";
|
|
642
|
+
runtime.failedInCycle.clear();
|
|
643
|
+
runtime.resolveFailureNotified = false;
|
|
644
|
+
try {
|
|
645
|
+
await runDropperStage(
|
|
646
|
+
pi,
|
|
647
|
+
runtime,
|
|
648
|
+
ctx,
|
|
649
|
+
resolveModel,
|
|
650
|
+
reflectorResult.sameRunReflections,
|
|
651
|
+
reflectorResult.effectiveReflectionCoverageId,
|
|
652
|
+
);
|
|
653
|
+
} catch (error) {
|
|
654
|
+
debugLog("dropper.error", {
|
|
655
|
+
errorMessage: runtime.recordConsolidationStageError(
|
|
656
|
+
ctx,
|
|
657
|
+
"dropper",
|
|
658
|
+
error,
|
|
659
|
+
),
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// Flush cursors to pending file after all stages complete (non‑blocking)
|
|
664
|
+
let sessionId: string;
|
|
665
|
+
try {
|
|
666
|
+
sessionId = ctx.sessionManager.getSessionId();
|
|
667
|
+
} catch (error) {
|
|
668
|
+
if (isStaleExtensionContextError(error)) {
|
|
669
|
+
debugLog("pipeline.stale_ctx", { error: String(error) });
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
throw error;
|
|
673
|
+
}
|
|
674
|
+
runtime.scheduleCursorFlush(sessionId);
|
|
675
|
+
const c = runtime.cursors ?? {};
|
|
676
|
+
debugLog("cursor.saved", {
|
|
677
|
+
observer: c.observer ?? null,
|
|
678
|
+
reflector: c.reflector ?? null,
|
|
679
|
+
dropper: c.dropper ?? null,
|
|
680
|
+
});
|
|
547
681
|
}
|
|
548
682
|
|
|
549
683
|
// ── Observer stage (with fallback) ──────────────────────────────────────────
|
|
550
684
|
|
|
551
685
|
async function runObserverStage(
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
686
|
+
pi: ExtensionAPI,
|
|
687
|
+
runtime: Runtime,
|
|
688
|
+
ctx: ConsolidationCtx,
|
|
689
|
+
resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
|
|
556
690
|
): Promise<StageOutcome> {
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
691
|
+
let entries: Entry[];
|
|
692
|
+
let sessionId: string;
|
|
693
|
+
try {
|
|
694
|
+
entries = ctx.sessionManager.getBranch() as Entry[];
|
|
695
|
+
sessionId = ctx.sessionManager.getSessionId();
|
|
696
|
+
} catch (error) {
|
|
697
|
+
if (isStaleExtensionContextError(error)) {
|
|
698
|
+
debugLog("observer.stale_ctx", { error: String(error) });
|
|
699
|
+
return "abort";
|
|
700
|
+
}
|
|
701
|
+
throw error;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
// Determine start index: cursor takes priority, fall back to coverage markers
|
|
705
|
+
const observerCursor = runtime.getCursor("observer");
|
|
706
|
+
let effectiveStart: number;
|
|
707
|
+
if (observerCursor) {
|
|
708
|
+
const cursorIdx = entryIndexForId(entries, observerCursor.entryId);
|
|
709
|
+
effectiveStart =
|
|
710
|
+
cursorIdx >= 0
|
|
711
|
+
? cursorIdx
|
|
712
|
+
: latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
|
|
713
|
+
} else {
|
|
714
|
+
const lastCoverageIdx = latestCoverageIndex(
|
|
715
|
+
entries,
|
|
716
|
+
OM_OBSERVATIONS_RECORDED,
|
|
717
|
+
);
|
|
718
|
+
effectiveStart =
|
|
719
|
+
lastCoverageIdx >= 0 ? lastCoverageIdx : findLastCompactionIndex(entries);
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
const tokens =
|
|
723
|
+
effectiveStart >= 0 ? rawTokensAfterIndex(entries, effectiveStart) : 0;
|
|
724
|
+
if (tokens < runtime.config.observeAfterTokens) {
|
|
725
|
+
// Not due — advance cursor to last source entry so we don't re-check immediately
|
|
726
|
+
const lastSourceId = [...entries]
|
|
727
|
+
.reverse()
|
|
728
|
+
.find((e: Entry) => isSourceEntry(e))?.id;
|
|
729
|
+
if (lastSourceId)
|
|
730
|
+
runtime.advanceCursor("observer", lastSourceId, "not_due");
|
|
731
|
+
return "continue";
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
let chunkEntries = sourceEntriesAfter(entries, effectiveStart);
|
|
735
|
+
|
|
736
|
+
// Cap observer input to observerChunkMaxTokens (newest-to-oldest)
|
|
737
|
+
const maxChunkTokens = runtime.config.observerChunkMaxTokens;
|
|
738
|
+
if (tokens > maxChunkTokens) {
|
|
739
|
+
chunkEntries = capSourceEntriesToTokens(chunkEntries, maxChunkTokens);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// coversUpToId must point to the LAST entry AFTER capping, not before
|
|
743
|
+
const coversUpToId = chunkEntries.at(-1)?.id;
|
|
744
|
+
if (!coversUpToId) return "continue";
|
|
745
|
+
|
|
746
|
+
const { text: chunk, sourceEntryIds } =
|
|
747
|
+
serializeSourceAddressedBranchEntries(chunkEntries);
|
|
748
|
+
if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
|
|
749
|
+
const chunkTokens = Math.ceil(chunk.length / 4);
|
|
750
|
+
|
|
751
|
+
const memory = fullProjection(entries);
|
|
752
|
+
let priorReflections = memory.reflections.map(reflectionToSummaryLine);
|
|
753
|
+
let priorObservations = memory.observations.map(observationToSummaryLine);
|
|
754
|
+
|
|
755
|
+
// In manual mode, append accumulated batch history to whatever
|
|
756
|
+
// fullProjection found in the branch (preserving pre-switch markers
|
|
757
|
+
// when transitioning from autoCompact to manual mode mid-session).
|
|
758
|
+
// The preamble is capped via observerPreambleMaxTokens so accumulated
|
|
759
|
+
// observations don't grow unbounded across turns.
|
|
760
|
+
if (isManualMode(runtime.config)) {
|
|
761
|
+
const pendingCtx = readPendingState(sessionId);
|
|
762
|
+
const accumulatedReflections = (pendingCtx.reflectionBatches ?? []).flatMap(
|
|
763
|
+
(b) => (b.data as any).reflections ?? [],
|
|
764
|
+
);
|
|
765
|
+
const accumulatedObservations = (
|
|
766
|
+
pendingCtx.observationBatches ?? []
|
|
767
|
+
).flatMap((b) => (b.data as any).observations ?? []);
|
|
768
|
+
|
|
769
|
+
// Capped preamble: high always kept, medium/low scored by relevance + recency
|
|
770
|
+
const preambleMaxTokens =
|
|
771
|
+
runtime.config.observerPreambleMaxTokens > 0
|
|
772
|
+
? runtime.config.observerPreambleMaxTokens
|
|
773
|
+
: Math.round(runtime.config.observerChunkMaxTokens * 0.3);
|
|
774
|
+
const allObservations = [
|
|
775
|
+
...memory.observations,
|
|
776
|
+
...accumulatedObservations,
|
|
777
|
+
];
|
|
778
|
+
priorObservations = selectPriorObservations(
|
|
779
|
+
allObservations,
|
|
780
|
+
preambleMaxTokens,
|
|
781
|
+
).map(observationToSummaryLine);
|
|
782
|
+
|
|
783
|
+
// Reflections are never trimmed — rare and always kept
|
|
784
|
+
priorReflections = [
|
|
785
|
+
...priorReflections,
|
|
786
|
+
...accumulatedReflections.map(reflectionToSummaryLine),
|
|
787
|
+
];
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
// If manual mode: skip if this exact chunk was already processed
|
|
791
|
+
if (
|
|
792
|
+
isManualMode(runtime.config) &&
|
|
793
|
+
isObservationChunkPending(sessionId, coversUpToId)
|
|
794
|
+
) {
|
|
795
|
+
debugLog("observer.pending_skip", { coversUpToId, sessionId });
|
|
796
|
+
return "continue";
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
|
|
800
|
+
const resolved = await resolveModel("observer");
|
|
801
|
+
if (!resolved) return "abort";
|
|
802
|
+
|
|
803
|
+
// Adjust accumulated for pending coverage in manual mode
|
|
804
|
+
let effectiveTokens = tokens;
|
|
805
|
+
if (isManualMode(runtime.config)) {
|
|
806
|
+
const pending = readPendingState(sessionId);
|
|
807
|
+
if (pending.observation?.coversUpToId) {
|
|
808
|
+
const idx = entryIndexForId(entries, pending.observation.coversUpToId);
|
|
809
|
+
if (idx >= 0) effectiveTokens = rawTokensAfterIndex(entries, idx);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
runtime.tryEmitInfo(
|
|
813
|
+
ctx.hasUI,
|
|
814
|
+
ctx.ui,
|
|
815
|
+
`Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk (of ${effectiveTokens.toLocaleString()} accumulated)`,
|
|
816
|
+
);
|
|
817
|
+
debugLog("observer.start", {
|
|
818
|
+
tokens,
|
|
819
|
+
coversUpToId,
|
|
820
|
+
sourceEntryIds,
|
|
821
|
+
sourceEntryCount: sourceEntryIds.length,
|
|
822
|
+
priorReflections: priorReflections.length,
|
|
823
|
+
priorObservations: priorObservations.length,
|
|
824
|
+
});
|
|
825
|
+
|
|
826
|
+
// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
|
|
827
|
+
const stageModelForThinking = runtime.findCandidateConfig(resolved.model, {
|
|
828
|
+
model: ctx.model,
|
|
829
|
+
modelRegistry: ctx.modelRegistry,
|
|
830
|
+
hasUI: ctx.hasUI,
|
|
831
|
+
ui: ctx.ui,
|
|
832
|
+
stageModel: stageModelConfig(runtime, "observer"),
|
|
833
|
+
stageFallbacks: stageFallbackModels(runtime, "observer"),
|
|
834
|
+
});
|
|
835
|
+
|
|
836
|
+
// Check if estimated input fits in model's context window
|
|
837
|
+
// Use actual chunk tokens (already computed) instead of the configured cap
|
|
838
|
+
const effectiveObsCtx = effectiveContextWindow(
|
|
839
|
+
resolved.model as any,
|
|
840
|
+
stageModelForThinking,
|
|
841
|
+
);
|
|
842
|
+
const observerEstimatedInput = chunkTokens + AGENT_LOOP_RESERVE;
|
|
843
|
+
if (observerEstimatedInput > effectiveObsCtx) {
|
|
844
|
+
debugLog("observer.context_window_exceeded", {
|
|
845
|
+
estimatedInput: observerEstimatedInput,
|
|
846
|
+
effectiveCtx: effectiveObsCtx,
|
|
847
|
+
model: `${(resolved.model as any).provider}/${(resolved.model as any).id}`,
|
|
848
|
+
});
|
|
849
|
+
runtime.recordRetryableError(
|
|
850
|
+
stageModelForThinking,
|
|
851
|
+
new Error(
|
|
852
|
+
`context window ${effectiveObsCtx} too small for estimated input ${observerEstimatedInput}`,
|
|
853
|
+
),
|
|
854
|
+
"observer",
|
|
855
|
+
);
|
|
856
|
+
runtime.tryEmitInfo(
|
|
857
|
+
ctx.hasUI,
|
|
858
|
+
ctx.ui,
|
|
859
|
+
`Observational memory: observer skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveObsCtx.toLocaleString()} too small for ~${observerEstimatedInput.toLocaleString()}-token input)`,
|
|
860
|
+
);
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
try {
|
|
865
|
+
const result = await runObserver({
|
|
866
|
+
model: resolved.model as any,
|
|
867
|
+
apiKey: resolved.apiKey,
|
|
868
|
+
headers: resolved.headers,
|
|
869
|
+
priorReflections,
|
|
870
|
+
priorObservations,
|
|
871
|
+
chunk,
|
|
872
|
+
allowedSourceEntryIds: sourceEntryIds,
|
|
873
|
+
maxTurns: runtime.config.agentMaxTurns,
|
|
874
|
+
thinkingLevel: stageThinkingLevel(
|
|
875
|
+
runtime,
|
|
876
|
+
"observer",
|
|
877
|
+
stageModelForThinking,
|
|
878
|
+
),
|
|
879
|
+
});
|
|
880
|
+
|
|
881
|
+
if (result.observations && result.observations.length > 0) {
|
|
882
|
+
const data = buildObservationsRecordedData(
|
|
883
|
+
result.observations,
|
|
884
|
+
coversUpToId,
|
|
885
|
+
);
|
|
886
|
+
if (!data) {
|
|
887
|
+
runtime.advanceCursor("observer", coversUpToId, "empty");
|
|
888
|
+
return "continue";
|
|
889
|
+
}
|
|
890
|
+
debugLog("observer.records", {
|
|
891
|
+
count: result.observations.length,
|
|
892
|
+
observationTokens: result.observations.reduce(
|
|
893
|
+
(s: number, o: any) => s + o.tokenCount,
|
|
894
|
+
0,
|
|
895
|
+
),
|
|
896
|
+
coversUpToId,
|
|
897
|
+
});
|
|
898
|
+
if (isManualMode(runtime.config)) {
|
|
899
|
+
savePendingObservation(sessionId, { coversUpToId, data });
|
|
900
|
+
debugLog("observer.pending", {
|
|
901
|
+
count: result.observations.length,
|
|
902
|
+
coversUpToId,
|
|
903
|
+
sessionId,
|
|
904
|
+
});
|
|
905
|
+
} else {
|
|
906
|
+
appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
|
|
907
|
+
debugLog("observer.appended", {
|
|
908
|
+
count: result.observations.length,
|
|
909
|
+
coversUpToId,
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
runtime.advanceCursor("observer", coversUpToId, "recorded");
|
|
913
|
+
runtime.tryEmitInfo(
|
|
914
|
+
ctx.hasUI,
|
|
915
|
+
ctx.ui,
|
|
916
|
+
`Observational memory: ${result.observations.length} observation${result.observations.length === 1 ? "" : "s"} recorded`,
|
|
917
|
+
);
|
|
918
|
+
return "continue";
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// No observations — diagnose the reason for the warning
|
|
922
|
+
const reason = result.emptyReason;
|
|
923
|
+
const reasonLabel = reason
|
|
924
|
+
? reason.kind === "tool_not_called"
|
|
925
|
+
? "model did not call the observation tool"
|
|
926
|
+
: reason.kind === "all_rejected"
|
|
927
|
+
? `${reason.count} observation(s) rejected for invalid sourceEntryIds`
|
|
928
|
+
: reason.kind === "all_duplicates"
|
|
929
|
+
? `${reason.count} observation(s) were duplicates of already-recorded entries`
|
|
930
|
+
: reason.kind === "empty_array"
|
|
931
|
+
? "model called the tool but submitted an empty observations array"
|
|
932
|
+
: "nothing new to record"
|
|
933
|
+
: "unknown reason";
|
|
934
|
+
const reasonLevel: "info" | "warning" = reason
|
|
935
|
+
? reason.kind === "no_new_content" || reason.kind === "all_duplicates"
|
|
936
|
+
? "info"
|
|
937
|
+
: "warning"
|
|
938
|
+
: "warning";
|
|
939
|
+
debugLog("observer.empty", { coversUpToId, reason: reason?.kind });
|
|
940
|
+
runtime.advanceCursor("observer", coversUpToId, "empty");
|
|
941
|
+
if (reasonLevel === "warning") {
|
|
942
|
+
if (ctx.hasUI)
|
|
943
|
+
ctx.ui?.notify(
|
|
944
|
+
`Observational memory: no observations — ${reasonLabel}`,
|
|
945
|
+
"warning",
|
|
946
|
+
);
|
|
947
|
+
} else {
|
|
948
|
+
runtime.tryEmitInfo(
|
|
949
|
+
ctx.hasUI,
|
|
950
|
+
ctx.ui,
|
|
951
|
+
`Observational memory: no observations — ${reasonLabel}`,
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
return "continue";
|
|
955
|
+
} catch (error) {
|
|
956
|
+
if (isStaleExtensionContextError(error)) {
|
|
957
|
+
debugLog("observer.stale_ctx", { error: String(error) });
|
|
958
|
+
return "abort";
|
|
959
|
+
}
|
|
960
|
+
// Always try next fallback — don't abort pipeline for a single model failure.
|
|
961
|
+
// Record cooldown so resolveModel skips this model in the next iteration.
|
|
962
|
+
const candidateConfig = runtime.findCandidateConfig(resolved.model, {
|
|
963
|
+
model: ctx.model,
|
|
964
|
+
modelRegistry: ctx.modelRegistry,
|
|
965
|
+
hasUI: ctx.hasUI,
|
|
966
|
+
ui: ctx.ui,
|
|
967
|
+
stageModel: stageModelConfig(runtime, "observer"),
|
|
968
|
+
stageFallbacks: stageFallbackModels(runtime, "observer"),
|
|
969
|
+
});
|
|
970
|
+
runtime.recordRetryableError(candidateConfig, error, "observer");
|
|
971
|
+
debugLog("observer.error", {
|
|
972
|
+
error: String(error),
|
|
973
|
+
retryable: isRetryableError(error),
|
|
974
|
+
});
|
|
975
|
+
// Continue loop — resolveModel will skip the cooled-down model
|
|
976
|
+
continue;
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
// All attempts exhausted
|
|
981
|
+
runtime.recordConsolidationStageError(
|
|
982
|
+
ctx,
|
|
983
|
+
"observer",
|
|
984
|
+
new Error("Observer: all model candidates exhausted"),
|
|
985
|
+
);
|
|
986
|
+
return "abort";
|
|
749
987
|
}
|
|
750
988
|
|
|
751
989
|
// ── Reflector stage (with fallback) ─────────────────────────────────────────
|
|
752
990
|
|
|
753
991
|
async function runReflectorStage(
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
992
|
+
pi: ExtensionAPI,
|
|
993
|
+
runtime: Runtime,
|
|
994
|
+
ctx: ConsolidationCtx,
|
|
995
|
+
resolveModel: (stage: "reflector") => Promise<ResolvedModel | undefined>,
|
|
758
996
|
): Promise<ReflectorStageResult> {
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
997
|
+
let entries: Entry[];
|
|
998
|
+
let sessionId: string;
|
|
999
|
+
try {
|
|
1000
|
+
entries = ctx.sessionManager.getBranch() as Entry[];
|
|
1001
|
+
sessionId = ctx.sessionManager.getSessionId();
|
|
1002
|
+
} catch (error) {
|
|
1003
|
+
if (isStaleExtensionContextError(error)) {
|
|
1004
|
+
debugLog("reflector.stale_ctx", { error: String(error) });
|
|
1005
|
+
return { outcome: "abort", sameRunReflections: [] };
|
|
1006
|
+
}
|
|
1007
|
+
throw error;
|
|
1008
|
+
}
|
|
1009
|
+
let reflectionTokens = 0;
|
|
1010
|
+
let observationCoverageId: string | undefined;
|
|
1011
|
+
if (isManualMode(runtime.config)) {
|
|
1012
|
+
const pending = readPendingState(sessionId);
|
|
1013
|
+
// Check any accumulated batch for unprocessed observations, not just the latest
|
|
1014
|
+
const hasPendingObs = (pending.observationBatches ?? []).some(
|
|
1015
|
+
(b: any) => (b.data as any)?.observations?.length,
|
|
1016
|
+
);
|
|
1017
|
+
if (!hasPendingObs) {
|
|
1018
|
+
runtime.advanceCursor(
|
|
1019
|
+
"reflector",
|
|
1020
|
+
entries.at(-1)?.id ?? "unknown",
|
|
1021
|
+
"skipped",
|
|
1022
|
+
);
|
|
1023
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1024
|
+
}
|
|
1025
|
+
observationCoverageId = pending.observation?.coversUpToId;
|
|
1026
|
+
if (pending.reflection?.coversUpToId) {
|
|
1027
|
+
const obsIdx = entryIndexForId(
|
|
1028
|
+
entries,
|
|
1029
|
+
pending.observation?.coversUpToId ?? "",
|
|
1030
|
+
);
|
|
1031
|
+
const refIdx = entryIndexForId(entries, pending.reflection.coversUpToId);
|
|
1032
|
+
if (obsIdx >= 0 && refIdx >= 0 && obsIdx <= refIdx) {
|
|
1033
|
+
runtime.advanceCursor(
|
|
1034
|
+
"reflector",
|
|
1035
|
+
pending.reflection.coversUpToId,
|
|
1036
|
+
"skipped",
|
|
1037
|
+
);
|
|
1038
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1039
|
+
}
|
|
1040
|
+
if (refIdx >= 0) {
|
|
1041
|
+
reflectionTokens = rawTokensAfterIndex(entries, refIdx);
|
|
1042
|
+
if (reflectionTokens < runtime.config.reflectAfterTokens) {
|
|
1043
|
+
runtime.advanceCursor(
|
|
1044
|
+
"reflector",
|
|
1045
|
+
pending.reflection.coversUpToId,
|
|
1046
|
+
"not_due",
|
|
1047
|
+
);
|
|
1048
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1049
|
+
}
|
|
1050
|
+
} else {
|
|
1051
|
+
reflectionTokens = rawTokensSinceObservationCoverage(entries);
|
|
1052
|
+
}
|
|
1053
|
+
} else {
|
|
1054
|
+
reflectionTokens = rawTokensSinceObservationCoverage(entries);
|
|
1055
|
+
}
|
|
1056
|
+
} else {
|
|
1057
|
+
reflectionTokens = rawTokensSinceReflectionCoverage(entries);
|
|
1058
|
+
if (reflectionTokens < runtime.config.reflectAfterTokens) {
|
|
1059
|
+
runtime.advanceCursor(
|
|
1060
|
+
"reflector",
|
|
1061
|
+
entries.at(-1)?.id ?? "unknown",
|
|
1062
|
+
"not_due",
|
|
1063
|
+
);
|
|
1064
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1065
|
+
}
|
|
1066
|
+
observationCoverageId = latestCoverageMarkerId(
|
|
1067
|
+
entries,
|
|
1068
|
+
OM_OBSERVATIONS_RECORDED,
|
|
1069
|
+
);
|
|
1070
|
+
if (!observationCoverageId) {
|
|
1071
|
+
runtime.advanceCursor(
|
|
1072
|
+
"reflector",
|
|
1073
|
+
entries.at(-1)?.id ?? "unknown",
|
|
1074
|
+
"skipped",
|
|
1075
|
+
);
|
|
1076
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
|
|
1081
|
+
const resolved = await resolveModel("reflector");
|
|
1082
|
+
if (!resolved) return { outcome: "abort", sameRunReflections: [] };
|
|
1083
|
+
|
|
1084
|
+
// Compute ahead for an accurate notification
|
|
1085
|
+
const folded = foldLedger(entries);
|
|
1086
|
+
const pending = isManualMode(runtime.config)
|
|
1087
|
+
? readPendingState(sessionId)
|
|
1088
|
+
: undefined;
|
|
1089
|
+
const lastReflectionIdx = pending
|
|
1090
|
+
? -1
|
|
1091
|
+
: latestCoverageIndex(entries, OM_REFLECTIONS_RECORDED);
|
|
1092
|
+
const newObservations = pending
|
|
1093
|
+
? pendingObservationsCreatedAfter(
|
|
1094
|
+
pending,
|
|
1095
|
+
entries,
|
|
1096
|
+
pending.reflection?.coversUpToId,
|
|
1097
|
+
)
|
|
1098
|
+
: observationsCreatedAfterIndex(entries, lastReflectionIdx);
|
|
1099
|
+
const newReflections = pending
|
|
1100
|
+
? []
|
|
1101
|
+
: reflectionsCreatedAfterIndex(entries, lastReflectionIdx);
|
|
1102
|
+
const newItemsTokens = Math.ceil(
|
|
1103
|
+
(newObservations.reduce((s: number, o: any) => s + o.content.length, 0) +
|
|
1104
|
+
newReflections.reduce((s: number, r: any) => s + r.content.length, 0)) /
|
|
1105
|
+
4,
|
|
1106
|
+
);
|
|
1107
|
+
const summaryBudget =
|
|
1108
|
+
Math.floor(runtime.config.reflectorInputMaxTokens * 0.15) * 2;
|
|
1109
|
+
const reflectorInputTokens = Math.min(
|
|
1110
|
+
newItemsTokens + summaryBudget,
|
|
1111
|
+
runtime.config.reflectorInputMaxTokens,
|
|
1112
|
+
);
|
|
1113
|
+
// Adjust accumulated for pending coverage in manual mode
|
|
1114
|
+
let effectiveReflectionTokens = reflectionTokens;
|
|
1115
|
+
if (isManualMode(runtime.config)) {
|
|
1116
|
+
if (pending?.reflection?.coversUpToId) {
|
|
1117
|
+
const idx = entryIndexForId(entries, pending.reflection.coversUpToId);
|
|
1118
|
+
if (idx >= 0)
|
|
1119
|
+
effectiveReflectionTokens = rawTokensAfterIndex(entries, idx);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
debugLog("reflector.start", {
|
|
1123
|
+
tokens: effectiveReflectionTokens,
|
|
1124
|
+
inputTokens: reflectorInputTokens,
|
|
1125
|
+
newObsCount: newObservations.length,
|
|
1126
|
+
newRefCount: newReflections.length,
|
|
1127
|
+
});
|
|
1128
|
+
runtime.tryEmitInfo(
|
|
1129
|
+
ctx.hasUI,
|
|
1130
|
+
ctx.ui,
|
|
1131
|
+
`Observational memory: reflector running (~${effectiveReflectionTokens.toLocaleString()} tokens accumulated, ~${reflectorInputTokens.toLocaleString()}-token input)`,
|
|
1132
|
+
);
|
|
1133
|
+
|
|
1134
|
+
// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
|
|
1135
|
+
const stageModelForThinking = runtime.findCandidateConfig(resolved.model, {
|
|
1136
|
+
model: ctx.model,
|
|
1137
|
+
modelRegistry: ctx.modelRegistry,
|
|
1138
|
+
hasUI: ctx.hasUI,
|
|
1139
|
+
ui: ctx.ui,
|
|
1140
|
+
stageModel: stageModelConfig(runtime, "reflector"),
|
|
1141
|
+
stageFallbacks: stageFallbackModels(runtime, "reflector"),
|
|
1142
|
+
});
|
|
1143
|
+
|
|
1144
|
+
// Check if estimated input fits in model's context window
|
|
1145
|
+
// Use actual computed input size (new items + summary budget) instead of cap
|
|
1146
|
+
const effectiveRefCtx = effectiveContextWindow(
|
|
1147
|
+
resolved.model as any,
|
|
1148
|
+
stageModelForThinking,
|
|
1149
|
+
);
|
|
1150
|
+
const reflectorEstimatedInput = reflectorInputTokens + AGENT_LOOP_RESERVE;
|
|
1151
|
+
if (reflectorEstimatedInput > effectiveRefCtx) {
|
|
1152
|
+
debugLog("reflector.context_window_exceeded", {
|
|
1153
|
+
estimatedInput: reflectorEstimatedInput,
|
|
1154
|
+
effectiveCtx: effectiveRefCtx,
|
|
1155
|
+
model: `${(resolved.model as any).provider}/${(resolved.model as any).id}`,
|
|
1156
|
+
});
|
|
1157
|
+
runtime.recordRetryableError(
|
|
1158
|
+
stageModelForThinking,
|
|
1159
|
+
new Error(
|
|
1160
|
+
`context window ${effectiveRefCtx} too small for estimated input ${reflectorEstimatedInput}`,
|
|
1161
|
+
),
|
|
1162
|
+
"reflector",
|
|
1163
|
+
);
|
|
1164
|
+
runtime.tryEmitInfo(
|
|
1165
|
+
ctx.hasUI,
|
|
1166
|
+
ctx.ui,
|
|
1167
|
+
`Observational memory: reflector skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveRefCtx.toLocaleString()} too small for ~${reflectorEstimatedInput.toLocaleString()}-token input)`,
|
|
1168
|
+
);
|
|
1169
|
+
continue;
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
try {
|
|
1173
|
+
// Existing memory summaries for context (capped).
|
|
1174
|
+
// In manual mode, merge accumulated pending batches with
|
|
1175
|
+
// branch data (preserving pre-switch markers).
|
|
1176
|
+
const sourceReflections = pending
|
|
1177
|
+
? [
|
|
1178
|
+
...folded.reflections,
|
|
1179
|
+
...(pending.reflectionBatches ?? []).flatMap(
|
|
1180
|
+
(b: any) => (b.data as any)?.reflections ?? [],
|
|
1181
|
+
),
|
|
1182
|
+
]
|
|
1183
|
+
: folded.reflections;
|
|
1184
|
+
const sourceObservations = pending
|
|
1185
|
+
? [
|
|
1186
|
+
...folded.activeObservations,
|
|
1187
|
+
...(pending.observationBatches ?? []).flatMap(
|
|
1188
|
+
(b: any) => (b.data as any)?.observations ?? [],
|
|
1189
|
+
),
|
|
1190
|
+
]
|
|
1191
|
+
: folded.activeObservations;
|
|
1192
|
+
const existingReflectionsSummary = buildExistingReflectionsSummary(
|
|
1193
|
+
sourceReflections,
|
|
1194
|
+
Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
|
|
1195
|
+
);
|
|
1196
|
+
const existingObservationsSummary = buildExistingObservationsSummary(
|
|
1197
|
+
sourceObservations.filter(
|
|
1198
|
+
(o: any) => !newObservations.some((no: any) => no.id === o.id),
|
|
1199
|
+
),
|
|
1200
|
+
Math.floor(runtime.config.reflectorInputMaxTokens * 0.15),
|
|
1201
|
+
);
|
|
1202
|
+
|
|
1203
|
+
const reflections = await runReflector({
|
|
1204
|
+
model: resolved.model as any,
|
|
1205
|
+
apiKey: resolved.apiKey,
|
|
1206
|
+
headers: resolved.headers,
|
|
1207
|
+
reflections: newReflections,
|
|
1208
|
+
observations: newObservations,
|
|
1209
|
+
existingReflectionsSummary: existingReflectionsSummary || undefined,
|
|
1210
|
+
existingObservationsSummary: existingObservationsSummary || undefined,
|
|
1211
|
+
maxTurns: runtime.config.agentMaxTurns,
|
|
1212
|
+
thinkingLevel: stageThinkingLevel(
|
|
1213
|
+
runtime,
|
|
1214
|
+
"reflector",
|
|
1215
|
+
stageModelForThinking,
|
|
1216
|
+
),
|
|
1217
|
+
});
|
|
1218
|
+
|
|
1219
|
+
if (!reflections || reflections.length === 0) {
|
|
1220
|
+
runtime.advanceCursor(
|
|
1221
|
+
"reflector",
|
|
1222
|
+
observationCoverageId ?? entries.at(-1)?.id ?? "unknown",
|
|
1223
|
+
"empty",
|
|
1224
|
+
);
|
|
1225
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1226
|
+
}
|
|
1227
|
+
if (!observationCoverageId) {
|
|
1228
|
+
runtime.advanceCursor(
|
|
1229
|
+
"reflector",
|
|
1230
|
+
entries.at(-1)?.id ?? "unknown",
|
|
1231
|
+
"empty",
|
|
1232
|
+
);
|
|
1233
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
const data = buildReflectionsRecordedData(
|
|
1237
|
+
reflections,
|
|
1238
|
+
observationCoverageId,
|
|
1239
|
+
);
|
|
1240
|
+
if (!data) {
|
|
1241
|
+
runtime.advanceCursor("reflector", observationCoverageId, "empty");
|
|
1242
|
+
return { outcome: "continue", sameRunReflections: [] };
|
|
1243
|
+
}
|
|
1244
|
+
if (isManualMode(runtime.config)) {
|
|
1245
|
+
savePendingReflection(sessionId, {
|
|
1246
|
+
coversUpToId: data.coversUpToId,
|
|
1247
|
+
data,
|
|
1248
|
+
});
|
|
1249
|
+
} else {
|
|
1250
|
+
appendEntry(pi, OM_REFLECTIONS_RECORDED, data);
|
|
1251
|
+
}
|
|
1252
|
+
runtime.advanceCursor("reflector", data.coversUpToId, "recorded");
|
|
1253
|
+
return {
|
|
1254
|
+
outcome: "continue",
|
|
1255
|
+
sameRunReflections: reflections,
|
|
1256
|
+
effectiveReflectionCoverageId: data.coversUpToId,
|
|
1257
|
+
};
|
|
1258
|
+
} catch (error) {
|
|
1259
|
+
if (isStaleExtensionContextError(error)) {
|
|
1260
|
+
debugLog("reflector.stale_ctx", { error: String(error) });
|
|
1261
|
+
return { outcome: "abort", sameRunReflections: [] };
|
|
1262
|
+
}
|
|
1263
|
+
const candidateConfig = runtime.findCandidateConfig(resolved.model, {
|
|
1264
|
+
model: ctx.model,
|
|
1265
|
+
modelRegistry: ctx.modelRegistry,
|
|
1266
|
+
hasUI: ctx.hasUI,
|
|
1267
|
+
ui: ctx.ui,
|
|
1268
|
+
stageModel: stageModelConfig(runtime, "reflector"),
|
|
1269
|
+
stageFallbacks: stageFallbackModels(runtime, "reflector"),
|
|
1270
|
+
});
|
|
1271
|
+
runtime.recordRetryableError(candidateConfig, error, "reflector");
|
|
1272
|
+
debugLog("reflector.error", {
|
|
1273
|
+
error: String(error),
|
|
1274
|
+
retryable: isRetryableError(error),
|
|
1275
|
+
});
|
|
1276
|
+
continue;
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
runtime.recordConsolidationStageError(
|
|
1281
|
+
ctx,
|
|
1282
|
+
"reflector",
|
|
1283
|
+
new Error("Reflector: all model candidates exhausted"),
|
|
1284
|
+
);
|
|
1285
|
+
return { outcome: "abort", sameRunReflections: [] };
|
|
914
1286
|
}
|
|
915
1287
|
|
|
916
1288
|
// ── Dropper stage (with fallback) ───────────────────────────────────────────
|
|
917
1289
|
|
|
918
1290
|
async function runDropperStage(
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
1291
|
+
pi: ExtensionAPI,
|
|
1292
|
+
runtime: Runtime,
|
|
1293
|
+
ctx: ConsolidationCtx,
|
|
1294
|
+
resolveModel: (stage: "dropper") => Promise<ResolvedModel | undefined>,
|
|
1295
|
+
sameRunReflections: Reflection[],
|
|
1296
|
+
sameRunReflectionCoverageId: string | undefined,
|
|
925
1297
|
): Promise<StageOutcome> {
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1298
|
+
let entries: Entry[];
|
|
1299
|
+
let sessionId: string;
|
|
1300
|
+
try {
|
|
1301
|
+
entries = ctx.sessionManager.getBranch() as Entry[];
|
|
1302
|
+
sessionId = ctx.sessionManager.getSessionId();
|
|
1303
|
+
} catch (error) {
|
|
1304
|
+
if (isStaleExtensionContextError(error)) {
|
|
1305
|
+
debugLog("dropper.stale_ctx", { error: String(error) });
|
|
1306
|
+
return "abort";
|
|
1307
|
+
}
|
|
1308
|
+
throw error;
|
|
1309
|
+
}
|
|
1310
|
+
let dropTokens = 0;
|
|
1311
|
+
let observationCoverageId: string | undefined;
|
|
1312
|
+
if (isManualMode(runtime.config)) {
|
|
1313
|
+
const pending = readPendingState(sessionId);
|
|
1314
|
+
// Check any accumulated batch for unprocessed observations, not just the latest
|
|
1315
|
+
const hasPendingObs = (pending.observationBatches ?? []).some(
|
|
1316
|
+
(b: any) => (b.data as any)?.observations?.length,
|
|
1317
|
+
);
|
|
1318
|
+
if (!hasPendingObs) {
|
|
1319
|
+
runtime.advanceCursor(
|
|
1320
|
+
"dropper",
|
|
1321
|
+
entries.at(-1)?.id ?? "unknown",
|
|
1322
|
+
"skipped",
|
|
1323
|
+
);
|
|
1324
|
+
return "continue";
|
|
1325
|
+
}
|
|
1326
|
+
observationCoverageId = pending.observation?.coversUpToId;
|
|
1327
|
+
if (pending.dropped?.coversUpToId) {
|
|
1328
|
+
const obsIdx = entryIndexForId(
|
|
1329
|
+
entries,
|
|
1330
|
+
pending.observation?.coversUpToId ?? "",
|
|
1331
|
+
);
|
|
1332
|
+
const dropIdx = entryIndexForId(entries, pending.dropped.coversUpToId);
|
|
1333
|
+
if (obsIdx >= 0 && dropIdx >= 0 && obsIdx <= dropIdx) {
|
|
1334
|
+
runtime.advanceCursor(
|
|
1335
|
+
"dropper",
|
|
1336
|
+
pending.dropped.coversUpToId,
|
|
1337
|
+
"skipped",
|
|
1338
|
+
);
|
|
1339
|
+
return "continue";
|
|
1340
|
+
}
|
|
1341
|
+
if (dropIdx >= 0) {
|
|
1342
|
+
dropTokens = rawTokensAfterIndex(entries, dropIdx);
|
|
1343
|
+
if (dropTokens < runtime.config.reflectAfterTokens) {
|
|
1344
|
+
runtime.advanceCursor(
|
|
1345
|
+
"dropper",
|
|
1346
|
+
pending.dropped.coversUpToId,
|
|
1347
|
+
"not_due",
|
|
1348
|
+
);
|
|
1349
|
+
return "continue";
|
|
1350
|
+
}
|
|
1351
|
+
} else {
|
|
1352
|
+
dropTokens = rawTokensSinceDropCoverage(entries);
|
|
1353
|
+
}
|
|
1354
|
+
} else {
|
|
1355
|
+
dropTokens = rawTokensSinceDropCoverage(entries);
|
|
1356
|
+
}
|
|
1357
|
+
} else {
|
|
1358
|
+
dropTokens = rawTokensSinceDropCoverage(entries);
|
|
1359
|
+
if (dropTokens < runtime.config.reflectAfterTokens) {
|
|
1360
|
+
runtime.advanceCursor(
|
|
1361
|
+
"dropper",
|
|
1362
|
+
entries.at(-1)?.id ?? "unknown",
|
|
1363
|
+
"not_due",
|
|
1364
|
+
);
|
|
1365
|
+
return "continue";
|
|
1366
|
+
}
|
|
1367
|
+
observationCoverageId = latestCoverageMarkerId(
|
|
1368
|
+
entries,
|
|
1369
|
+
OM_OBSERVATIONS_RECORDED,
|
|
1370
|
+
);
|
|
1371
|
+
if (!observationCoverageId) {
|
|
1372
|
+
runtime.advanceCursor(
|
|
1373
|
+
"dropper",
|
|
1374
|
+
entries.at(-1)?.id ?? "unknown",
|
|
1375
|
+
"skipped",
|
|
1376
|
+
);
|
|
1377
|
+
return "continue";
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
for (let attempt = 0; attempt < MAX_STAGE_ATTEMPTS; attempt++) {
|
|
1382
|
+
const resolved = await resolveModel("dropper");
|
|
1383
|
+
if (!resolved) return "abort";
|
|
1384
|
+
|
|
1385
|
+
// Compute ahead for an accurate notification
|
|
1386
|
+
const folded = foldLedger(entries);
|
|
1387
|
+
const pending = isManualMode(runtime.config)
|
|
1388
|
+
? readPendingState(sessionId)
|
|
1389
|
+
: undefined;
|
|
1390
|
+
const lastDropIdx = pending
|
|
1391
|
+
? -1
|
|
1392
|
+
: latestCoverageIndex(entries, OM_OBSERVATIONS_DROPPED);
|
|
1393
|
+
const newObservations = pending
|
|
1394
|
+
? pendingObservationsCreatedAfter(
|
|
1395
|
+
pending,
|
|
1396
|
+
entries,
|
|
1397
|
+
pending.dropped?.coversUpToId,
|
|
1398
|
+
)
|
|
1399
|
+
: observationsCreatedAfterIndex(entries, lastDropIdx);
|
|
1400
|
+
const dropperNewObsTokens = Math.ceil(
|
|
1401
|
+
newObservations.reduce((s: number, o: any) => s + o.content.length, 0) /
|
|
1402
|
+
4,
|
|
1403
|
+
);
|
|
1404
|
+
const dropperSummaryBudget = Math.floor(
|
|
1405
|
+
runtime.config.dropperInputMaxTokens * 0.2,
|
|
1406
|
+
);
|
|
1407
|
+
const dropperInputTokens = Math.min(
|
|
1408
|
+
dropperNewObsTokens + dropperSummaryBudget,
|
|
1409
|
+
runtime.config.dropperInputMaxTokens,
|
|
1410
|
+
);
|
|
1411
|
+
// Adjust accumulated for pending coverage in manual mode
|
|
1412
|
+
let effectiveDropTokens = dropTokens;
|
|
1413
|
+
if (isManualMode(runtime.config)) {
|
|
1414
|
+
if (pending?.dropped?.coversUpToId) {
|
|
1415
|
+
const idx = entryIndexForId(entries, pending.dropped.coversUpToId);
|
|
1416
|
+
if (idx >= 0) effectiveDropTokens = rawTokensAfterIndex(entries, idx);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
runtime.tryEmitInfo(
|
|
1420
|
+
ctx.hasUI,
|
|
1421
|
+
ctx.ui,
|
|
1422
|
+
`Observational memory: dropper running (~${effectiveDropTokens.toLocaleString()} tokens accumulated, ~${dropperInputTokens.toLocaleString()}-token input)`,
|
|
1423
|
+
);
|
|
1424
|
+
|
|
1425
|
+
try {
|
|
1426
|
+
// Existing active observations summary for context (capped).
|
|
1427
|
+
// In manual mode, merge accumulated pending batches with
|
|
1428
|
+
// branch data (preserving pre-switch markers).
|
|
1429
|
+
const sourceObsForDropper = pending
|
|
1430
|
+
? [
|
|
1431
|
+
...folded.activeObservations,
|
|
1432
|
+
...(pending.observationBatches ?? []).flatMap(
|
|
1433
|
+
(b: any) => (b.data as any)?.observations ?? [],
|
|
1434
|
+
),
|
|
1435
|
+
]
|
|
1436
|
+
: folded.activeObservations;
|
|
1437
|
+
const existingObservationsSummary = buildExistingObservationsSummary(
|
|
1438
|
+
sourceObsForDropper.filter(
|
|
1439
|
+
(o: any) => !newObservations.some((no: any) => no.id === o.id),
|
|
1440
|
+
),
|
|
1441
|
+
Math.floor(runtime.config.dropperInputMaxTokens * 0.2),
|
|
1442
|
+
);
|
|
1443
|
+
// In manual mode, merge accumulated reflection batches with
|
|
1444
|
+
// branch data (preserving pre-switch markers), matching the
|
|
1445
|
+
// dropper's full autoCompact context.
|
|
1446
|
+
const pendingReflections = pending
|
|
1447
|
+
? [
|
|
1448
|
+
...folded.reflections,
|
|
1449
|
+
...(pending.reflectionBatches ?? []).flatMap(
|
|
1450
|
+
(b: any) => (b.data as any)?.reflections ?? [],
|
|
1451
|
+
),
|
|
1452
|
+
]
|
|
1453
|
+
: folded.reflections;
|
|
1454
|
+
const reflectionsForDropper = mergeReflections(
|
|
1455
|
+
pendingReflections,
|
|
1456
|
+
sameRunReflections,
|
|
1457
|
+
);
|
|
1458
|
+
|
|
1459
|
+
// Resolve thinking level for the specific model (fallbacks may have their own thinking config)
|
|
1460
|
+
const stageModelForThinking = runtime.findCandidateConfig(
|
|
1461
|
+
resolved.model,
|
|
1462
|
+
{
|
|
1463
|
+
model: ctx.model,
|
|
1464
|
+
modelRegistry: ctx.modelRegistry,
|
|
1465
|
+
hasUI: ctx.hasUI,
|
|
1466
|
+
ui: ctx.ui,
|
|
1467
|
+
stageModel: stageModelConfig(runtime, "dropper"),
|
|
1468
|
+
stageFallbacks: stageFallbackModels(runtime, "dropper"),
|
|
1469
|
+
},
|
|
1470
|
+
);
|
|
1471
|
+
|
|
1472
|
+
// Check if estimated input fits in model's context window
|
|
1473
|
+
// Use actual computed input size (new observations + summary budget) instead of cap
|
|
1474
|
+
const effectiveDropCtx = effectiveContextWindow(
|
|
1475
|
+
resolved.model as any,
|
|
1476
|
+
stageModelForThinking,
|
|
1477
|
+
);
|
|
1478
|
+
const dropperEstimatedInput = dropperInputTokens + AGENT_LOOP_RESERVE;
|
|
1479
|
+
if (dropperEstimatedInput > effectiveDropCtx) {
|
|
1480
|
+
debugLog("dropper.context_window_exceeded", {
|
|
1481
|
+
estimatedInput: dropperEstimatedInput,
|
|
1482
|
+
effectiveCtx: effectiveDropCtx,
|
|
1483
|
+
model: `${(resolved.model as any).provider}/${(resolved.model as any).id}`,
|
|
1484
|
+
});
|
|
1485
|
+
runtime.recordRetryableError(
|
|
1486
|
+
stageModelForThinking,
|
|
1487
|
+
new Error(
|
|
1488
|
+
`context window ${effectiveDropCtx} too small for estimated input ${dropperEstimatedInput}`,
|
|
1489
|
+
),
|
|
1490
|
+
"dropper",
|
|
1491
|
+
);
|
|
1492
|
+
runtime.tryEmitInfo(
|
|
1493
|
+
ctx.hasUI,
|
|
1494
|
+
ctx.ui,
|
|
1495
|
+
`Observational memory: dropper skipping ${(resolved.model as any).provider}/${(resolved.model as any).id} (context window ${effectiveDropCtx.toLocaleString()} too small for ~${dropperEstimatedInput.toLocaleString()}-token input)`,
|
|
1496
|
+
);
|
|
1497
|
+
continue;
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
const droppedIds = await runDropper({
|
|
1501
|
+
model: resolved.model as any,
|
|
1502
|
+
apiKey: resolved.apiKey,
|
|
1503
|
+
headers: resolved.headers,
|
|
1504
|
+
reflections: reflectionsForDropper,
|
|
1505
|
+
observations: newObservations,
|
|
1506
|
+
existingObservationsSummary: existingObservationsSummary || undefined,
|
|
1507
|
+
budgetTokens: runtime.config.observationsPoolMaxTokens,
|
|
1508
|
+
skipFullness: runtime.config.dropperPoolFullnessThreshold,
|
|
1509
|
+
maxTurns: runtime.config.agentMaxTurns,
|
|
1510
|
+
thinkingLevel: stageThinkingLevel(
|
|
1511
|
+
runtime,
|
|
1512
|
+
"dropper",
|
|
1513
|
+
stageModelForThinking,
|
|
1514
|
+
),
|
|
1515
|
+
});
|
|
1516
|
+
const latestReflectionCoverageId = isManualMode(runtime.config)
|
|
1517
|
+
? pending?.reflection?.coversUpToId
|
|
1518
|
+
: latestCoverageMarkerId(entries, OM_REFLECTIONS_RECORDED);
|
|
1519
|
+
const effectiveReflectionCoverageId =
|
|
1520
|
+
sameRunReflectionCoverageId ?? latestReflectionCoverageId;
|
|
1521
|
+
const coversUpToId = earlierCoverageMarkerId(
|
|
1522
|
+
entries,
|
|
1523
|
+
observationCoverageId,
|
|
1524
|
+
effectiveReflectionCoverageId,
|
|
1525
|
+
);
|
|
1526
|
+
const data =
|
|
1527
|
+
coversUpToId && droppedIds
|
|
1528
|
+
? buildObservationsDroppedData(droppedIds, coversUpToId)
|
|
1529
|
+
: undefined;
|
|
1530
|
+
if (data && coversUpToId) {
|
|
1531
|
+
if (isManualMode(runtime.config)) {
|
|
1532
|
+
savePendingDropped(sessionId, { coversUpToId, data });
|
|
1533
|
+
} else {
|
|
1534
|
+
appendEntry(pi, OM_OBSERVATIONS_DROPPED, data);
|
|
1535
|
+
}
|
|
1536
|
+
runtime.advanceCursor("dropper", coversUpToId, "recorded");
|
|
1537
|
+
} else {
|
|
1538
|
+
// No drops selected (maxDropsAllowed=0 or LLM returned no candidates)
|
|
1539
|
+
runtime.advanceCursor(
|
|
1540
|
+
"dropper",
|
|
1541
|
+
coversUpToId ??
|
|
1542
|
+
observationCoverageId ??
|
|
1543
|
+
entries.at(-1)?.id ??
|
|
1544
|
+
"unknown",
|
|
1545
|
+
"empty",
|
|
1546
|
+
);
|
|
1547
|
+
}
|
|
1548
|
+
return "continue";
|
|
1549
|
+
} catch (error) {
|
|
1550
|
+
if (isStaleExtensionContextError(error)) {
|
|
1551
|
+
debugLog("dropper.stale_ctx", { error: String(error) });
|
|
1552
|
+
return "abort";
|
|
1553
|
+
}
|
|
1554
|
+
const candidateConfig = runtime.findCandidateConfig(resolved.model, {
|
|
1555
|
+
model: ctx.model,
|
|
1556
|
+
modelRegistry: ctx.modelRegistry,
|
|
1557
|
+
hasUI: ctx.hasUI,
|
|
1558
|
+
ui: ctx.ui,
|
|
1559
|
+
stageModel: stageModelConfig(runtime, "dropper"),
|
|
1560
|
+
stageFallbacks: stageFallbackModels(runtime, "dropper"),
|
|
1561
|
+
});
|
|
1562
|
+
runtime.recordRetryableError(candidateConfig, error, "dropper");
|
|
1563
|
+
debugLog("dropper.error", {
|
|
1564
|
+
error: String(error),
|
|
1565
|
+
retryable: isRetryableError(error),
|
|
1566
|
+
});
|
|
1567
|
+
continue;
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
runtime.recordConsolidationStageError(
|
|
1572
|
+
ctx,
|
|
1573
|
+
"dropper",
|
|
1574
|
+
new Error("Dropper: all model candidates exhausted"),
|
|
1575
|
+
);
|
|
1576
|
+
return "abort";
|
|
1070
1577
|
}
|