pi-observational-memory 3.0.3 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -11
- package/package.json +5 -5
- package/src/agents/dropper/agent.ts +20 -5
- package/src/agents/dropper/pool.ts +6 -2
- package/src/agents/observer/agent.ts +53 -8
- package/src/agents/reflector/agent.ts +20 -5
- package/src/agents/stream-errors.ts +22 -0
- package/src/agents/worker-stream.ts +65 -0
- package/src/commands/status.ts +1 -1
- package/src/config.ts +63 -1
- package/src/hooks/compaction-hook.ts +10 -2
- package/src/hooks/compaction-trigger.ts +15 -34
- package/src/hooks/consolidation-trigger.ts +165 -34
- package/src/runtime.ts +227 -5
- package/src/serialize.ts +59 -3
- package/src/session-ledger/progress.ts +100 -0
- package/src/tokens.ts +18 -0
|
@@ -3,43 +3,20 @@ import { resolveCompactAfterTokens } from "../config.js";
|
|
|
3
3
|
import { rawTokensSinceLastCompaction, type Entry } from "../session-ledger/index.js";
|
|
4
4
|
import type { Runtime } from "../runtime.js";
|
|
5
5
|
|
|
6
|
-
/**
|
|
7
|
-
* Regex matching Pi's internal retryable error detection.
|
|
8
|
-
* When the last assistant message in agent_end has stopReason "error" matching this pattern,
|
|
9
|
-
* Pi will auto-retry — we must not trigger compaction between attempts.
|
|
10
|
-
*/
|
|
11
|
-
const RETRYABLE_ERROR_RE =
|
|
12
|
-
/overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|http2 request did not get a response|timed? out|timeout|terminated|retry delay/i;
|
|
13
|
-
|
|
14
6
|
export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): void {
|
|
15
|
-
|
|
7
|
+
// Pi emits agent_settled only after retries, automatic compaction, and queued
|
|
8
|
+
// continuation have finished, so retry policy stays owned by Pi.
|
|
9
|
+
pi.on("agent_settled", (_event, ctx) => {
|
|
16
10
|
runtime.ensureConfig(ctx.cwd);
|
|
17
11
|
if (runtime.config.passive === true) return;
|
|
18
12
|
if (runtime.compactInFlight) return;
|
|
19
13
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const lastAssistant = [...event.messages].reverse().find(
|
|
24
|
-
(m): m is Extract<typeof m, { role: "assistant" }> => m.role === "assistant",
|
|
25
|
-
);
|
|
26
|
-
if (
|
|
27
|
-
lastAssistant
|
|
28
|
-
&& lastAssistant.stopReason === "error"
|
|
29
|
-
&& lastAssistant.errorMessage
|
|
30
|
-
&& RETRYABLE_ERROR_RE.test(lastAssistant.errorMessage)
|
|
31
|
-
) {
|
|
32
|
-
return;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const entries = ctx.sessionManager.getBranch() as Entry[];
|
|
36
|
-
const tokens = rawTokensSinceLastCompaction(entries);
|
|
37
|
-
// Resolve the proactive-compaction threshold from the active model's context
|
|
38
|
-
// window when ratio mode is configured. ctx.model is the current session model
|
|
39
|
-
// (Model<any> | undefined per ExtensionContext).
|
|
14
|
+
const entries = ctx.sessionManager?.getBranch?.() as Entry[] | undefined;
|
|
15
|
+
if (!entries) return;
|
|
16
|
+
const progress = rawTokensSinceLastCompaction(entries);
|
|
40
17
|
const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : undefined;
|
|
41
18
|
const threshold = resolveCompactAfterTokens(runtime.config, contextWindow);
|
|
42
|
-
if (
|
|
19
|
+
if (progress < threshold) return;
|
|
43
20
|
|
|
44
21
|
// Capture ctx properties synchronously — the setTimeout + async work below
|
|
45
22
|
// may outlive the extension ctx (stale after session replacement/reload).
|
|
@@ -47,7 +24,7 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
|
|
|
47
24
|
const ui = ctx.ui;
|
|
48
25
|
|
|
49
26
|
if (hasUI) ui?.notify(
|
|
50
|
-
`Observational memory: compaction threshold reached (~${
|
|
27
|
+
`Observational memory: compaction threshold reached (~${progress.toLocaleString()} estimated source tokens); triggering compaction`,
|
|
51
28
|
"info",
|
|
52
29
|
);
|
|
53
30
|
|
|
@@ -62,9 +39,13 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
|
|
|
62
39
|
);
|
|
63
40
|
return;
|
|
64
41
|
}
|
|
65
|
-
const currentEntries = ctx.sessionManager
|
|
66
|
-
|
|
67
|
-
|
|
42
|
+
const currentEntries = ctx.sessionManager?.getBranch?.() as Entry[] | undefined;
|
|
43
|
+
if (!currentEntries) {
|
|
44
|
+
runtime.compactInFlight = false;
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const currentProgress = rawTokensSinceLastCompaction(currentEntries);
|
|
48
|
+
if (currentProgress < threshold) {
|
|
68
49
|
runtime.compactInFlight = false;
|
|
69
50
|
if (hasUI) ui?.notify(
|
|
70
51
|
"Observational memory: compaction skipped — another compaction already ran before deferred compaction",
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import { runDropper } from "../agents/dropper/agent.js";
|
|
3
3
|
import { observationPoolMetrics } from "../agents/dropper/pool.js";
|
|
4
|
-
import { runObserver } from "../agents/observer/agent.js";
|
|
4
|
+
import { ObserverStreamError, runObserver } from "../agents/observer/agent.js";
|
|
5
5
|
import { runReflector } from "../agents/reflector/agent.js";
|
|
6
6
|
import { debugLog, withDebugLogContext } from "../debug-log.js";
|
|
7
|
-
import {
|
|
7
|
+
import { resolveObserverChunkMaxTokens } from "../config.js";
|
|
8
|
+
import type { ResolveResult, Runtime } from "../runtime.js";
|
|
8
9
|
import { serializeSourceAddressedBranchEntries } from "../serialize.js";
|
|
9
10
|
import {
|
|
10
11
|
OM_OBSERVATIONS_DROPPED,
|
|
@@ -20,11 +21,14 @@ import {
|
|
|
20
21
|
latestCoverageIndex,
|
|
21
22
|
latestCoverageMarkerId,
|
|
22
23
|
observationToSummaryLine,
|
|
24
|
+
realTokensSinceAnchor,
|
|
23
25
|
rawTokensSinceObservationCoverage,
|
|
24
26
|
rawTokensSinceReflectionCoverage,
|
|
25
27
|
reflectionToSummaryLine,
|
|
26
28
|
type Entry,
|
|
29
|
+
type Observation,
|
|
27
30
|
type Reflection,
|
|
31
|
+
type V3MemoryCustomType,
|
|
28
32
|
} from "../session-ledger/index.js";
|
|
29
33
|
|
|
30
34
|
type ResolvedModel = Extract<ResolveResult, { ok: true }>;
|
|
@@ -35,6 +39,7 @@ type ConsolidationCtx = {
|
|
|
35
39
|
ui?: { notify: (message: string, type?: "warning" | "info" | "error") => void };
|
|
36
40
|
model: unknown;
|
|
37
41
|
modelRegistry: any;
|
|
42
|
+
getContextUsage?: () => { tokens?: number | null; contextWindow?: number } | undefined;
|
|
38
43
|
sessionManager: {
|
|
39
44
|
getBranch: () => unknown;
|
|
40
45
|
getSessionId?: () => string;
|
|
@@ -69,9 +74,43 @@ function mergeReflections(existing: Reflection[], additional: Reflection[]): Ref
|
|
|
69
74
|
return merged;
|
|
70
75
|
}
|
|
71
76
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
77
|
+
/**
|
|
78
|
+
* Real current context tokens from the session (provider-reported usage, the
|
|
79
|
+
* same basis the footer percentage uses). Falls back to undefined when the
|
|
80
|
+
* host pi lacks getContextUsage or the count is unknown (e.g. right after a
|
|
81
|
+
* compaction, before the next valid assistant response).
|
|
82
|
+
*/
|
|
83
|
+
function realContextTokens(ctx: ConsolidationCtx): number | undefined {
|
|
84
|
+
const usage = typeof ctx.getContextUsage === "function" ? ctx.getContextUsage() : undefined;
|
|
85
|
+
const tokens = usage?.tokens;
|
|
86
|
+
return typeof tokens === "number" && Number.isFinite(tokens) ? tokens : undefined;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function stageDue(
|
|
90
|
+
entries: Entry[],
|
|
91
|
+
runtime: Runtime,
|
|
92
|
+
currentTokens: number | undefined,
|
|
93
|
+
customType: V3MemoryCustomType,
|
|
94
|
+
rawEstimateFn: (entries: Entry[]) => number,
|
|
95
|
+
threshold: number,
|
|
96
|
+
): boolean {
|
|
97
|
+
if (currentTokens !== undefined) {
|
|
98
|
+
const real = realTokensSinceAnchor(entries, customType, currentTokens);
|
|
99
|
+
if (real !== undefined) return real >= threshold;
|
|
100
|
+
}
|
|
101
|
+
// Real delta unmeasurable (no usage baseline, or accounting basis changed) or
|
|
102
|
+
// old pi host without getContextUsage — fall back to the raw estimate, which
|
|
103
|
+
// self-limits after coverage and cannot over-fire or starve.
|
|
104
|
+
return rawEstimateFn(entries) >= threshold;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function anyStageDue(entries: Entry[], runtime: Runtime, currentTokens: number | undefined): boolean {
|
|
108
|
+
return stageDue(entries, runtime, currentTokens, OM_OBSERVATIONS_RECORDED, rawTokensSinceObservationCoverage, runtime.config.observeAfterTokens)
|
|
109
|
+
|| stageDue(entries, runtime, currentTokens, OM_REFLECTIONS_RECORDED, rawTokensSinceReflectionCoverage, runtime.config.reflectAfterTokens);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function shouldNotifyWorker(runtime: Runtime, ctx: ConsolidationCtx): boolean {
|
|
113
|
+
return runtime.config.showWorkerNotifications && ctx.hasUI;
|
|
75
114
|
}
|
|
76
115
|
|
|
77
116
|
function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "observer" | "reflector" | "dropper") => Promise<ResolvedModel | undefined> {
|
|
@@ -85,6 +124,22 @@ function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "ob
|
|
|
85
124
|
});
|
|
86
125
|
if (cached.ok) {
|
|
87
126
|
runtime.resolveFailureNotified = false;
|
|
127
|
+
// Console Go (opencode.ai) rejects requests without x-opencode-session
|
|
128
|
+
// (400 MissingSessionID). Mirror pi's own session headers on worker calls.
|
|
129
|
+
const model = (cached.model ?? {}) as { provider?: string; baseUrl?: string };
|
|
130
|
+
if (model.provider === "opencode" || model.provider === "opencode-go" || (typeof model.baseUrl === "string" && model.baseUrl.includes("opencode.ai"))) {
|
|
131
|
+
const sessionId = ctx.sessionManager.getSessionId?.();
|
|
132
|
+
if (sessionId) {
|
|
133
|
+
return {
|
|
134
|
+
...cached,
|
|
135
|
+
headers: {
|
|
136
|
+
...(cached.headers ?? {}),
|
|
137
|
+
"x-opencode-session": sessionId,
|
|
138
|
+
"x-opencode-client": "pi",
|
|
139
|
+
},
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
}
|
|
88
143
|
return cached;
|
|
89
144
|
}
|
|
90
145
|
debugLog(`${stage}.model_unavailable`, { reason: cached.reason });
|
|
@@ -121,7 +176,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
|
|
|
121
176
|
if (runtime.consolidationInFlight) return;
|
|
122
177
|
|
|
123
178
|
const entries = ctx.sessionManager.getBranch() as Entry[];
|
|
124
|
-
if (!anyStageDue(entries, runtime)) return;
|
|
179
|
+
if (!anyStageDue(entries, runtime, realContextTokens(ctx))) return;
|
|
125
180
|
|
|
126
181
|
const runId = `consolidation-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
|
|
127
182
|
const consolidationCtx: ConsolidationCtx = {
|
|
@@ -130,6 +185,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
|
|
|
130
185
|
ui: ctx.ui,
|
|
131
186
|
model: ctx.model,
|
|
132
187
|
modelRegistry: ctx.modelRegistry,
|
|
188
|
+
getContextUsage: ctx.getContextUsage,
|
|
133
189
|
sessionManager: ctx.sessionManager,
|
|
134
190
|
};
|
|
135
191
|
|
|
@@ -185,27 +241,79 @@ async function runObserverStage(
|
|
|
185
241
|
resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
|
|
186
242
|
): Promise<StageOutcome> {
|
|
187
243
|
const entries = ctx.sessionManager.getBranch() as Entry[];
|
|
188
|
-
const
|
|
244
|
+
const currentTokens = realContextTokens(ctx);
|
|
245
|
+
const real = currentTokens !== undefined ? realTokensSinceAnchor(entries, OM_OBSERVATIONS_RECORDED, currentTokens) : undefined;
|
|
246
|
+
const tokens = real !== undefined ? real : rawTokensSinceObservationCoverage(entries); // fallback: no usage baseline / basis change
|
|
189
247
|
if (tokens < runtime.config.observeAfterTokens) return "continue";
|
|
190
248
|
|
|
249
|
+
const sessionMetadata = debugSessionMetadata(ctx);
|
|
250
|
+
const sessionIdentity = sessionMetadata.sessionId ?? sessionMetadata.sessionFile;
|
|
251
|
+
const coverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
|
|
252
|
+
|
|
253
|
+
// Deliberate-empty backoff (#23): an intentional "nothing to record" verdict
|
|
254
|
+
// must not re-fire the observer every turn over the same span. Retry only
|
|
255
|
+
// after another observeAfterTokens worth of new source tokens arrives, and
|
|
256
|
+
// drop the backoff as soon as coverage advances.
|
|
257
|
+
const backoff = runtime.observerEmptyBackoff;
|
|
258
|
+
if (backoff) {
|
|
259
|
+
if (
|
|
260
|
+
sessionIdentity !== backoff.sessionIdentity
|
|
261
|
+
|| coverageId !== backoff.coverageId
|
|
262
|
+
|| tokens >= backoff.tokensAtEmpty + runtime.config.observeAfterTokens
|
|
263
|
+
) {
|
|
264
|
+
runtime.observerEmptyBackoff = undefined;
|
|
265
|
+
} else {
|
|
266
|
+
debugLog("observer.empty_backoff", { tokens, resumeAtTokens: backoff.tokensAtEmpty + runtime.config.observeAfterTokens });
|
|
267
|
+
return "continue";
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Resolve the model before building the chunk: the default chunk cap
|
|
272
|
+
// derives from the resolved model's context window.
|
|
273
|
+
const resolved = await resolveModel("observer");
|
|
274
|
+
if (!resolved) return "abort";
|
|
275
|
+
|
|
191
276
|
const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
|
|
192
|
-
const
|
|
193
|
-
|
|
277
|
+
const backlogEntries = sourceEntriesAfter(entries, lastCoverageIdx);
|
|
278
|
+
|
|
279
|
+
// Budget the text that is actually sent to the observer, including source
|
|
280
|
+
// labels and rendered message content. Complete entries are kept intact.
|
|
281
|
+
// Only a first entry that cannot fit by itself is represented by a clearly
|
|
282
|
+
// marked head/tail excerpt; the original ledger entry remains untouched.
|
|
283
|
+
const contextWindow = (resolved.model as { contextWindow?: number }).contextWindow;
|
|
284
|
+
const maxChunkTokens = resolveObserverChunkMaxTokens(runtime.config, contextWindow);
|
|
285
|
+
const {
|
|
286
|
+
text: chunk,
|
|
287
|
+
sourceEntryIds,
|
|
288
|
+
estimatedTokens: chunkTokens,
|
|
289
|
+
truncatedSourceEntryIds,
|
|
290
|
+
} = serializeSourceAddressedBranchEntries(backlogEntries, { maxTokens: maxChunkTokens });
|
|
291
|
+
if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
|
|
292
|
+
const coversUpToId = sourceEntryIds.at(-1);
|
|
194
293
|
if (!coversUpToId) return "continue";
|
|
195
294
|
|
|
196
|
-
|
|
197
|
-
|
|
295
|
+
if (sourceEntryIds.length < backlogEntries.length || truncatedSourceEntryIds.length > 0) {
|
|
296
|
+
debugLog("observer.chunk_capped", {
|
|
297
|
+
maxChunkTokens,
|
|
298
|
+
backlogEntries: backlogEntries.length,
|
|
299
|
+
backlogTokens: tokens,
|
|
300
|
+
chunkEntries: sourceEntryIds.length,
|
|
301
|
+
chunkTokens,
|
|
302
|
+
truncatedSourceEntryIds,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
198
305
|
|
|
199
306
|
const memory = fullProjection(entries);
|
|
200
307
|
const priorReflections = memory.reflections.map(reflectionToSummaryLine);
|
|
201
308
|
const priorObservations = memory.observations.map(observationToSummaryLine);
|
|
202
309
|
|
|
203
|
-
if (ctx
|
|
204
|
-
`Observational memory: observer running on ~${
|
|
310
|
+
if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
|
|
311
|
+
`Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk`,
|
|
205
312
|
"info",
|
|
206
313
|
);
|
|
207
314
|
debugLog("observer.start", {
|
|
208
315
|
tokens,
|
|
316
|
+
chunkTokens,
|
|
209
317
|
coversUpToId,
|
|
210
318
|
sourceEntryIds,
|
|
211
319
|
sourceEntryCount: sourceEntryIds.length,
|
|
@@ -213,28 +321,43 @@ async function runObserverStage(
|
|
|
213
321
|
priorObservations: priorObservations.length,
|
|
214
322
|
});
|
|
215
323
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
324
|
+
let observations: Observation[] | undefined;
|
|
325
|
+
try {
|
|
326
|
+
observations = await runObserver({
|
|
327
|
+
model: resolved.model as any,
|
|
328
|
+
apiKey: resolved.apiKey,
|
|
329
|
+
headers: resolved.headers,
|
|
330
|
+
env: resolved.env,
|
|
331
|
+
priorReflections,
|
|
332
|
+
priorObservations,
|
|
333
|
+
chunk,
|
|
334
|
+
allowedSourceEntryIds: sourceEntryIds,
|
|
335
|
+
maxTurns: runtime.config.agentMaxTurns,
|
|
336
|
+
maxOutputTokens: runtime.config.agentMaxTokens,
|
|
337
|
+
thinkingLevel: runtime.config.model?.thinking ?? "low",
|
|
338
|
+
modelRegistry: ctx.modelRegistry,
|
|
339
|
+
});
|
|
340
|
+
} catch (error) {
|
|
341
|
+
if (error instanceof ObserverStreamError) {
|
|
342
|
+
// API/stream failure is not a clean empty (#32): surface it as a real
|
|
343
|
+
// failure instead of the "no observations" path. Coverage stays put.
|
|
344
|
+
runtime.recordConsolidationStageError(ctx, "observer", error);
|
|
345
|
+
return "abort";
|
|
346
|
+
}
|
|
347
|
+
throw error;
|
|
348
|
+
}
|
|
230
349
|
if (!observations || observations.length === 0) {
|
|
350
|
+
// Deliberate empty: routine info, not a warning, and back off re-fires
|
|
351
|
+
// over the same span (#23).
|
|
231
352
|
debugLog("observer.empty", { coversUpToId });
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
"
|
|
353
|
+
runtime.observerEmptyBackoff = { sessionIdentity, coverageId, tokensAtEmpty: tokens };
|
|
354
|
+
if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
|
|
355
|
+
"Observational memory: observer found nothing new in this chunk (coverage unchanged; will retry later)",
|
|
356
|
+
"info",
|
|
235
357
|
);
|
|
236
358
|
return "continue";
|
|
237
359
|
}
|
|
360
|
+
runtime.observerEmptyBackoff = undefined;
|
|
238
361
|
|
|
239
362
|
const data = buildObservationsRecordedData(observations, coversUpToId);
|
|
240
363
|
if (!data) return "continue";
|
|
@@ -245,7 +368,7 @@ async function runObserverStage(
|
|
|
245
368
|
});
|
|
246
369
|
appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
|
|
247
370
|
debugLog("observer.appended", { count: observations.length, coversUpToId });
|
|
248
|
-
if (ctx
|
|
371
|
+
if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
|
|
249
372
|
`Observational memory: ${observations.length} observation${observations.length === 1 ? "" : "s"} recorded`,
|
|
250
373
|
"info",
|
|
251
374
|
);
|
|
@@ -259,13 +382,15 @@ async function runReflectorStage(
|
|
|
259
382
|
resolveModel: (stage: "reflector") => Promise<ResolvedModel | undefined>,
|
|
260
383
|
): Promise<ReflectorStageResult> {
|
|
261
384
|
const entries = ctx.sessionManager.getBranch() as Entry[];
|
|
262
|
-
const
|
|
385
|
+
const currentTokens = realContextTokens(ctx);
|
|
386
|
+
const real = currentTokens !== undefined ? realTokensSinceAnchor(entries, OM_REFLECTIONS_RECORDED, currentTokens) : undefined;
|
|
387
|
+
const reflectionTokens = real !== undefined ? real : rawTokensSinceReflectionCoverage(entries); // fallback: no usage baseline / basis change
|
|
263
388
|
if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
|
|
264
389
|
|
|
265
390
|
const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
|
|
266
391
|
if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
|
|
267
392
|
|
|
268
|
-
if (ctx
|
|
393
|
+
if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
|
|
269
394
|
`Observational memory: reflector running (~${reflectionTokens.toLocaleString()} tokens)`,
|
|
270
395
|
"info",
|
|
271
396
|
);
|
|
@@ -277,10 +402,13 @@ async function runReflectorStage(
|
|
|
277
402
|
model: resolved.model as any,
|
|
278
403
|
apiKey: resolved.apiKey,
|
|
279
404
|
headers: resolved.headers,
|
|
405
|
+
env: resolved.env,
|
|
280
406
|
reflections: folded.reflections,
|
|
281
407
|
observations: folded.activeObservations,
|
|
282
408
|
maxTurns: runtime.config.agentMaxTurns,
|
|
409
|
+
maxOutputTokens: runtime.config.agentMaxTokens,
|
|
283
410
|
thinkingLevel: runtime.config.model?.thinking ?? "low",
|
|
411
|
+
modelRegistry: ctx.modelRegistry,
|
|
284
412
|
});
|
|
285
413
|
if (!reflections) return { outcome: "continue", sameRunReflections: [] };
|
|
286
414
|
|
|
@@ -337,7 +465,7 @@ async function runDropperStage(
|
|
|
337
465
|
maxDropsAllowed: metrics.maxDropsAllowed,
|
|
338
466
|
});
|
|
339
467
|
|
|
340
|
-
if (ctx
|
|
468
|
+
if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
|
|
341
469
|
`Observational memory: dropper running after reflection — active observation pool ~${metrics.observationTokens.toLocaleString()} / ${metrics.targetTokens.toLocaleString()} target tokens (${Math.round(metrics.fullness * 100).toLocaleString()}%)`,
|
|
342
470
|
"info",
|
|
343
471
|
);
|
|
@@ -349,11 +477,14 @@ async function runDropperStage(
|
|
|
349
477
|
model: resolved.model as any,
|
|
350
478
|
apiKey: resolved.apiKey,
|
|
351
479
|
headers: resolved.headers,
|
|
480
|
+
env: resolved.env,
|
|
352
481
|
reflections: reflectionsForDropper,
|
|
353
482
|
observations: folded.activeObservations,
|
|
354
483
|
targetTokens: runtime.config.observationsPoolTargetTokens,
|
|
355
484
|
maxTurns: runtime.config.agentMaxTurns,
|
|
485
|
+
maxOutputTokens: runtime.config.agentMaxTokens,
|
|
356
486
|
thinkingLevel: runtime.config.model?.thinking ?? "low",
|
|
487
|
+
modelRegistry: ctx.modelRegistry,
|
|
357
488
|
});
|
|
358
489
|
const coversUpToId = earlierCoverageMarkerId(entries, observationCoverageId, sameRunReflectionCoverageId);
|
|
359
490
|
const data = coversUpToId && droppedIds ? buildObservationsDroppedData(droppedIds, coversUpToId) : undefined;
|
package/src/runtime.ts
CHANGED
|
@@ -1,13 +1,89 @@
|
|
|
1
1
|
import { type Config, DEFAULTS, loadConfig } from "./config.js";
|
|
2
|
+
import { debugLog } from "./debug-log.js";
|
|
2
3
|
|
|
3
4
|
export type ResolveResult =
|
|
4
|
-
| { ok: true; model: unknown; apiKey
|
|
5
|
+
| { ok: true; model: unknown; apiKey?: string; headers?: Record<string, string>; env?: Record<string, string>; baseUrl?: string }
|
|
5
6
|
| { ok: false; reason: string };
|
|
6
7
|
|
|
8
|
+
/**
|
|
9
|
+
* Mirrors pi's own request-auth acceptance rule (`AgentSession._getRequiredRequestAuth`):
|
|
10
|
+
* resolved auth is usable when it carries an apiKey OR at least one header value.
|
|
11
|
+
* OAuth providers (kimi-coding, xai, openai-codex, anthropic OAuth, …) authenticate via
|
|
12
|
+
* `toAuth()` returning `{ headers: { Authorization: "Bearer …" } }` with no apiKey, and
|
|
13
|
+
* pi-ai providers accept a caller-supplied Authorization header in place of an apiKey.
|
|
14
|
+
*
|
|
15
|
+
* NOTE: a `false` result does NOT mean "unauthenticated" — see `resolveModel`. Providers
|
|
16
|
+
* that authenticate at request time (Amazon Bedrock SigV4 from AWS_PROFILE/SSO, Google
|
|
17
|
+
* Vertex ADC) legitimately expose neither an apiKey nor a header, because pi signs their
|
|
18
|
+
* requests itself.
|
|
19
|
+
*/
|
|
20
|
+
function hasUsableAuth(auth: { apiKey?: unknown; headers?: unknown }): boolean {
|
|
21
|
+
if (typeof auth.apiKey === "string" && auth.apiKey.length > 0) return true;
|
|
22
|
+
return countUsableHeaders(auth.headers) > 0;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** How many headers the auth payload carries at all (diagnostics only, never values). */
|
|
26
|
+
function countHeaders(headers: unknown): number {
|
|
27
|
+
return headers && typeof headers === "object" ? Object.keys(headers as Record<string, unknown>).length : 0;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** How many headers carry a non-empty string value — the ones pi could actually send. */
|
|
31
|
+
function countUsableHeaders(headers: unknown): number {
|
|
32
|
+
if (!headers || typeof headers !== "object") return 0;
|
|
33
|
+
return Object.values(headers as Record<string, unknown>).filter(
|
|
34
|
+
(value) => typeof value === "string" && value.length > 0,
|
|
35
|
+
).length;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* How long to wait for the availability re-check in `recheckProviderCredential`, and how
|
|
40
|
+
* long before the same provider may be re-checked again.
|
|
41
|
+
*
|
|
42
|
+
* The re-check is network-free and measured at ~1ms on a warm Bedrock/SSO host, but
|
|
43
|
+
* `checkAuth` can block on a provider's own credential resolution, so it is bounded. The
|
|
44
|
+
* re-arm interval keeps an unauthenticated host from paying the cost on every
|
|
45
|
+
* consolidation while still recovering within a session when credentials are renewed out
|
|
46
|
+
* of band (`aws sso login` in another terminal, `gcloud auth application-default login`).
|
|
47
|
+
*/
|
|
48
|
+
const AVAILABILITY_RECHECK_TIMEOUT_MS = 5_000;
|
|
49
|
+
const AVAILABILITY_RECHECK_REARM_MS = 60_000;
|
|
50
|
+
|
|
7
51
|
type NotifyLevel = "warning" | "info" | "error";
|
|
8
52
|
type Notify = (message: string, type?: NotifyLevel) => void;
|
|
9
53
|
export type ConsolidationPhase = "observer" | "reflector" | "dropper";
|
|
10
54
|
|
|
55
|
+
/**
|
|
56
|
+
* Whether pi positively reports a working credential source for this model's provider.
|
|
57
|
+
*
|
|
58
|
+
* `ModelRegistry.hasConfiguredAuth(model)` is true when pi's availability check
|
|
59
|
+
* (`ModelRuntime.checkAuth`) resolved *something* for the provider — an API key, a
|
|
60
|
+
* stored credential, or an ambient source such as `AWS_PROFILE` / `AWS_ACCESS_KEY_ID`
|
|
61
|
+
* / gcloud ADC. Combined with `auth.ok === true` and an auth payload that carries
|
|
62
|
+
* nothing, that is the signature of a provider pi signs at request time:
|
|
63
|
+
*
|
|
64
|
+
* pi has a credential source, and deliberately hands the caller nothing to attach.
|
|
65
|
+
*
|
|
66
|
+
* Measured on a Bedrock/SSO host (pi 0.84.2), with `AWS_PROFILE` exported:
|
|
67
|
+
* checkAuth("amazon-bedrock") -> { source: "AWS_PROFILE", type: "api_key" }
|
|
68
|
+
* hasConfiguredAuth(model) -> true
|
|
69
|
+
* getApiKeyAndHeaders(model) -> { ok: true, apiKey: undefined, headers: undefined }
|
|
70
|
+
* `googleVertexProvider`'s ADC branch returns the same empty-auth resolution.
|
|
71
|
+
*
|
|
72
|
+
* The inverse case — `hasConfiguredAuth === false` with an empty auth payload — is a
|
|
73
|
+
* provider pi could not authenticate at all (no key, no ambient source). That must
|
|
74
|
+
* keep failing: it is the ordinary "not logged in" state, not ambient auth.
|
|
75
|
+
*
|
|
76
|
+
* Defensive: older pi versions and partial test doubles may not expose this, and an
|
|
77
|
+
* unknown answer must not be read as "authenticated".
|
|
78
|
+
*/
|
|
79
|
+
function hasConfiguredProviderCredential(registry: unknown, model: unknown): boolean {
|
|
80
|
+
try {
|
|
81
|
+
return (registry as { hasConfiguredAuth?: (m: unknown) => unknown }).hasConfiguredAuth?.(model) === true;
|
|
82
|
+
} catch {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
11
87
|
export interface ResolveCtx {
|
|
12
88
|
model: unknown;
|
|
13
89
|
modelRegistry: any;
|
|
@@ -32,6 +108,14 @@ export class Runtime {
|
|
|
32
108
|
lastObserverError: string | undefined;
|
|
33
109
|
lastReflectorError: string | undefined;
|
|
34
110
|
lastDropperError: string | undefined;
|
|
111
|
+
/** provider -> epoch ms of the last availability re-check (see `recheckProviderCredential`). */
|
|
112
|
+
availabilityRecheckedAt = new Map<string, number>();
|
|
113
|
+
/** Deliberate-empty backoff (#23): skip observer re-fires over the same span until enough new tokens arrive. */
|
|
114
|
+
observerEmptyBackoff: {
|
|
115
|
+
sessionIdentity: string | undefined;
|
|
116
|
+
coverageId: string | undefined;
|
|
117
|
+
tokensAtEmpty: number;
|
|
118
|
+
} | undefined;
|
|
35
119
|
|
|
36
120
|
ensureConfig(cwd: string): void {
|
|
37
121
|
if (this.configLoaded) return;
|
|
@@ -54,11 +138,149 @@ export class Runtime {
|
|
|
54
138
|
}
|
|
55
139
|
if (!model) return { ok: false, reason: "no model available (session has no model and no observational-memory model configured)" };
|
|
56
140
|
const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
141
|
+
const provider = (model as { provider?: string }).provider ?? "unknown";
|
|
142
|
+
const isOAuth = ctx.modelRegistry.isUsingOAuth?.(model) === true;
|
|
143
|
+
// `auth.ok === false` is the only unambiguous failure: pi returns it when a
|
|
144
|
+
// provider requires a request auth header and no credential resolved.
|
|
145
|
+
//
|
|
146
|
+
// `auth.ok === true` with neither apiKey nor headers, for a provider pi DOES
|
|
147
|
+
// report a credential source for, is not a failure — it is how pi describes a
|
|
148
|
+
// provider that authenticates at request time: Amazon Bedrock signing SigV4 from
|
|
149
|
+
// ambient AWS credentials (`bedrockAuth.resolve` returns `{ auth: {}, source:
|
|
150
|
+
// "AWS_PROFILE" }`), Google Vertex using ADC (same empty resolution). pi's own
|
|
151
|
+
// native streaming path forwards no apiKey either, and its pre-prompt gate is
|
|
152
|
+
// merely `hasConfiguredAuth(provider) || checkAuth(provider) !== undefined` —
|
|
153
|
+
// om's pre-flight check must not be stricter than pi's own. Treating it as "no
|
|
154
|
+
// auth" aborted consolidation before the model was ever called, disabling
|
|
155
|
+
// observational memory silently — no error, no cost, no latency — on such hosts.
|
|
156
|
+
//
|
|
157
|
+
// Three cases deliberately keep failing: OAuth providers, where an empty
|
|
158
|
+
// resolution means the credentials no longer resolve and the user must log in
|
|
159
|
+
// again; a credential that resolved to an empty *string* key, which is a
|
|
160
|
+
// misconfiguration rather than ambient auth; and a provider pi reports no
|
|
161
|
+
// credential source for at all, which is simply unauthenticated.
|
|
162
|
+
const usable = hasUsableAuth(auth);
|
|
163
|
+
const resolvedEmptyApiKey = typeof auth.apiKey === "string" && auth.apiKey.length === 0;
|
|
164
|
+
let providerCredentialConfigured = hasConfiguredProviderCredential(ctx.modelRegistry, model);
|
|
165
|
+
// pi's gate has TWO halves and never trusts the snapshot alone (agent-session.js):
|
|
166
|
+
//
|
|
167
|
+
// hasConfiguredAuth(provider) || (await checkAuth(provider)) !== undefined
|
|
168
|
+
//
|
|
169
|
+
// `hasConfiguredAuth` reads `snapshot.configuredProviders`, which is populated by an
|
|
170
|
+
// availability pass — and left untouched when that pass is skipped
|
|
171
|
+
// (`refreshOnCreate: false`), aborted, or FAILS (its catch records `availabilityError`
|
|
172
|
+
// and returns). A provider whose credential could not be checked at startup — an
|
|
173
|
+
// expired SSO token, say — is therefore absent from the snapshot for the rest of the
|
|
174
|
+
// session, even after the user renews it out of band. pi recovers on the next turn
|
|
175
|
+
// because its second half re-checks live; reading only the snapshot half would leave
|
|
176
|
+
// consolidation dead for the whole session, which is the same silent-failure class as
|
|
177
|
+
// the bug this gate was fixed for.
|
|
178
|
+
//
|
|
179
|
+
// The facade exposes no `checkAuth`, but `refresh({ providers })` performs the same
|
|
180
|
+
// live check and then updates the snapshot, so re-reading afterwards is equivalent.
|
|
181
|
+
// Only attempted when everything else already looks like the ambient shape, so an
|
|
182
|
+
// ordinary unauthenticated provider still fails on the first call.
|
|
183
|
+
if (auth.ok === true && !usable && !isOAuth && !resolvedEmptyApiKey && !providerCredentialConfigured) {
|
|
184
|
+
providerCredentialConfigured = await this.recheckProviderCredential(ctx.modelRegistry, model, provider);
|
|
60
185
|
}
|
|
61
|
-
|
|
186
|
+
const signsAtRequestTime =
|
|
187
|
+
auth.ok === true && !isOAuth && !resolvedEmptyApiKey && providerCredentialConfigured;
|
|
188
|
+
if (!auth.ok || (!usable && !signsAtRequestTime)) {
|
|
189
|
+
const reason = isOAuth
|
|
190
|
+
? `authentication failed for provider "${provider}" — OAuth credentials may have expired; run '/login ${provider}' to re-authenticate`
|
|
191
|
+
: `no API key or auth headers for provider "${provider}"`;
|
|
192
|
+
// The reason string alone cannot tell `ok: false` from `ok: true` with nothing to
|
|
193
|
+
// carry, which is what made the ambient-credential outage un-diagnosable from the
|
|
194
|
+
// debug log. Record the decision inputs — booleans and counts only, never values.
|
|
195
|
+
debugLog("resolve.rejected", {
|
|
196
|
+
provider,
|
|
197
|
+
reason,
|
|
198
|
+
authOk: auth.ok === true,
|
|
199
|
+
hasApiKey: typeof auth.apiKey === "string" && auth.apiKey.length > 0,
|
|
200
|
+
resolvedEmptyApiKey,
|
|
201
|
+
headerCount: countHeaders(auth.headers),
|
|
202
|
+
usableHeaderCount: countUsableHeaders(auth.headers),
|
|
203
|
+
isOAuth,
|
|
204
|
+
providerCredentialConfigured,
|
|
205
|
+
signsAtRequestTime,
|
|
206
|
+
});
|
|
207
|
+
return { ok: false, reason };
|
|
208
|
+
}
|
|
209
|
+
if (!usable) {
|
|
210
|
+
debugLog("resolve.request_time_signing", { provider, providerCredentialConfigured });
|
|
211
|
+
}
|
|
212
|
+
// Match pi's request model: OAuth may route to an account-specific endpoint
|
|
213
|
+
// (e.g. Copilot Business). Do not mutate the shared session/registry model.
|
|
214
|
+
const requestModel = auth.baseUrl ? { ...(model as object), baseUrl: auth.baseUrl } : model;
|
|
215
|
+
return {
|
|
216
|
+
ok: true,
|
|
217
|
+
model: requestModel,
|
|
218
|
+
apiKey: auth.apiKey as string | undefined,
|
|
219
|
+
headers: auth.headers as Record<string, string> | undefined,
|
|
220
|
+
env: auth.env as Record<string, string> | undefined,
|
|
221
|
+
baseUrl: auth.baseUrl as string | undefined,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Re-check one provider's credential live, then re-read pi's snapshot.
|
|
227
|
+
*
|
|
228
|
+
* Implements the second half of pi's own auth gate for the only case that needs it: an
|
|
229
|
+
* otherwise-ambient-looking resolution whose provider is missing from a stale or never
|
|
230
|
+
* populated availability snapshot. Bounded and rate-limited; never throws.
|
|
231
|
+
*/
|
|
232
|
+
private async recheckProviderCredential(registry: unknown, model: unknown, provider: string): Promise<boolean> {
|
|
233
|
+
const last = this.availabilityRecheckedAt.get(provider);
|
|
234
|
+
const now = Date.now();
|
|
235
|
+
if (last !== undefined && now - last < AVAILABILITY_RECHECK_REARM_MS) return false;
|
|
236
|
+
this.availabilityRecheckedAt.set(provider, now);
|
|
237
|
+
|
|
238
|
+
const refresh = (registry as { refresh?: (options?: unknown) => Promise<unknown> }).refresh;
|
|
239
|
+
if (typeof refresh !== "function") {
|
|
240
|
+
debugLog("resolve.availability_recheck", { provider, refreshed: false, reason: "registry exposes no refresh()" });
|
|
241
|
+
return false;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const controller = new AbortController();
|
|
245
|
+
const timer = setTimeout(() => controller.abort(), AVAILABILITY_RECHECK_TIMEOUT_MS);
|
|
246
|
+
let refreshError: string | undefined;
|
|
247
|
+
let timedOut = false;
|
|
248
|
+
try {
|
|
249
|
+
// allowNetwork:false — a credential re-check must not wait on a model-catalog fetch.
|
|
250
|
+
// providers:[provider] — scope the work, and the snapshot writes, to the one provider.
|
|
251
|
+
//
|
|
252
|
+
// Both are honoured from pi 0.84; on pi 0.81 the facade is `refresh()` with no
|
|
253
|
+
// parameters, delegating to `runtime.reloadConfig()`, which reloads models.json and
|
|
254
|
+
// then runs a FULL, network-permitted availability pass. Passing the options is
|
|
255
|
+
// harmless there, but the work is wider and slower — hence the race below rather
|
|
256
|
+
// than relying on the abort signal, which that version never sees.
|
|
257
|
+
await Promise.race([
|
|
258
|
+
refresh.call(registry, { allowNetwork: false, providers: [provider], signal: controller.signal }),
|
|
259
|
+
new Promise<void>((resolve) => {
|
|
260
|
+
controller.signal.addEventListener("abort", () => {
|
|
261
|
+
timedOut = true;
|
|
262
|
+
resolve();
|
|
263
|
+
});
|
|
264
|
+
}),
|
|
265
|
+
]);
|
|
266
|
+
} catch (error) {
|
|
267
|
+
refreshError = error instanceof Error ? error.message : String(error);
|
|
268
|
+
} finally {
|
|
269
|
+
clearTimeout(timer);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Re-read even when the refresh reported an error or timed out: a scoped pass can
|
|
273
|
+
// update the snapshot for this provider and still fail elsewhere.
|
|
274
|
+
const recovered = hasConfiguredProviderCredential(registry, model);
|
|
275
|
+
debugLog("resolve.availability_recheck", {
|
|
276
|
+
provider,
|
|
277
|
+
refreshed: refreshError === undefined && !timedOut,
|
|
278
|
+
recovered,
|
|
279
|
+
elapsedMs: Date.now() - now,
|
|
280
|
+
timedOut,
|
|
281
|
+
...(refreshError === undefined ? {} : { refreshError }),
|
|
282
|
+
});
|
|
283
|
+
return recovered;
|
|
62
284
|
}
|
|
63
285
|
|
|
64
286
|
launchConsolidationTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> {
|