pi-observational-memory 3.0.3 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,43 +3,20 @@ import { resolveCompactAfterTokens } from "../config.js";
3
3
  import { rawTokensSinceLastCompaction, type Entry } from "../session-ledger/index.js";
4
4
  import type { Runtime } from "../runtime.js";
5
5
 
6
- /**
7
- * Regex matching Pi's internal retryable error detection.
8
- * When the last assistant message in agent_end has stopReason "error" matching this pattern,
9
- * Pi will auto-retry — we must not trigger compaction between attempts.
10
- */
11
- const RETRYABLE_ERROR_RE =
12
- /overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|http2 request did not get a response|timed? out|timeout|terminated|retry delay/i;
13
-
14
6
  export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): void {
15
- pi.on("agent_end", (event: any, ctx: any) => {
7
+ // Pi emits agent_settled only after retries, automatic compaction, and queued
8
+ // continuation have finished, so retry policy stays owned by Pi.
9
+ pi.on("agent_settled", (_event, ctx) => {
16
10
  runtime.ensureConfig(ctx.cwd);
17
11
  if (runtime.config.passive === true) return;
18
12
  if (runtime.compactInFlight) return;
19
13
 
20
- // Don't trigger compaction if Pi will auto-retry — the agent hasn't truly finished.
21
- // Pi emits agent_end before its own retry check, so we must detect this ourselves.
22
- // The next agent_end (after retry succeeds or exhausts attempts) will re-evaluate.
23
- const lastAssistant = [...event.messages].reverse().find(
24
- (m): m is Extract<typeof m, { role: "assistant" }> => m.role === "assistant",
25
- );
26
- if (
27
- lastAssistant
28
- && lastAssistant.stopReason === "error"
29
- && lastAssistant.errorMessage
30
- && RETRYABLE_ERROR_RE.test(lastAssistant.errorMessage)
31
- ) {
32
- return;
33
- }
34
-
35
- const entries = ctx.sessionManager.getBranch() as Entry[];
36
- const tokens = rawTokensSinceLastCompaction(entries);
37
- // Resolve the proactive-compaction threshold from the active model's context
38
- // window when ratio mode is configured. ctx.model is the current session model
39
- // (Model<any> | undefined per ExtensionContext).
14
+ const entries = ctx.sessionManager?.getBranch?.() as Entry[] | undefined;
15
+ if (!entries) return;
16
+ const progress = rawTokensSinceLastCompaction(entries);
40
17
  const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : undefined;
41
18
  const threshold = resolveCompactAfterTokens(runtime.config, contextWindow);
42
- if (tokens < threshold) return;
19
+ if (progress < threshold) return;
43
20
 
44
21
  // Capture ctx properties synchronously — the setTimeout + async work below
45
22
  // may outlive the extension ctx (stale after session replacement/reload).
@@ -47,7 +24,7 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
47
24
  const ui = ctx.ui;
48
25
 
49
26
  if (hasUI) ui?.notify(
50
- `Observational memory: compaction threshold reached (~${tokens.toLocaleString()} tokens); triggering compaction`,
27
+ `Observational memory: compaction threshold reached (~${progress.toLocaleString()} estimated source tokens); triggering compaction`,
51
28
  "info",
52
29
  );
53
30
 
@@ -62,9 +39,13 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
62
39
  );
63
40
  return;
64
41
  }
65
- const currentEntries = ctx.sessionManager.getBranch() as Entry[];
66
- const currentTokens = rawTokensSinceLastCompaction(currentEntries);
67
- if (currentTokens < threshold) {
42
+ const currentEntries = ctx.sessionManager?.getBranch?.() as Entry[] | undefined;
43
+ if (!currentEntries) {
44
+ runtime.compactInFlight = false;
45
+ return;
46
+ }
47
+ const currentProgress = rawTokensSinceLastCompaction(currentEntries);
48
+ if (currentProgress < threshold) {
68
49
  runtime.compactInFlight = false;
69
50
  if (hasUI) ui?.notify(
70
51
  "Observational memory: compaction skipped — another compaction already ran before deferred compaction",
@@ -1,10 +1,11 @@
1
1
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2
2
  import { runDropper } from "../agents/dropper/agent.js";
3
3
  import { observationPoolMetrics } from "../agents/dropper/pool.js";
4
- import { runObserver } from "../agents/observer/agent.js";
4
+ import { ObserverStreamError, runObserver } from "../agents/observer/agent.js";
5
5
  import { runReflector } from "../agents/reflector/agent.js";
6
6
  import { debugLog, withDebugLogContext } from "../debug-log.js";
7
- import { type ResolveResult, type Runtime } from "../runtime.js";
7
+ import { resolveObserverChunkMaxTokens } from "../config.js";
8
+ import type { ResolveResult, Runtime } from "../runtime.js";
8
9
  import { serializeSourceAddressedBranchEntries } from "../serialize.js";
9
10
  import {
10
11
  OM_OBSERVATIONS_DROPPED,
@@ -20,11 +21,14 @@ import {
20
21
  latestCoverageIndex,
21
22
  latestCoverageMarkerId,
22
23
  observationToSummaryLine,
24
+ realTokensSinceAnchor,
23
25
  rawTokensSinceObservationCoverage,
24
26
  rawTokensSinceReflectionCoverage,
25
27
  reflectionToSummaryLine,
26
28
  type Entry,
29
+ type Observation,
27
30
  type Reflection,
31
+ type V3MemoryCustomType,
28
32
  } from "../session-ledger/index.js";
29
33
 
30
34
  type ResolvedModel = Extract<ResolveResult, { ok: true }>;
@@ -35,6 +39,7 @@ type ConsolidationCtx = {
35
39
  ui?: { notify: (message: string, type?: "warning" | "info" | "error") => void };
36
40
  model: unknown;
37
41
  modelRegistry: any;
42
+ getContextUsage?: () => { tokens?: number | null; contextWindow?: number } | undefined;
38
43
  sessionManager: {
39
44
  getBranch: () => unknown;
40
45
  getSessionId?: () => string;
@@ -69,9 +74,43 @@ function mergeReflections(existing: Reflection[], additional: Reflection[]): Ref
69
74
  return merged;
70
75
  }
71
76
 
72
- function anyStageDue(entries: Entry[], runtime: Runtime): boolean {
73
- return rawTokensSinceObservationCoverage(entries) >= runtime.config.observeAfterTokens
74
- || rawTokensSinceReflectionCoverage(entries) >= runtime.config.reflectAfterTokens;
77
+ /**
78
+ * Real current context tokens from the session (provider-reported usage, the
79
+ * same basis the footer percentage uses). Falls back to undefined when the
80
+ * host pi lacks getContextUsage or the count is unknown (e.g. right after a
81
+ * compaction, before the next valid assistant response).
82
+ */
83
+ function realContextTokens(ctx: ConsolidationCtx): number | undefined {
84
+ const usage = typeof ctx.getContextUsage === "function" ? ctx.getContextUsage() : undefined;
85
+ const tokens = usage?.tokens;
86
+ return typeof tokens === "number" && Number.isFinite(tokens) ? tokens : undefined;
87
+ }
88
+
89
+ function stageDue(
90
+ entries: Entry[],
91
+ runtime: Runtime,
92
+ currentTokens: number | undefined,
93
+ customType: V3MemoryCustomType,
94
+ rawEstimateFn: (entries: Entry[]) => number,
95
+ threshold: number,
96
+ ): boolean {
97
+ if (currentTokens !== undefined) {
98
+ const real = realTokensSinceAnchor(entries, customType, currentTokens);
99
+ if (real !== undefined) return real >= threshold;
100
+ }
101
+ // Real delta unmeasurable (no usage baseline, or accounting basis changed) or
102
+ // old pi host without getContextUsage — fall back to the raw estimate, which
103
+ // self-limits after coverage and cannot over-fire or starve.
104
+ return rawEstimateFn(entries) >= threshold;
105
+ }
106
+
107
+ function anyStageDue(entries: Entry[], runtime: Runtime, currentTokens: number | undefined): boolean {
108
+ return stageDue(entries, runtime, currentTokens, OM_OBSERVATIONS_RECORDED, rawTokensSinceObservationCoverage, runtime.config.observeAfterTokens)
109
+ || stageDue(entries, runtime, currentTokens, OM_REFLECTIONS_RECORDED, rawTokensSinceReflectionCoverage, runtime.config.reflectAfterTokens);
110
+ }
111
+
112
+ function shouldNotifyWorker(runtime: Runtime, ctx: ConsolidationCtx): boolean {
113
+ return runtime.config.showWorkerNotifications && ctx.hasUI;
75
114
  }
76
115
 
77
116
  function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "observer" | "reflector" | "dropper") => Promise<ResolvedModel | undefined> {
@@ -85,6 +124,22 @@ function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "ob
85
124
  });
86
125
  if (cached.ok) {
87
126
  runtime.resolveFailureNotified = false;
127
+ // Console Go (opencode.ai) rejects requests without x-opencode-session
128
+ // (400 MissingSessionID). Mirror pi's own session headers on worker calls.
129
+ const model = (cached.model ?? {}) as { provider?: string; baseUrl?: string };
130
+ if (model.provider === "opencode" || model.provider === "opencode-go" || (typeof model.baseUrl === "string" && model.baseUrl.includes("opencode.ai"))) {
131
+ const sessionId = ctx.sessionManager.getSessionId?.();
132
+ if (sessionId) {
133
+ return {
134
+ ...cached,
135
+ headers: {
136
+ ...(cached.headers ?? {}),
137
+ "x-opencode-session": sessionId,
138
+ "x-opencode-client": "pi",
139
+ },
140
+ };
141
+ }
142
+ }
88
143
  return cached;
89
144
  }
90
145
  debugLog(`${stage}.model_unavailable`, { reason: cached.reason });
@@ -121,7 +176,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
121
176
  if (runtime.consolidationInFlight) return;
122
177
 
123
178
  const entries = ctx.sessionManager.getBranch() as Entry[];
124
- if (!anyStageDue(entries, runtime)) return;
179
+ if (!anyStageDue(entries, runtime, realContextTokens(ctx))) return;
125
180
 
126
181
  const runId = `consolidation-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
127
182
  const consolidationCtx: ConsolidationCtx = {
@@ -130,6 +185,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
130
185
  ui: ctx.ui,
131
186
  model: ctx.model,
132
187
  modelRegistry: ctx.modelRegistry,
188
+ getContextUsage: ctx.getContextUsage,
133
189
  sessionManager: ctx.sessionManager,
134
190
  };
135
191
 
@@ -185,27 +241,79 @@ async function runObserverStage(
185
241
  resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
186
242
  ): Promise<StageOutcome> {
187
243
  const entries = ctx.sessionManager.getBranch() as Entry[];
188
- const tokens = rawTokensSinceObservationCoverage(entries);
244
+ const currentTokens = realContextTokens(ctx);
245
+ const real = currentTokens !== undefined ? realTokensSinceAnchor(entries, OM_OBSERVATIONS_RECORDED, currentTokens) : undefined;
246
+ const tokens = real !== undefined ? real : rawTokensSinceObservationCoverage(entries); // fallback: no usage baseline / basis change
189
247
  if (tokens < runtime.config.observeAfterTokens) return "continue";
190
248
 
249
+ const sessionMetadata = debugSessionMetadata(ctx);
250
+ const sessionIdentity = sessionMetadata.sessionId ?? sessionMetadata.sessionFile;
251
+ const coverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
252
+
253
+ // Deliberate-empty backoff (#23): an intentional "nothing to record" verdict
254
+ // must not re-fire the observer every turn over the same span. Retry only
255
+ // after another observeAfterTokens worth of new source tokens arrives, and
256
+ // drop the backoff as soon as coverage advances.
257
+ const backoff = runtime.observerEmptyBackoff;
258
+ if (backoff) {
259
+ if (
260
+ sessionIdentity !== backoff.sessionIdentity
261
+ || coverageId !== backoff.coverageId
262
+ || tokens >= backoff.tokensAtEmpty + runtime.config.observeAfterTokens
263
+ ) {
264
+ runtime.observerEmptyBackoff = undefined;
265
+ } else {
266
+ debugLog("observer.empty_backoff", { tokens, resumeAtTokens: backoff.tokensAtEmpty + runtime.config.observeAfterTokens });
267
+ return "continue";
268
+ }
269
+ }
270
+
271
+ // Resolve the model before building the chunk: the default chunk cap
272
+ // derives from the resolved model's context window.
273
+ const resolved = await resolveModel("observer");
274
+ if (!resolved) return "abort";
275
+
191
276
  const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
192
- const chunkEntries = sourceEntriesAfter(entries, lastCoverageIdx);
193
- const coversUpToId = chunkEntries.at(-1)?.id;
277
+ const backlogEntries = sourceEntriesAfter(entries, lastCoverageIdx);
278
+
279
+ // Budget the text that is actually sent to the observer, including source
280
+ // labels and rendered message content. Complete entries are kept intact.
281
+ // Only a first entry that cannot fit by itself is represented by a clearly
282
+ // marked head/tail excerpt; the original ledger entry remains untouched.
283
+ const contextWindow = (resolved.model as { contextWindow?: number }).contextWindow;
284
+ const maxChunkTokens = resolveObserverChunkMaxTokens(runtime.config, contextWindow);
285
+ const {
286
+ text: chunk,
287
+ sourceEntryIds,
288
+ estimatedTokens: chunkTokens,
289
+ truncatedSourceEntryIds,
290
+ } = serializeSourceAddressedBranchEntries(backlogEntries, { maxTokens: maxChunkTokens });
291
+ if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
292
+ const coversUpToId = sourceEntryIds.at(-1);
194
293
  if (!coversUpToId) return "continue";
195
294
 
196
- const { text: chunk, sourceEntryIds } = serializeSourceAddressedBranchEntries(chunkEntries);
197
- if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
295
+ if (sourceEntryIds.length < backlogEntries.length || truncatedSourceEntryIds.length > 0) {
296
+ debugLog("observer.chunk_capped", {
297
+ maxChunkTokens,
298
+ backlogEntries: backlogEntries.length,
299
+ backlogTokens: tokens,
300
+ chunkEntries: sourceEntryIds.length,
301
+ chunkTokens,
302
+ truncatedSourceEntryIds,
303
+ });
304
+ }
198
305
 
199
306
  const memory = fullProjection(entries);
200
307
  const priorReflections = memory.reflections.map(reflectionToSummaryLine);
201
308
  const priorObservations = memory.observations.map(observationToSummaryLine);
202
309
 
203
- if (ctx.hasUI) ctx.ui?.notify(
204
- `Observational memory: observer running on ~${tokens.toLocaleString()}-token chunk`,
310
+ if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
311
+ `Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk`,
205
312
  "info",
206
313
  );
207
314
  debugLog("observer.start", {
208
315
  tokens,
316
+ chunkTokens,
209
317
  coversUpToId,
210
318
  sourceEntryIds,
211
319
  sourceEntryCount: sourceEntryIds.length,
@@ -213,28 +321,43 @@ async function runObserverStage(
213
321
  priorObservations: priorObservations.length,
214
322
  });
215
323
 
216
- const resolved = await resolveModel("observer");
217
- if (!resolved) return "abort";
218
-
219
- const observations = await runObserver({
220
- model: resolved.model as any,
221
- apiKey: resolved.apiKey,
222
- headers: resolved.headers,
223
- priorReflections,
224
- priorObservations,
225
- chunk,
226
- allowedSourceEntryIds: sourceEntryIds,
227
- maxTurns: runtime.config.agentMaxTurns,
228
- thinkingLevel: runtime.config.model?.thinking ?? "low",
229
- });
324
+ let observations: Observation[] | undefined;
325
+ try {
326
+ observations = await runObserver({
327
+ model: resolved.model as any,
328
+ apiKey: resolved.apiKey,
329
+ headers: resolved.headers,
330
+ env: resolved.env,
331
+ priorReflections,
332
+ priorObservations,
333
+ chunk,
334
+ allowedSourceEntryIds: sourceEntryIds,
335
+ maxTurns: runtime.config.agentMaxTurns,
336
+ maxOutputTokens: runtime.config.agentMaxTokens,
337
+ thinkingLevel: runtime.config.model?.thinking ?? "low",
338
+ modelRegistry: ctx.modelRegistry,
339
+ });
340
+ } catch (error) {
341
+ if (error instanceof ObserverStreamError) {
342
+ // API/stream failure is not a clean empty (#32): surface it as a real
343
+ // failure instead of the "no observations" path. Coverage stays put.
344
+ runtime.recordConsolidationStageError(ctx, "observer", error);
345
+ return "abort";
346
+ }
347
+ throw error;
348
+ }
230
349
  if (!observations || observations.length === 0) {
350
+ // Deliberate empty: routine info, not a warning, and back off re-fires
351
+ // over the same span (#23).
231
352
  debugLog("observer.empty", { coversUpToId });
232
- if (ctx.hasUI) ctx.ui?.notify(
233
- "Observational memory: observer returned no observations",
234
- "warning",
353
+ runtime.observerEmptyBackoff = { sessionIdentity, coverageId, tokensAtEmpty: tokens };
354
+ if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
355
+ "Observational memory: observer found nothing new in this chunk (coverage unchanged; will retry later)",
356
+ "info",
235
357
  );
236
358
  return "continue";
237
359
  }
360
+ runtime.observerEmptyBackoff = undefined;
238
361
 
239
362
  const data = buildObservationsRecordedData(observations, coversUpToId);
240
363
  if (!data) return "continue";
@@ -245,7 +368,7 @@ async function runObserverStage(
245
368
  });
246
369
  appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
247
370
  debugLog("observer.appended", { count: observations.length, coversUpToId });
248
- if (ctx.hasUI) ctx.ui?.notify(
371
+ if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
249
372
  `Observational memory: ${observations.length} observation${observations.length === 1 ? "" : "s"} recorded`,
250
373
  "info",
251
374
  );
@@ -259,13 +382,15 @@ async function runReflectorStage(
259
382
  resolveModel: (stage: "reflector") => Promise<ResolvedModel | undefined>,
260
383
  ): Promise<ReflectorStageResult> {
261
384
  const entries = ctx.sessionManager.getBranch() as Entry[];
262
- const reflectionTokens = rawTokensSinceReflectionCoverage(entries);
385
+ const currentTokens = realContextTokens(ctx);
386
+ const real = currentTokens !== undefined ? realTokensSinceAnchor(entries, OM_REFLECTIONS_RECORDED, currentTokens) : undefined;
387
+ const reflectionTokens = real !== undefined ? real : rawTokensSinceReflectionCoverage(entries); // fallback: no usage baseline / basis change
263
388
  if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
264
389
 
265
390
  const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
266
391
  if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
267
392
 
268
- if (ctx.hasUI) ctx.ui?.notify(
393
+ if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
269
394
  `Observational memory: reflector running (~${reflectionTokens.toLocaleString()} tokens)`,
270
395
  "info",
271
396
  );
@@ -277,10 +402,13 @@ async function runReflectorStage(
277
402
  model: resolved.model as any,
278
403
  apiKey: resolved.apiKey,
279
404
  headers: resolved.headers,
405
+ env: resolved.env,
280
406
  reflections: folded.reflections,
281
407
  observations: folded.activeObservations,
282
408
  maxTurns: runtime.config.agentMaxTurns,
409
+ maxOutputTokens: runtime.config.agentMaxTokens,
283
410
  thinkingLevel: runtime.config.model?.thinking ?? "low",
411
+ modelRegistry: ctx.modelRegistry,
284
412
  });
285
413
  if (!reflections) return { outcome: "continue", sameRunReflections: [] };
286
414
 
@@ -337,7 +465,7 @@ async function runDropperStage(
337
465
  maxDropsAllowed: metrics.maxDropsAllowed,
338
466
  });
339
467
 
340
- if (ctx.hasUI) ctx.ui?.notify(
468
+ if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
341
469
  `Observational memory: dropper running after reflection — active observation pool ~${metrics.observationTokens.toLocaleString()} / ${metrics.targetTokens.toLocaleString()} target tokens (${Math.round(metrics.fullness * 100).toLocaleString()}%)`,
342
470
  "info",
343
471
  );
@@ -349,11 +477,14 @@ async function runDropperStage(
349
477
  model: resolved.model as any,
350
478
  apiKey: resolved.apiKey,
351
479
  headers: resolved.headers,
480
+ env: resolved.env,
352
481
  reflections: reflectionsForDropper,
353
482
  observations: folded.activeObservations,
354
483
  targetTokens: runtime.config.observationsPoolTargetTokens,
355
484
  maxTurns: runtime.config.agentMaxTurns,
485
+ maxOutputTokens: runtime.config.agentMaxTokens,
356
486
  thinkingLevel: runtime.config.model?.thinking ?? "low",
487
+ modelRegistry: ctx.modelRegistry,
357
488
  });
358
489
  const coversUpToId = earlierCoverageMarkerId(entries, observationCoverageId, sameRunReflectionCoverageId);
359
490
  const data = coversUpToId && droppedIds ? buildObservationsDroppedData(droppedIds, coversUpToId) : undefined;
package/src/runtime.ts CHANGED
@@ -1,13 +1,89 @@
1
1
  import { type Config, DEFAULTS, loadConfig } from "./config.js";
2
+ import { debugLog } from "./debug-log.js";
2
3
 
3
4
  export type ResolveResult =
4
- | { ok: true; model: unknown; apiKey: string; headers?: Record<string, string> }
5
+ | { ok: true; model: unknown; apiKey?: string; headers?: Record<string, string>; env?: Record<string, string>; baseUrl?: string }
5
6
  | { ok: false; reason: string };
6
7
 
8
+ /**
9
+ * Mirrors pi's own request-auth acceptance rule (`AgentSession._getRequiredRequestAuth`):
10
+ * resolved auth is usable when it carries an apiKey OR at least one header value.
11
+ * OAuth providers (kimi-coding, xai, openai-codex, anthropic OAuth, …) authenticate via
12
+ * `toAuth()` returning `{ headers: { Authorization: "Bearer …" } }` with no apiKey, and
13
+ * pi-ai providers accept a caller-supplied Authorization header in place of an apiKey.
14
+ *
15
+ * NOTE: a `false` result does NOT mean "unauthenticated" — see `resolveModel`. Providers
16
+ * that authenticate at request time (Amazon Bedrock SigV4 from AWS_PROFILE/SSO, Google
17
+ * Vertex ADC) legitimately expose neither an apiKey nor a header, because pi signs their
18
+ * requests itself.
19
+ */
20
+ function hasUsableAuth(auth: { apiKey?: unknown; headers?: unknown }): boolean {
21
+ if (typeof auth.apiKey === "string" && auth.apiKey.length > 0) return true;
22
+ return countUsableHeaders(auth.headers) > 0;
23
+ }
24
+
25
+ /** How many headers the auth payload carries at all (diagnostics only, never values). */
26
+ function countHeaders(headers: unknown): number {
27
+ return headers && typeof headers === "object" ? Object.keys(headers as Record<string, unknown>).length : 0;
28
+ }
29
+
30
+ /** How many headers carry a non-empty string value — the ones pi could actually send. */
31
+ function countUsableHeaders(headers: unknown): number {
32
+ if (!headers || typeof headers !== "object") return 0;
33
+ return Object.values(headers as Record<string, unknown>).filter(
34
+ (value) => typeof value === "string" && value.length > 0,
35
+ ).length;
36
+ }
37
+
38
+ /**
39
+ * How long to wait for the availability re-check in `recheckProviderCredential`, and how
40
+ * long before the same provider may be re-checked again.
41
+ *
42
+ * The re-check is network-free and measured at ~1ms on a warm Bedrock/SSO host, but
43
+ * `checkAuth` can block on a provider's own credential resolution, so it is bounded. The
44
+ * re-arm interval keeps an unauthenticated host from paying the cost on every
45
+ * consolidation while still recovering within a session when credentials are renewed out
46
+ * of band (`aws sso login` in another terminal, `gcloud auth application-default login`).
47
+ */
48
+ const AVAILABILITY_RECHECK_TIMEOUT_MS = 5_000;
49
+ const AVAILABILITY_RECHECK_REARM_MS = 60_000;
50
+
7
51
  type NotifyLevel = "warning" | "info" | "error";
8
52
  type Notify = (message: string, type?: NotifyLevel) => void;
9
53
  export type ConsolidationPhase = "observer" | "reflector" | "dropper";
10
54
 
55
+ /**
56
+ * Whether pi positively reports a working credential source for this model's provider.
57
+ *
58
+ * `ModelRegistry.hasConfiguredAuth(model)` is true when pi's availability check
59
+ * (`ModelRuntime.checkAuth`) resolved *something* for the provider — an API key, a
60
+ * stored credential, or an ambient source such as `AWS_PROFILE` / `AWS_ACCESS_KEY_ID`
61
+ * / gcloud ADC. Combined with `auth.ok === true` and an auth payload that carries
62
+ * nothing, that is the signature of a provider pi signs at request time:
63
+ *
64
+ * pi has a credential source, and deliberately hands the caller nothing to attach.
65
+ *
66
+ * Measured on a Bedrock/SSO host (pi 0.84.2), with `AWS_PROFILE` exported:
67
+ * checkAuth("amazon-bedrock") -> { source: "AWS_PROFILE", type: "api_key" }
68
+ * hasConfiguredAuth(model) -> true
69
+ * getApiKeyAndHeaders(model) -> { ok: true, apiKey: undefined, headers: undefined }
70
+ * `googleVertexProvider`'s ADC branch returns the same empty-auth resolution.
71
+ *
72
+ * The inverse case — `hasConfiguredAuth === false` with an empty auth payload — is a
73
+ * provider pi could not authenticate at all (no key, no ambient source). That must
74
+ * keep failing: it is the ordinary "not logged in" state, not ambient auth.
75
+ *
76
+ * Defensive: older pi versions and partial test doubles may not expose this, and an
77
+ * unknown answer must not be read as "authenticated".
78
+ */
79
+ function hasConfiguredProviderCredential(registry: unknown, model: unknown): boolean {
80
+ try {
81
+ return (registry as { hasConfiguredAuth?: (m: unknown) => unknown }).hasConfiguredAuth?.(model) === true;
82
+ } catch {
83
+ return false;
84
+ }
85
+ }
86
+
11
87
  export interface ResolveCtx {
12
88
  model: unknown;
13
89
  modelRegistry: any;
@@ -32,6 +108,14 @@ export class Runtime {
32
108
  lastObserverError: string | undefined;
33
109
  lastReflectorError: string | undefined;
34
110
  lastDropperError: string | undefined;
111
+ /** provider -> epoch ms of the last availability re-check (see `recheckProviderCredential`). */
112
+ availabilityRecheckedAt = new Map<string, number>();
113
+ /** Deliberate-empty backoff (#23): skip observer re-fires over the same span until enough new tokens arrive. */
114
+ observerEmptyBackoff: {
115
+ sessionIdentity: string | undefined;
116
+ coverageId: string | undefined;
117
+ tokensAtEmpty: number;
118
+ } | undefined;
35
119
 
36
120
  ensureConfig(cwd: string): void {
37
121
  if (this.configLoaded) return;
@@ -54,11 +138,149 @@ export class Runtime {
54
138
  }
55
139
  if (!model) return { ok: false, reason: "no model available (session has no model and no observational-memory model configured)" };
56
140
  const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
57
- if (!auth.ok || !auth.apiKey) {
58
- const provider = (model as { provider?: string }).provider ?? "unknown";
59
- return { ok: false, reason: `no API key for provider "${provider}"` };
141
+ const provider = (model as { provider?: string }).provider ?? "unknown";
142
+ const isOAuth = ctx.modelRegistry.isUsingOAuth?.(model) === true;
143
+ // `auth.ok === false` is the only unambiguous failure: pi returns it when a
144
+ // provider requires a request auth header and no credential resolved.
145
+ //
146
+ // `auth.ok === true` with neither apiKey nor headers, for a provider pi DOES
147
+ // report a credential source for, is not a failure — it is how pi describes a
148
+ // provider that authenticates at request time: Amazon Bedrock signing SigV4 from
149
+ // ambient AWS credentials (`bedrockAuth.resolve` returns `{ auth: {}, source:
150
+ // "AWS_PROFILE" }`), Google Vertex using ADC (same empty resolution). pi's own
151
+ // native streaming path forwards no apiKey either, and its pre-prompt gate is
152
+ // merely `hasConfiguredAuth(provider) || checkAuth(provider) !== undefined` —
153
+ // om's pre-flight check must not be stricter than pi's own. Treating it as "no
154
+ // auth" aborted consolidation before the model was ever called, disabling
155
+ // observational memory silently — no error, no cost, no latency — on such hosts.
156
+ //
157
+ // Three cases deliberately keep failing: OAuth providers, where an empty
158
+ // resolution means the credentials no longer resolve and the user must log in
159
+ // again; a credential that resolved to an empty *string* key, which is a
160
+ // misconfiguration rather than ambient auth; and a provider pi reports no
161
+ // credential source for at all, which is simply unauthenticated.
162
+ const usable = hasUsableAuth(auth);
163
+ const resolvedEmptyApiKey = typeof auth.apiKey === "string" && auth.apiKey.length === 0;
164
+ let providerCredentialConfigured = hasConfiguredProviderCredential(ctx.modelRegistry, model);
165
+ // pi's gate has TWO halves and never trusts the snapshot alone (agent-session.js):
166
+ //
167
+ // hasConfiguredAuth(provider) || (await checkAuth(provider)) !== undefined
168
+ //
169
+ // `hasConfiguredAuth` reads `snapshot.configuredProviders`, which is populated by an
170
+ // availability pass — and left untouched when that pass is skipped
171
+ // (`refreshOnCreate: false`), aborted, or FAILS (its catch records `availabilityError`
172
+ // and returns). A provider whose credential could not be checked at startup — an
173
+ // expired SSO token, say — is therefore absent from the snapshot for the rest of the
174
+ // session, even after the user renews it out of band. pi recovers on the next turn
175
+ // because its second half re-checks live; reading only the snapshot half would leave
176
+ // consolidation dead for the whole session, which is the same silent-failure class as
177
+ // the bug this gate was fixed for.
178
+ //
179
+ // The facade exposes no `checkAuth`, but `refresh({ providers })` performs the same
180
+ // live check and then updates the snapshot, so re-reading afterwards is equivalent.
181
+ // Only attempted when everything else already looks like the ambient shape, so an
182
+ // ordinary unauthenticated provider still fails on the first call.
183
+ if (auth.ok === true && !usable && !isOAuth && !resolvedEmptyApiKey && !providerCredentialConfigured) {
184
+ providerCredentialConfigured = await this.recheckProviderCredential(ctx.modelRegistry, model, provider);
60
185
  }
61
- return { ok: true, model, apiKey: auth.apiKey as string, headers: auth.headers as Record<string, string> | undefined };
186
+ const signsAtRequestTime =
187
+ auth.ok === true && !isOAuth && !resolvedEmptyApiKey && providerCredentialConfigured;
188
+ if (!auth.ok || (!usable && !signsAtRequestTime)) {
189
+ const reason = isOAuth
190
+ ? `authentication failed for provider "${provider}" — OAuth credentials may have expired; run '/login ${provider}' to re-authenticate`
191
+ : `no API key or auth headers for provider "${provider}"`;
192
+ // The reason string alone cannot tell `ok: false` from `ok: true` with nothing to
193
+ // carry, which is what made the ambient-credential outage un-diagnosable from the
194
+ // debug log. Record the decision inputs — booleans and counts only, never values.
195
+ debugLog("resolve.rejected", {
196
+ provider,
197
+ reason,
198
+ authOk: auth.ok === true,
199
+ hasApiKey: typeof auth.apiKey === "string" && auth.apiKey.length > 0,
200
+ resolvedEmptyApiKey,
201
+ headerCount: countHeaders(auth.headers),
202
+ usableHeaderCount: countUsableHeaders(auth.headers),
203
+ isOAuth,
204
+ providerCredentialConfigured,
205
+ signsAtRequestTime,
206
+ });
207
+ return { ok: false, reason };
208
+ }
209
+ if (!usable) {
210
+ debugLog("resolve.request_time_signing", { provider, providerCredentialConfigured });
211
+ }
212
+ // Match pi's request model: OAuth may route to an account-specific endpoint
213
+ // (e.g. Copilot Business). Do not mutate the shared session/registry model.
214
+ const requestModel = auth.baseUrl ? { ...(model as object), baseUrl: auth.baseUrl } : model;
215
+ return {
216
+ ok: true,
217
+ model: requestModel,
218
+ apiKey: auth.apiKey as string | undefined,
219
+ headers: auth.headers as Record<string, string> | undefined,
220
+ env: auth.env as Record<string, string> | undefined,
221
+ baseUrl: auth.baseUrl as string | undefined,
222
+ };
223
+ }
224
+
225
+ /**
226
+ * Re-check one provider's credential live, then re-read pi's snapshot.
227
+ *
228
+ * Implements the second half of pi's own auth gate for the only case that needs it: an
229
+ * otherwise-ambient-looking resolution whose provider is missing from a stale or never
230
+ * populated availability snapshot. Bounded and rate-limited; never throws.
231
+ */
232
+ private async recheckProviderCredential(registry: unknown, model: unknown, provider: string): Promise<boolean> {
233
+ const last = this.availabilityRecheckedAt.get(provider);
234
+ const now = Date.now();
235
+ if (last !== undefined && now - last < AVAILABILITY_RECHECK_REARM_MS) return false;
236
+ this.availabilityRecheckedAt.set(provider, now);
237
+
238
+ const refresh = (registry as { refresh?: (options?: unknown) => Promise<unknown> }).refresh;
239
+ if (typeof refresh !== "function") {
240
+ debugLog("resolve.availability_recheck", { provider, refreshed: false, reason: "registry exposes no refresh()" });
241
+ return false;
242
+ }
243
+
244
+ const controller = new AbortController();
245
+ const timer = setTimeout(() => controller.abort(), AVAILABILITY_RECHECK_TIMEOUT_MS);
246
+ let refreshError: string | undefined;
247
+ let timedOut = false;
248
+ try {
249
+ // allowNetwork:false — a credential re-check must not wait on a model-catalog fetch.
250
+ // providers:[provider] — scope the work, and the snapshot writes, to the one provider.
251
+ //
252
+ // Both are honoured from pi 0.84; on pi 0.81 the facade is `refresh()` with no
253
+ // parameters, delegating to `runtime.reloadConfig()`, which reloads models.json and
254
+ // then runs a FULL, network-permitted availability pass. Passing the options is
255
+ // harmless there, but the work is wider and slower — hence the race below rather
256
+ // than relying on the abort signal, which that version never sees.
257
+ await Promise.race([
258
+ refresh.call(registry, { allowNetwork: false, providers: [provider], signal: controller.signal }),
259
+ new Promise<void>((resolve) => {
260
+ controller.signal.addEventListener("abort", () => {
261
+ timedOut = true;
262
+ resolve();
263
+ });
264
+ }),
265
+ ]);
266
+ } catch (error) {
267
+ refreshError = error instanceof Error ? error.message : String(error);
268
+ } finally {
269
+ clearTimeout(timer);
270
+ }
271
+
272
+ // Re-read even when the refresh reported an error or timed out: a scoped pass can
273
+ // update the snapshot for this provider and still fail elsewhere.
274
+ const recovered = hasConfiguredProviderCredential(registry, model);
275
+ debugLog("resolve.availability_recheck", {
276
+ provider,
277
+ refreshed: refreshError === undefined && !timedOut,
278
+ recovered,
279
+ elapsedMs: Date.now() - now,
280
+ timedOut,
281
+ ...(refreshError === undefined ? {} : { refreshError }),
282
+ });
283
+ return recovered;
62
284
  }
63
285
 
64
286
  launchConsolidationTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> {