pi-mega-compact 0.8.24 → 0.8.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/extensions/mega-compact-s38.test.js +263 -14
- package/dist/extensions/mega-compact.js +15 -0
- package/dist/extensions/mega-config.js +3 -0
- package/dist/extensions/mega-events/agent-handlers.js +211 -26
- package/dist/extensions/mega-events/context-handler.js +45 -7
- package/dist/extensions/mega-events/error-classifier.js +125 -18
- package/dist/extensions/mega-pipeline/compact.js +24 -13
- package/dist/extensions/mega-pipeline/recall.js +31 -2
- package/dist/extensions/mega-runtime/dashboard-snapshot.js +4 -0
- package/dist/extensions/mega-runtime/runtime-snapshot.js +4 -0
- package/dist/extensions/mega-runtime/runtime.js +58 -5
- package/dist/src/boundary.js +79 -43
- package/dist/src/boundary.test.js +119 -2
- package/dist/src/canary.js +10 -0
- package/dist/src/config/dedup.js +14 -0
- package/dist/src/config.js +3 -1
- package/dist/src/dedup/raptor/buildHistory.js +164 -0
- package/dist/src/dedup/raptor/buildHistory.test.js +292 -0
- package/dist/src/dedup/raptor/index.js +38 -0
- package/dist/src/dedup/raptor/multilevel-serve.test.js +229 -0
- package/dist/src/dedup/raptor/multilevel.js +17 -5
- package/dist/src/dedup/raptor/multilevel.test.js +36 -1
- package/dist/src/dedup/raptor/raptor.test.js +43 -0
- package/dist/src/dedup/raptor/retrieval.js +14 -2
- package/dist/src/dedup/raptor/retrieval.test.js +95 -0
- package/dist/src/dedup/raptor/serve-gate.test.js +298 -0
- package/dist/src/dedup/raptor/summarizer.js +1 -0
- package/dist/src/dedup/raptor/tree.js +16 -2
- package/dist/src/engine.js +18 -2
- package/dist/src/httpEmbedder.js +96 -6
- package/dist/src/httpEmbedder.test.js +277 -0
- package/dist/src/mechanical-fix.test.js +65 -0
- package/dist/src/raptor-inject-summaries.test.js +162 -0
- package/dist/src/recall.js +153 -24
- package/dist/src/recall.test.js +179 -4
- package/dist/src/store/sqlite/dedup-mirror.js +32 -15
- package/dist/src/store/sqlite/maintenance.js +2 -2
- package/dist/src/store/sqlite/mechanical-fix.test.js +146 -0
- package/dist/src/store/sqlite/memories.js +5 -5
- package/dist/src/store/sqlite/meta.js +1 -1
- package/dist/src/store/sqlite/raptor.js +56 -17
- package/dist/src/store/sqlite/raptor.test.js +106 -0
- package/dist/src/store/sqlite/schema.js +90 -1
- package/dist/src/store/sqlite/session-state.js +9 -3
- package/dist/src/store/sqlite/stats.js +9 -5
- package/dist/src/store/sqlite/turns.js +179 -0
- package/dist/src/store/sqlite/turns.test.js +183 -0
- package/dist/src/store/sqlite/utils.js +15 -4
- package/dist/src/store/sqlite.js +1 -0
- package/dist/src/store.js +2 -2
- package/dist/src/vector-search-cache.test.js +157 -0
- package/dist/src/vector-search.js +107 -15
- package/dist/src/vectorStore.js +36 -8
- package/extensions/mega-compact-s38.test.ts +259 -14
- package/extensions/mega-compact.ts +15 -0
- package/extensions/mega-config.ts +18 -0
- package/extensions/mega-dashboard.ts +10 -1
- package/extensions/mega-events/agent-handlers.ts +211 -26
- package/extensions/mega-events/context-handler.ts +43 -7
- package/extensions/mega-events/error-classifier.ts +125 -17
- package/extensions/mega-pipeline/compact.ts +28 -16
- package/extensions/mega-pipeline/recall.ts +34 -2
- package/extensions/mega-runtime/dashboard-snapshot.ts +8 -0
- package/extensions/mega-runtime/helpers.ts +25 -1
- package/extensions/mega-runtime/runtime-snapshot.ts +4 -0
- package/extensions/mega-runtime/runtime.ts +69 -23
- package/package.json +1 -1
- package/src/boundary.test.ts +128 -2
- package/src/boundary.ts +75 -39
- package/src/canary.ts +10 -0
- package/src/config/dedup.ts +25 -0
- package/src/config.ts +3 -1
- package/src/dedup/raptor/buildHistory.test.ts +353 -0
- package/src/dedup/raptor/buildHistory.ts +259 -0
- package/src/dedup/raptor/index.ts +38 -0
- package/src/dedup/raptor/multilevel-serve.test.ts +273 -0
- package/src/dedup/raptor/multilevel.test.ts +47 -0
- package/src/dedup/raptor/multilevel.ts +18 -8
- package/src/dedup/raptor/raptor.test.ts +59 -0
- package/src/dedup/raptor/retrieval.test.ts +118 -0
- package/src/dedup/raptor/retrieval.ts +14 -2
- package/src/dedup/raptor/serve-gate.test.ts +348 -0
- package/src/dedup/raptor/summarizer.ts +1 -0
- package/src/dedup/raptor/tree.ts +17 -2
- package/src/engine.ts +32 -3
- package/src/httpEmbedder.test.ts +286 -0
- package/src/httpEmbedder.ts +98 -8
- package/src/mechanical-fix.test.ts +70 -0
- package/src/raptor-inject-summaries.test.ts +228 -0
- package/src/recall.test.ts +220 -4
- package/src/recall.ts +462 -265
- package/src/store/sqlite/dedup-mirror.ts +35 -18
- package/src/store/sqlite/maintenance.ts +2 -2
- package/src/store/sqlite/mechanical-fix.test.ts +162 -0
- package/src/store/sqlite/memories.ts +5 -5
- package/src/store/sqlite/meta.ts +1 -1
- package/src/store/sqlite/raptor.test.ts +139 -0
- package/src/store/sqlite/raptor.ts +135 -81
- package/src/store/sqlite/schema.ts +90 -1
- package/src/store/sqlite/session-state.ts +9 -3
- package/src/store/sqlite/stats.ts +10 -8
- package/src/store/sqlite/turns.test.ts +218 -0
- package/src/store/sqlite/turns.ts +302 -0
- package/src/store/sqlite/utils.ts +14 -4
- package/src/store/sqlite.ts +1 -0
- package/src/store.ts +9 -2
- package/src/vector-search-cache.test.ts +190 -0
- package/src/vector-search.ts +273 -156
- package/src/vectorStore.ts +443 -382
- package/extensions/mega-runtime/reset-runtime.ts +0 -80
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { piCompactWouldNoop, runMemoryReview, } from "../mega-pipeline.js";
|
|
2
2
|
import { memoryReviewCadence, } from "../mega-config.js";
|
|
3
|
-
import { recordScore } from "../../src/store/sqlite.js";
|
|
3
|
+
import { recordScore, recordTurn } from "../../src/store/sqlite.js";
|
|
4
4
|
import { evaluateAndUnlockAchievements } from "../../src/store/sqlite/game-achievements.js";
|
|
5
|
+
import { ensureConversationId } from "../../src/store/sqlite/turns.js";
|
|
5
6
|
import { isMegaCache } from "../../src/game/scoring.js";
|
|
6
7
|
import { resolveRepoRoot } from "../mega-config.js";
|
|
7
|
-
import { classifyError } from "./error-classifier.js";
|
|
8
|
+
import { classifyError, errorRetryBackoffMs, extractErrorSignature } from "./error-classifier.js";
|
|
8
9
|
import { safeSendUserMessage } from "./send-safe.js";
|
|
9
10
|
import { vectorStats } from "../../src/vectorStore.js";
|
|
10
11
|
/** Register agent/turn tracking event handlers. */
|
|
@@ -203,12 +204,41 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
203
204
|
runtime.currentTurn = event.turnIndex;
|
|
204
205
|
runtime.rt.lengthStopPending = false; // S28: re-arm defensively each user turn
|
|
205
206
|
runtime.rt.errorRetryCount = 0; // S38: reset error-retry counter each user turn
|
|
207
|
+
// R4 (turn_end hygiene): a genuine new user prompt consumes any queued
|
|
208
|
+
// retry nudge (deliverAs:'followUp') — pi dispatches it as the prompt for
|
|
209
|
+
// this turn. Clearing retryNudgePending re-arms the dedup gate so the
|
|
210
|
+
// next error turn can fire a fresh nudge (subject to backoff).
|
|
211
|
+
// NOTE: the poisoned-repeat tracker (lastErrorText / errorTextRepeatCount)
|
|
212
|
+
// is NOT reset here — a retry turn consuming the queued nudge is still the
|
|
213
|
+
// same error sequence. The tracker resets on a SUCCESSFUL turn (null) or
|
|
214
|
+
// when a different error text appears, not on the turn boundary.
|
|
215
|
+
runtime.rt.retryNudgePending = false;
|
|
206
216
|
runtime.dashboard.event("turn_start", { turnIndex: event.turnIndex });
|
|
207
217
|
runtime.snapshot(ctx);
|
|
208
218
|
});
|
|
209
219
|
pi.on("turn_end", async (event, ctx) => {
|
|
210
220
|
runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
|
|
211
221
|
runtime.snapshot(ctx);
|
|
222
|
+
// S43 (per-turn tracking): record one turn row with the cached metrics so
|
|
223
|
+
// the turn layer is queryable + forkable. Best-effort + non-fatal: a write
|
|
224
|
+
// failure never breaks the agent loop.
|
|
225
|
+
try {
|
|
226
|
+
const convId = ensureConversationId(runtime.rt.sessionId, runtime.currentStateDir);
|
|
227
|
+
recordTurn({
|
|
228
|
+
conversationId: convId,
|
|
229
|
+
sessionId: runtime.rt.sessionId,
|
|
230
|
+
turnIndex: event.turnIndex,
|
|
231
|
+
endedAt: Date.now(),
|
|
232
|
+
startedAt: undefined,
|
|
233
|
+
ctxTokens: runtime.lastCtxTokens ?? undefined,
|
|
234
|
+
ctxPercent: runtime.lastCtxPercent ?? undefined,
|
|
235
|
+
pressureBand: runtime.pressureBand ?? undefined,
|
|
236
|
+
modelId: runtime.currentModel?.modelId ?? undefined,
|
|
237
|
+
}, runtime.currentStateDir);
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
/* non-fatal: per-turn tracking never breaks the agent loop */
|
|
241
|
+
}
|
|
212
242
|
// S33: game-mode scoring — record turns + cache metrics per repo, and arm
|
|
213
243
|
// the MEGA CACHE flare (oopsie gag) when the real dedup hit rate exceeds
|
|
214
244
|
// 100%. Gated behind game_mode_on (no scoring when off). Best-effort +
|
|
@@ -282,9 +312,11 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
282
312
|
}
|
|
283
313
|
// S38: broader error-retry safety net. S28 only catches stopReason==='length';
|
|
284
314
|
// this catches ALL other error types (provider failure, network timeout, 5xx,
|
|
285
|
-
// 429, auth, compaction-noop) that surface at turn_end.
|
|
286
|
-
// try/catch so a classifier/retry failure never breaks
|
|
287
|
-
// PREVENT-PI-003: retry nudge fires via pi.sendUserMessage
|
|
315
|
+
// 429, auth, compaction-noop, poisoned-context) that surface at turn_end.
|
|
316
|
+
// Non-fatal: wrapped in try/catch so a classifier/retry failure never breaks
|
|
317
|
+
// the agent loop. PREVENT-PI-003: retry nudge fires via pi.sendUserMessage
|
|
318
|
+
// (user-role). R1-R4 (retry redesign) bound the loop: in-flight dedup,
|
|
319
|
+
// gating backoff, session-global cap, poisoned-context detection.
|
|
288
320
|
try {
|
|
289
321
|
// (1) S28 owns length — skip the classifier entirely for it.
|
|
290
322
|
const sr = event.message?.stopReason;
|
|
@@ -293,18 +325,52 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
293
325
|
}
|
|
294
326
|
else {
|
|
295
327
|
const category = classifyError(event.message);
|
|
296
|
-
|
|
328
|
+
// R3: stateful poisoned-context signal — repeated identical error text
|
|
329
|
+
// across consecutive turns. The classifier is stateless, so this
|
|
330
|
+
// upgrade happens here. A 'transient' (or 'poisoned-context') turn
|
|
331
|
+
// whose normalized error signature matches the previous one bumps
|
|
332
|
+
// errorTextRepeatCount; once it crosses the threshold, the category
|
|
333
|
+
// is upgraded to 'poisoned-context' (the request is deterministic).
|
|
334
|
+
// Conservative: only upgrade, never downgrade, and only for transient
|
|
335
|
+
// (a 'permanent' auth error repeating is still permanent).
|
|
336
|
+
let effectiveCategory = category;
|
|
337
|
+
const errSig = extractErrorSignature(event.message);
|
|
338
|
+
if (category === 'transient' || category === 'poisoned-context') {
|
|
339
|
+
if (errSig) {
|
|
340
|
+
if (runtime.rt.lastErrorText === errSig) {
|
|
341
|
+
runtime.rt.errorTextRepeatCount++;
|
|
342
|
+
}
|
|
343
|
+
else {
|
|
344
|
+
runtime.rt.lastErrorText = errSig;
|
|
345
|
+
runtime.rt.errorTextRepeatCount = 1;
|
|
346
|
+
}
|
|
347
|
+
if (runtime.rt.errorTextRepeatCount >= config.poisonedContextRepeatThreshold &&
|
|
348
|
+
category === 'transient') {
|
|
349
|
+
effectiveCategory = 'poisoned-context';
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
else {
|
|
354
|
+
// Non-transient (success, cancelled, compaction-noop, context-overflow,
|
|
355
|
+
// permanent): a fresh non-identical signal clears the repeat tracker.
|
|
356
|
+
runtime.rt.lastErrorText = undefined;
|
|
357
|
+
runtime.rt.errorTextRepeatCount = 0;
|
|
358
|
+
}
|
|
359
|
+
if (effectiveCategory === null) {
|
|
297
360
|
// (3) success / normal flow / unknown-but-non-retryable — reset.
|
|
298
361
|
runtime.rt.errorRetryCount = 0;
|
|
299
362
|
runtime.rt.consecutiveErrors = 0; // S38.6: circuit-breaker reset on success
|
|
363
|
+
// R4: a successful assistant turn consumes any queued nudge.
|
|
364
|
+
runtime.rt.retryNudgePending = false;
|
|
300
365
|
}
|
|
301
|
-
else if (
|
|
366
|
+
else if (effectiveCategory === 'compaction-noop') {
|
|
302
367
|
// (4) pi race / manual compact catch — NOT retryable. The compaction
|
|
303
368
|
// already succeeded via pi's native path; retrying would race again
|
|
304
369
|
// (FAIL-2026071701). Log a diagnostic, reset the counter, and surface
|
|
305
370
|
// the original error WITHOUT firing a retry nudge.
|
|
306
371
|
runtime.rt.errorRetryCount = 0;
|
|
307
372
|
runtime.rt.consecutiveErrors = 0; // S38.6: circuit-breaker reset
|
|
373
|
+
runtime.rt.retryNudgePending = false; // R4: terminal for this burst
|
|
308
374
|
runtime.dashboard.event('compaction_noop_diagnostic', {
|
|
309
375
|
turnIndex: event.turnIndex,
|
|
310
376
|
sessionId: runtime.rt.sessionId,
|
|
@@ -314,13 +380,14 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
314
380
|
turnIndex: event.turnIndex,
|
|
315
381
|
});
|
|
316
382
|
}
|
|
317
|
-
else if (
|
|
383
|
+
else if (effectiveCategory === 'cancelled') {
|
|
318
384
|
// (4c) User ESC / Ctrl-C — stopReason === 'aborted'. NOT retryable:
|
|
319
385
|
// nudging would restart a task the user explicitly stopped.
|
|
320
386
|
// Reset both counters (a cancel is not an error for circuit-breaker
|
|
321
387
|
// purposes). Emit a diagnostic so the dashboard shows it.
|
|
322
388
|
runtime.rt.errorRetryCount = 0;
|
|
323
389
|
runtime.rt.consecutiveErrors = 0;
|
|
390
|
+
runtime.rt.retryNudgePending = false; // R4: terminal for this burst
|
|
324
391
|
runtime.dashboard.event('error_retry_cancelled', {
|
|
325
392
|
turnIndex: event.turnIndex,
|
|
326
393
|
sessionId: runtime.rt.sessionId,
|
|
@@ -330,7 +397,7 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
330
397
|
turnIndex: event.turnIndex,
|
|
331
398
|
});
|
|
332
399
|
}
|
|
333
|
-
else if (
|
|
400
|
+
else if (effectiveCategory === 'context-overflow') {
|
|
334
401
|
// (4b) context-window overflow 400 ("too long... even after compaction").
|
|
335
402
|
// NOT a blind retry: re-submitting the same oversized prompt would just
|
|
336
403
|
// re-400 and busy-loop. Reset the counters (this turn is terminal, not
|
|
@@ -343,6 +410,7 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
343
410
|
// separate nudge here.
|
|
344
411
|
runtime.rt.errorRetryCount = 0;
|
|
345
412
|
runtime.rt.consecutiveErrors = 0;
|
|
413
|
+
runtime.rt.retryNudgePending = false; // R4: terminal for this burst
|
|
346
414
|
runtime.dashboard.event('context_overflow', {
|
|
347
415
|
turnIndex: event.turnIndex,
|
|
348
416
|
sessionId: runtime.rt.sessionId,
|
|
@@ -377,13 +445,78 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
377
445
|
}
|
|
378
446
|
}
|
|
379
447
|
}
|
|
448
|
+
else if (effectiveCategory === 'poisoned-context') {
|
|
449
|
+
// R3: poisoned context — a DETERMINISTIC request-rejection that
|
|
450
|
+
// retrying cannot fix (0-token 'error', generic "request failed",
|
|
451
|
+
// 400 non-overflow, or repeated identical error text). NO blind
|
|
452
|
+
// retry nudge — re-submitting the same poisoned prompt re-triggers
|
|
453
|
+
// the same rejection (the 2026-07-28 incident: ~60-message spam).
|
|
454
|
+
// Instead: (a) dashboard event + warn log naming the suspected
|
|
455
|
+
// poison; (b) ONE user-role advise message (/clear or /new) per
|
|
456
|
+
// session, throttled; (c) if config.auto, ONE guarded compact per
|
|
457
|
+
// error signature (same race-guarded deferred path as context-
|
|
458
|
+
// overflow) — only if not already attempted for this signature.
|
|
459
|
+
runtime.rt.errorRetryCount = 0; // poisoned is terminal for this burst
|
|
460
|
+
runtime.rt.consecutiveErrors++; // still counts toward the circuit breaker
|
|
461
|
+
runtime.rt.poisonedCount++; // R7: dashboard counter
|
|
462
|
+
const sig = errSig || 'unknown';
|
|
463
|
+
// (a) dashboard + log
|
|
464
|
+
runtime.dashboard.event('poisoned_context', {
|
|
465
|
+
signature: sig,
|
|
466
|
+
repeatCount: runtime.rt.errorTextRepeatCount,
|
|
467
|
+
turnIndex: event.turnIndex,
|
|
468
|
+
sessionId: runtime.rt.sessionId,
|
|
469
|
+
});
|
|
470
|
+
runtime.logger.warn('poisoned-context', {
|
|
471
|
+
sessionId: runtime.rt.sessionId,
|
|
472
|
+
turnIndex: event.turnIndex,
|
|
473
|
+
signature: sig,
|
|
474
|
+
repeatCount: runtime.rt.errorTextRepeatCount,
|
|
475
|
+
});
|
|
476
|
+
// (b) one-per-session advise message (throttled by poisonedAdviseSent)
|
|
477
|
+
if (!runtime.rt.poisonedAdviseSent) {
|
|
478
|
+
runtime.rt.poisonedAdviseSent = true;
|
|
479
|
+
// PREVENT-PI-003: user-role sendUserMessage only.
|
|
480
|
+
await safeSendUserMessage(pi, '[mega-compact] this session\'s context may be poisoned (the provider is rejecting every request). Run /clear or /new to start a fresh context.');
|
|
481
|
+
}
|
|
482
|
+
// (c) one guarded compact per error signature (attempt to remove
|
|
483
|
+
// the poisoned region). Race-guarded + deferred, mirroring the
|
|
484
|
+
// context-overflow path. Only if not already attempted for this
|
|
485
|
+
// signature (poisonedCompactSignatures).
|
|
486
|
+
if (config.auto && !runtime.rt.poisonedCompactSignatures.has(sig)) {
|
|
487
|
+
runtime.rt.poisonedCompactSignatures.add(sig);
|
|
488
|
+
const nowP = Date.now();
|
|
489
|
+
const cooldownMsP = config.raceGuardStrict ? 30_000 : 10_000;
|
|
490
|
+
const sinceCompactP = nowP - (runtime.rt.lastNativeCompactAt ?? 0);
|
|
491
|
+
if (sinceCompactP >= cooldownMsP && !piCompactWouldNoop(ctx)) {
|
|
492
|
+
runtime.debounceUntil = nowP + 0;
|
|
493
|
+
const stampP = runtime.rt.lastNativeCompactAt;
|
|
494
|
+
const liveSidP = runtime.rt.sessionId;
|
|
495
|
+
setTimeout(() => {
|
|
496
|
+
try {
|
|
497
|
+
if (runtime.rt.sessionId !== liveSidP)
|
|
498
|
+
return; // session reset
|
|
499
|
+
const sinceP = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
|
|
500
|
+
if (runtime.rt.lastNativeCompactAt !== stampP && sinceP < cooldownMsP)
|
|
501
|
+
return;
|
|
502
|
+
if (piCompactWouldNoop(ctx))
|
|
503
|
+
return;
|
|
504
|
+
ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated. One guarded compact after a poisoned-context detection.
|
|
505
|
+
}
|
|
506
|
+
catch {
|
|
507
|
+
/* non-fatal */
|
|
508
|
+
}
|
|
509
|
+
}, 500);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
}
|
|
380
513
|
else {
|
|
381
514
|
// (5) transient or permanent — retry with exponential backoff.
|
|
382
515
|
// S38.7: hard-stop switch — bypass ALL retry logic when set.
|
|
383
516
|
if (config.errorRetryHardStop) {
|
|
384
517
|
runtime.rt.errorRetryCount = 0;
|
|
385
518
|
runtime.dashboard.event('error_retry_disabled', {
|
|
386
|
-
category,
|
|
519
|
+
category: effectiveCategory,
|
|
387
520
|
turnIndex: event.turnIndex,
|
|
388
521
|
reason: 'hard-stop',
|
|
389
522
|
});
|
|
@@ -404,7 +537,7 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
404
537
|
});
|
|
405
538
|
return; // early exit — circuit breaker tripped
|
|
406
539
|
}
|
|
407
|
-
const max =
|
|
540
|
+
const max = effectiveCategory === 'transient'
|
|
408
541
|
? config.autoRetryTransientMax
|
|
409
542
|
: config.autoRetryPermanentMax;
|
|
410
543
|
// max === 0 disables the category entirely (revert to S28-only).
|
|
@@ -416,40 +549,92 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
416
549
|
if (runtime.rt.errorRetryCount > max) {
|
|
417
550
|
// Exhausted — surface the error, reset for the next burst.
|
|
418
551
|
runtime.dashboard.event('error_retry_exhausted', {
|
|
419
|
-
category,
|
|
552
|
+
category: effectiveCategory,
|
|
420
553
|
count: runtime.rt.errorRetryCount,
|
|
421
554
|
max,
|
|
422
555
|
turnIndex: event.turnIndex,
|
|
423
556
|
});
|
|
424
557
|
runtime.logger.info('error-retry-exhausted', {
|
|
425
558
|
sessionId: runtime.rt.sessionId,
|
|
426
|
-
category,
|
|
559
|
+
category: effectiveCategory,
|
|
427
560
|
count: runtime.rt.errorRetryCount,
|
|
428
561
|
max,
|
|
429
562
|
});
|
|
430
563
|
runtime.rt.errorRetryCount = 0;
|
|
431
564
|
}
|
|
432
565
|
else {
|
|
433
|
-
//
|
|
434
|
-
//
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
566
|
+
// R2: session-global cap — total S38 nudges per session across
|
|
567
|
+
// ALL bursts. Independent of the per-burst max and the circuit
|
|
568
|
+
// breaker. Hitting it is terminal for the session: log +
|
|
569
|
+
// dashboard event, stop nudging. `0` disables (reverts to
|
|
570
|
+
// per-burst + circuit-breaker only).
|
|
571
|
+
if (config.errorRetrySessionMax > 0 &&
|
|
572
|
+
runtime.rt.errorRetrySessionCount >= config.errorRetrySessionMax) {
|
|
573
|
+
runtime.dashboard.event('error_retry_session_exhausted', {
|
|
574
|
+
count: runtime.rt.errorRetrySessionCount,
|
|
575
|
+
max: config.errorRetrySessionMax,
|
|
576
|
+
category: effectiveCategory,
|
|
577
|
+
turnIndex: event.turnIndex,
|
|
578
|
+
});
|
|
579
|
+
runtime.logger.warn('error-retry-session-exhausted', {
|
|
580
|
+
sessionId: runtime.rt.sessionId,
|
|
581
|
+
count: runtime.rt.errorRetrySessionCount,
|
|
582
|
+
max: config.errorRetrySessionMax,
|
|
583
|
+
category: effectiveCategory,
|
|
584
|
+
});
|
|
585
|
+
runtime.rt.errorRetryCount = 0;
|
|
586
|
+
return; // terminal for the session — no nudge
|
|
587
|
+
}
|
|
588
|
+
// R1: in-flight nudge dedup — a nudge queued via
|
|
589
|
+
// deliverAs:'followUp' must not be re-sent until it has been
|
|
590
|
+
// consumed by an actual new agent turn (turn_start resets
|
|
591
|
+
// retryNudgePending). Without this, a fast-erroring provider +
|
|
592
|
+
// a per-turn nudge → N nudges queue up and pi dispatches N
|
|
593
|
+
// retry turns, each re-submitting the same failing prompt
|
|
594
|
+
// (the 2026-07-28 incident). errorRetryCount still advances
|
|
595
|
+
// (this IS an error turn), so the per-burst max + circuit
|
|
596
|
+
// breaker still bound the burst.
|
|
597
|
+
if (runtime.rt.retryNudgePending) {
|
|
598
|
+
runtime.dashboard.event('error_retry_dedup_skip', {
|
|
599
|
+
category: effectiveCategory,
|
|
600
|
+
count: runtime.rt.errorRetryCount,
|
|
601
|
+
max,
|
|
602
|
+
turnIndex: event.turnIndex,
|
|
603
|
+
});
|
|
604
|
+
return; // pending nudge not yet consumed — skip
|
|
605
|
+
}
|
|
606
|
+
// R1: gating backoff — errorRetryUntil is now GATING
|
|
607
|
+
// (previously documented as non-gating). A nudge cannot fire
|
|
608
|
+
// before the previous backoff elapses. This paces retries
|
|
609
|
+
// (5s/10s/20s/30s by default) so a fast-erroring provider
|
|
610
|
+
// doesn't slam N nudges in <1s.
|
|
611
|
+
const now = Date.now();
|
|
612
|
+
if (now < runtime.rt.errorRetryUntil) {
|
|
613
|
+
runtime.dashboard.event('error_retry_backoff_skip', {
|
|
614
|
+
category: effectiveCategory,
|
|
615
|
+
count: runtime.rt.errorRetryCount,
|
|
616
|
+
max,
|
|
617
|
+
turnIndex: event.turnIndex,
|
|
618
|
+
});
|
|
619
|
+
return; // backoff not elapsed — skip
|
|
620
|
+
}
|
|
621
|
+
// Fire the retry nudge. Set pending (R1 dedup) + backoff
|
|
622
|
+
// (R1 pacing) + session count (R2 cap) BEFORE the await so a
|
|
623
|
+
// re-entrant turn_end during the send can't double-fire.
|
|
624
|
+
runtime.rt.retryNudgePending = true;
|
|
625
|
+
runtime.rt.lastErrorRetryAt = now;
|
|
626
|
+
runtime.rt.errorRetryUntil =
|
|
627
|
+
now + errorRetryBackoffMs(runtime.rt.errorRetryCount, config.errorRetryBackoffMs);
|
|
628
|
+
runtime.rt.errorRetrySessionCount++;
|
|
444
629
|
runtime.dashboard.event('error_retry', {
|
|
445
|
-
category,
|
|
630
|
+
category: effectiveCategory,
|
|
446
631
|
count: runtime.rt.errorRetryCount,
|
|
447
632
|
max,
|
|
448
633
|
turnIndex: event.turnIndex,
|
|
449
634
|
});
|
|
450
635
|
runtime.logger.info('error-retry', {
|
|
451
636
|
sessionId: runtime.rt.sessionId,
|
|
452
|
-
category,
|
|
637
|
+
category: effectiveCategory,
|
|
453
638
|
count: runtime.rt.errorRetryCount,
|
|
454
639
|
max,
|
|
455
640
|
});
|
|
@@ -5,11 +5,44 @@ import { estimateSessionTokens } from "../../src/tokens.js";
|
|
|
5
5
|
import { runCompact, piCompactWouldNoop } from "../mega-pipeline.js";
|
|
6
6
|
import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
|
|
7
7
|
import { pressureFromPct, pressureRatio, } from "../mega-config.js";
|
|
8
|
-
import {
|
|
8
|
+
import { computeContentDigest } from "../../src/dedup/digest.js";
|
|
9
|
+
/**
|
|
10
|
+
* Recursively canonicalize a value for deterministic JSON serialization:
|
|
11
|
+
* - Objects: keys sorted alphabetically, values recursively canonicalized
|
|
12
|
+
* - Arrays: elements recursively canonicalized (array ORDER is preserved)
|
|
13
|
+
* - Primitives: returned as-is
|
|
14
|
+
*
|
|
15
|
+
* F5 fix: shallow-sorted JSON.stringify(content, Object.keys(content).sort())
|
|
16
|
+
* omitted nested keys not in the top-level keys array and only sorted one level.
|
|
17
|
+
* This recursively sorts every object at every depth so semantically-equal
|
|
18
|
+
* differently-ordered content hashes identically.
|
|
19
|
+
*/
|
|
20
|
+
function canonicalize(value) {
|
|
21
|
+
if (value === null || value === undefined)
|
|
22
|
+
return value;
|
|
23
|
+
if (typeof value !== "object")
|
|
24
|
+
return value;
|
|
25
|
+
if (Array.isArray(value))
|
|
26
|
+
return value.map(canonicalize);
|
|
27
|
+
// Plain object: sort keys, recursively canonicalize values.
|
|
28
|
+
const sorted = Object.keys(value).sort();
|
|
29
|
+
const out = {};
|
|
30
|
+
for (const k of sorted) {
|
|
31
|
+
out[k] = canonicalize(value[k]);
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
9
35
|
/**
|
|
10
36
|
* Convert a pi AgentMessage to a RawTranscriptRow for the DB mirror.
|
|
11
|
-
* content_bytes is canonical JSON (sorted keys) for deterministic
|
|
12
|
-
*
|
|
37
|
+
* content_bytes is canonical JSON (sorted keys, recursive) for deterministic
|
|
38
|
+
* storage; contentHash is the canonical digest (normalize + hash) that matches
|
|
39
|
+
* what dedupTranscript computes so the two pipelines use the same linkage key.
|
|
40
|
+
*
|
|
41
|
+
* F4 fix: the hash key MUST match on both sides of the append/dedup split.
|
|
42
|
+
* Using computeContentDigest here (normalize → hash) ensures content_ref set
|
|
43
|
+
* by dedupTranscript always resolves to an existing dedup_mirror row even for
|
|
44
|
+
* whitespace/case-variant content. F5 fix: recursive canonicalize replaces the
|
|
45
|
+
* broken shallow-sorted JSON.stringify replacer.
|
|
13
46
|
*/
|
|
14
47
|
function toRawTranscriptRow(msg, sessionId, epochId) {
|
|
15
48
|
// Narrow to Message union (has content + timestamp).
|
|
@@ -17,11 +50,16 @@ function toRawTranscriptRow(msg, sessionId, epochId) {
|
|
|
17
50
|
const content = m.content;
|
|
18
51
|
if (content == null || content === "")
|
|
19
52
|
return null;
|
|
20
|
-
// Canonical
|
|
53
|
+
// Canonical bytes: string content → normalize for consistent byte content;
|
|
54
|
+
// object content → recursive canonicalize then JSON (no replacer).
|
|
21
55
|
const contentBytes = typeof content === "string"
|
|
22
|
-
? content
|
|
23
|
-
: JSON.stringify(
|
|
24
|
-
|
|
56
|
+
? content // F5: strings are primitives; their byte content is fixed
|
|
57
|
+
: JSON.stringify(canonicalize(content));
|
|
58
|
+
// F4 fix: use the same digest as dedupTranscript so contentHash is consistent
|
|
59
|
+
// on both sides of the append/dedup split. computeContentDigest normalizes
|
|
60
|
+
// (strip ANSI, NFC, case-fold, collapse whitespace) then hashes → the same
|
|
61
|
+
// key is stored in raw_transcript.content_hash AND used as the dedup link.
|
|
62
|
+
const { contentHash } = computeContentDigest(contentBytes);
|
|
25
63
|
return {
|
|
26
64
|
contentHash,
|
|
27
65
|
sessionId,
|
|
@@ -4,6 +4,12 @@
|
|
|
4
4
|
* Classifies turn-end error/stop signals into retry categories.
|
|
5
5
|
* Ordering matters: compaction-noop is matched BEFORE generic transient
|
|
6
6
|
* so a pi race / manual compact catch is never misclassified.
|
|
7
|
+
*
|
|
8
|
+
* R3 (retry redesign): a new 'poisoned-context' category for DETERMINISTIC
|
|
9
|
+
* request-rejection failures that retrying cannot fix. See agent-handlers.ts
|
|
10
|
+
* for the poisoned handling (no blind retry; one /clear advise; one guarded
|
|
11
|
+
* compact per signature). Conservative: when unsure between transient and
|
|
12
|
+
* poisoned, choose transient (bounded by R1 dedup + R2 session cap).
|
|
7
13
|
*/
|
|
8
14
|
/** S38.2: classify a turn-end error/stop signal into a retry category.
|
|
9
15
|
*
|
|
@@ -11,11 +17,19 @@
|
|
|
11
17
|
* exclusively (its agent_end nudge path is separate and must not be doubled).
|
|
12
18
|
*
|
|
13
19
|
* @param message the event.message (a pi AgentMessage) or an error string
|
|
14
|
-
* @returns 'transient' | 'permanent' | 'compaction-noop' | 'context-overflow' | 'cancelled' | null (success/unknown)
|
|
20
|
+
* @returns 'transient' | 'permanent' | 'compaction-noop' | 'context-overflow' | 'cancelled' | 'poisoned-context' | null (success/unknown)
|
|
15
21
|
*/
|
|
16
22
|
export function classifyError(message) {
|
|
17
23
|
// Resolve a searchable text blob from a pi AgentMessage or raw string.
|
|
18
24
|
let text = '';
|
|
25
|
+
// R3: usage-token signal. Only when `usage` is explicitly present with
|
|
26
|
+
// inputTokens=0 AND outputTokens=0 do we know the turn never reached the
|
|
27
|
+
// model. An ABSENT usage field means "unknown" (the event didn't carry
|
|
28
|
+
// token counts) — we do NOT treat that as the 0-token poisoned signal,
|
|
29
|
+
// because mid-response stream deaths and partial-content turns routinely
|
|
30
|
+
// omit usage while still being transient (preserves all pre-R3 tests).
|
|
31
|
+
let usagePresent = false;
|
|
32
|
+
let totalTokens = 0;
|
|
19
33
|
if (typeof message === 'string') {
|
|
20
34
|
text = message;
|
|
21
35
|
}
|
|
@@ -32,6 +46,15 @@ export function classifyError(message) {
|
|
|
32
46
|
// Success / normal tool flow — not an error, nothing to retry.
|
|
33
47
|
if (sr === 'stop' || sr === 'toolUse' || sr === 'tool_use')
|
|
34
48
|
return null;
|
|
49
|
+
// R3: extract usage tokens for the 0-token poisoned-context signal.
|
|
50
|
+
const usage = m.usage;
|
|
51
|
+
usagePresent = usage != null && typeof usage === 'object';
|
|
52
|
+
if (usagePresent) {
|
|
53
|
+
const u = usage;
|
|
54
|
+
totalTokens =
|
|
55
|
+
(typeof u.inputTokens === 'number' ? u.inputTokens : 0) +
|
|
56
|
+
(typeof u.outputTokens === 'number' ? u.outputTokens : 0);
|
|
57
|
+
}
|
|
35
58
|
const parts = [];
|
|
36
59
|
if (sr)
|
|
37
60
|
parts.push(sr);
|
|
@@ -111,36 +134,120 @@ export function classifyError(message) {
|
|
|
111
134
|
if (/too long|context window|maximum context length|context length exceeded|requires at least \d+ tokens|even after compaction|reduce the conversation|reduce your input/.test(s)) {
|
|
112
135
|
return 'context-overflow';
|
|
113
136
|
}
|
|
114
|
-
// --- transient (
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
// NOTE: 'aborted' stopReason is handled by the sr==='aborted' early-return above.
|
|
119
|
-
// The text-based s.includes('aborted') was removed — it was the old path that
|
|
120
|
-
// misclassified user ESC/Ctrl-C as 'transient' (5 retry nudges after cancel).
|
|
137
|
+
// --- transient (specific markers FIRST — these override poisoned signals) ---
|
|
138
|
+
// R3: network failures (timeout, ECONNRESET, 5xx, 429) MUST stay transient
|
|
139
|
+
// even when usage is 0 tokens, because they are retryable. The specific
|
|
140
|
+
// markers below return before any poisoned signal is evaluated.
|
|
121
141
|
if (/max(imum)? output token/.test(s))
|
|
122
142
|
return 'transient';
|
|
123
143
|
if (/rate[\s.-]?limit|429|too many requests/.test(s))
|
|
124
144
|
return 'transient';
|
|
125
145
|
if (/5\d\d|internal server|bad gateway|service unavailable/.test(s))
|
|
126
146
|
return 'transient';
|
|
127
|
-
|
|
147
|
+
// 'connection aborted' is a network failure (ECONNABORTED), NOT a user ESC
|
|
148
|
+
// (which is stopReason 'aborted', handled by the early-return above). Added
|
|
149
|
+
// so "Connection aborted by peer" stays transient under R3. ECONNRESET and
|
|
150
|
+
// common errno names are included so a bare 0-token 'ECONNRESET' turn is not
|
|
151
|
+
// misclassified as poisoned (R6: ECONNRESET → transient).
|
|
152
|
+
if (/network|timeout|econnreset|econnrefused|epipe|connection (lost|refused|reset|aborted)|stream (interrupted|closed|ended|failed)|disconnected/.test(s))
|
|
128
153
|
return 'transient';
|
|
154
|
+
// --- poisoned-context (R3: ORDER AFTER specific transient markers, BEFORE
|
|
155
|
+
// the generic 'error' transient fallthrough) ---
|
|
156
|
+
// A DETERMINISTIC request-rejection that retrying cannot fix. Re-submitting
|
|
157
|
+
// the same prompt re-triggers the same rejection. The 2026-07-28 incident
|
|
158
|
+
// was a provider returning 0-token "Request failed — please retry." turns
|
|
159
|
+
// for every request; classifyError mapped stopReason 'error' → 'transient'
|
|
160
|
+
// and the session emitted ~9 nudge bursts (~60 context-bloating messages)
|
|
161
|
+
// before stopping.
|
|
162
|
+
//
|
|
163
|
+
// Signal 1: provider request-validation 400 that is NOT context-overflow
|
|
164
|
+
// (context-overflow already returned above). Orphaned tool result / malformed
|
|
165
|
+
// message structure / unexpected role ordering / empty content. These were
|
|
166
|
+
// previously classified 'permanent' (max 1 retry) — but a single retry
|
|
167
|
+
// re-submits the SAME malformed structure and re-400s, so they are poisoned,
|
|
168
|
+
// not permanent. Auth/permission 400s stay permanent (separate check below).
|
|
169
|
+
if (/invalid_request_error|invalid request|malformed|bad request|orphaned tool|tool (result )?(without|for )|unexpected role|role (ordering|sequence)|empty (content|message|request)/.test(s)) {
|
|
170
|
+
return 'poisoned-context';
|
|
171
|
+
}
|
|
172
|
+
// Signal 2: generic catch-all "request failed" / "request error" with no
|
|
173
|
+
// specific transient marker (transient markers already returned above).
|
|
174
|
+
// This is the exact phrasing the 2026-07-28 incident used. Without a
|
|
175
|
+
// specific retryable cause, treat the rejection as deterministic.
|
|
176
|
+
if (/request failed|request error/.test(s)) {
|
|
177
|
+
return 'poisoned-context';
|
|
178
|
+
}
|
|
179
|
+
// Signal 3: stopReason 'error' + usage explicitly 0 tokens (the turn never
|
|
180
|
+
// reached the model). Conservative: only when `usage` is PRESENT and 0 — an
|
|
181
|
+
// absent usage field means "unknown" and stays transient (mid-response
|
|
182
|
+
// deaths / partial-content turns often omit usage). The transient markers
|
|
183
|
+
// above already returned, so reaching here with 'error' text and 0 tokens
|
|
184
|
+
// means the request was rejected before any model work.
|
|
185
|
+
if (usagePresent && totalTokens === 0 && /error/.test(s)) {
|
|
186
|
+
return 'poisoned-context';
|
|
187
|
+
}
|
|
188
|
+
// --- transient (generic 'error' stopReason fallthrough) ---
|
|
189
|
+
// Reaches here for a generic stopReason 'error' with no specific marker and
|
|
190
|
+
// no 0-token signal (usage absent or > 0). Stays transient — conservative.
|
|
191
|
+
if (s.includes('error') && !/\b(auth|unauthorized|invalid (api )?key|permission)\b/.test(s)) {
|
|
192
|
+
return 'transient'; // generic pi stopReason 'error'
|
|
193
|
+
}
|
|
194
|
+
// NOTE: 'aborted' stopReason is handled by the sr==='aborted' early-return above.
|
|
195
|
+
// The text-based s.includes('aborted') was removed — it was the old path that
|
|
196
|
+
// misclassified user ESC/Ctrl-C as 'transient' (5 retry nudges after cancel).
|
|
129
197
|
// --- permanent (NOT retryable beyond 1) ---
|
|
130
198
|
if (/auth|unauthorized|invalid (api )?key|permission/.test(s))
|
|
131
199
|
return 'permanent';
|
|
132
|
-
if (/invalid request|malformed|bad request/.test(s))
|
|
133
|
-
return 'permanent';
|
|
134
200
|
// Unknown — do not retry (avoid busy-looping on an unclassified signal).
|
|
135
201
|
return null;
|
|
136
202
|
}
|
|
137
|
-
/** S38.2: exponential backoff for error-retry nudges
|
|
138
|
-
* count is 1-based (the retry about to fire).
|
|
139
|
-
|
|
203
|
+
/** S38.2: exponential backoff for error-retry nudges.
|
|
204
|
+
* count is 1-based (the retry about to fire). baseMs is the unit
|
|
205
|
+
* (config.errorRetryBackoffMs, default 5000 → 5s/10s/20s/30s cap).
|
|
206
|
+
* R1: errorRetryUntil is now GATING — a nudge cannot fire before the previous
|
|
207
|
+
* backoff elapses (previously documented as non-gating). */
|
|
208
|
+
export function errorRetryBackoffMs(count, baseMs = 5000) {
|
|
140
209
|
switch (count) {
|
|
141
|
-
case 1: return
|
|
142
|
-
case 2: return
|
|
143
|
-
case 3: return
|
|
144
|
-
default: return
|
|
210
|
+
case 1: return baseMs;
|
|
211
|
+
case 2: return baseMs * 2;
|
|
212
|
+
case 3: return baseMs * 4;
|
|
213
|
+
default: return baseMs * 6; // cap from the 4th retry onward
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
/** R3: extract a normalized error-signature string from a turn_end message for
|
|
217
|
+
* the stateful "repeated identical error text" signal. Returns the lowercased
|
|
218
|
+
* content + error.message blob (WITHOUT the stopReason, which is constant
|
|
219
|
+
* 'error' for retryable turns) so two turns with the same error content share
|
|
220
|
+
* a signature. Empty string when there is no error text (e.g. a bare
|
|
221
|
+
* stopReason 'error' with no content) — the caller skips repeat detection in
|
|
222
|
+
* that case (the stateless 0-token signal handles bare errors). */
|
|
223
|
+
export function extractErrorSignature(message) {
|
|
224
|
+
if (typeof message === 'string')
|
|
225
|
+
return message.toLowerCase().trim();
|
|
226
|
+
if (!message || typeof message !== 'object')
|
|
227
|
+
return '';
|
|
228
|
+
const m = message;
|
|
229
|
+
const parts = [];
|
|
230
|
+
const c = m.content;
|
|
231
|
+
if (typeof c === 'string')
|
|
232
|
+
parts.push(c);
|
|
233
|
+
else if (Array.isArray(c)) {
|
|
234
|
+
for (const b of c) {
|
|
235
|
+
if (b && typeof b === 'object' && 'text' in b) {
|
|
236
|
+
parts.push(String(b.text ?? ''));
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (m.error) {
|
|
241
|
+
const err = m.error;
|
|
242
|
+
if (typeof err === 'string')
|
|
243
|
+
parts.push(err);
|
|
244
|
+
else if (err && typeof err === 'object') {
|
|
245
|
+
const errObj = err;
|
|
246
|
+
if (typeof errObj.message === 'string')
|
|
247
|
+
parts.push(errObj.message);
|
|
248
|
+
else
|
|
249
|
+
parts.push(JSON.stringify(err));
|
|
250
|
+
}
|
|
145
251
|
}
|
|
252
|
+
return parts.join(' ').toLowerCase().trim();
|
|
146
253
|
}
|