pi-mega-compact 0.11.3 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/mega-compact-s38.test.js +577 -14
- package/dist/extensions/mega-config.js +1 -0
- package/dist/extensions/mega-events/agent-handlers.js +22 -2
- package/dist/extensions/mega-events/error-classifier.js +106 -127
- package/dist/extensions/mega-events/outage-advisor.js +53 -0
- package/dist/extensions/mega-runtime/reset-runtime.js +1 -0
- package/dist/extensions/mega-runtime/runtime.js +1 -0
- package/extensions/mega-compact-s38.test.ts +575 -14
- package/extensions/mega-config.ts +8 -0
- package/extensions/mega-events/agent-handlers.ts +24 -1
- package/extensions/mega-events/error-classifier.ts +145 -124
- package/extensions/mega-events/outage-advisor.ts +73 -0
- package/extensions/mega-runtime/helpers.ts +2 -0
- package/extensions/mega-runtime/reset-runtime.ts +1 -0
- package/extensions/mega-runtime/runtime.ts +1 -0
- package/package.json +1 -1
|
@@ -148,6 +148,7 @@ export function loadConfig() {
|
|
|
148
148
|
errorRetryBackoffMs: envFlag("MEGACOMPACT_ERROR_RETRY_BACKOFF_MS", 5000),
|
|
149
149
|
errorRetrySessionMax: envFlag("MEGACOMPACT_ERROR_RETRY_SESSION_MAX", 3),
|
|
150
150
|
poisonedContextRepeatThreshold: envFlag("MEGACOMPACT_POISONED_REPEAT_THRESHOLD", 3),
|
|
151
|
+
providerOutageAdviseThreshold: envFlag("MEGACOMPACT_PROVIDER_OUTAGE_THRESHOLD", 3),
|
|
151
152
|
autoPctTrigger,
|
|
152
153
|
autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
|
|
153
154
|
dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
|
|
@@ -5,8 +5,9 @@ import { evaluateAndUnlockAchievements } from "../../src/store/sqlite/game-achie
|
|
|
5
5
|
import { ensureConversationIdFor, recordTurnWrite, } from "../mega-turn-store.js";
|
|
6
6
|
import { isMegaCache } from "../../src/game/scoring.js";
|
|
7
7
|
import { resolveRepoRoot } from "../mega-config.js";
|
|
8
|
-
import { classifyError, errorRetryBackoffMs, extractErrorSignature, isKnownRetryableTransient, } from "./error-classifier.js";
|
|
8
|
+
import { classifyError, classifyErrorDetailed, errorRetryBackoffMs, extractErrorSignature, isKnownRetryableTransient, } from "./error-classifier.js";
|
|
9
9
|
import { safeSendUserMessage } from "./send-safe.js";
|
|
10
|
+
import { maybeSendProviderOutageAdvisory } from "./outage-advisor.js";
|
|
10
11
|
import { vectorStats } from "../../src/vectorStore.js";
|
|
11
12
|
/** Register agent/turn tracking event handlers. */
|
|
12
13
|
export function registerAgentHandlers(pi, runtime, config) {
|
|
@@ -336,6 +337,7 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
336
337
|
}
|
|
337
338
|
else {
|
|
338
339
|
const category = classifyError(event.message);
|
|
340
|
+
const detail = classifyErrorDetailed(event.message);
|
|
339
341
|
// R3: stateful poisoned-context signal — repeated identical error text
|
|
340
342
|
// across consecutive turns. The classifier is stateless, so this
|
|
341
343
|
// upgrade happens here. A 'transient' (or 'poisoned-context') turn
|
|
@@ -373,6 +375,13 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
373
375
|
if (!isKnownRetryableTransient(errSig)) {
|
|
374
376
|
effectiveCategory = "poisoned-context";
|
|
375
377
|
}
|
|
378
|
+
else {
|
|
379
|
+
runtime.logger.info("repeat-upgrade-declined", {
|
|
380
|
+
sessionId: runtime.rt.sessionId,
|
|
381
|
+
signature: (errSig || '').slice(0, 500),
|
|
382
|
+
repeatCount: runtime.rt.errorTextRepeatCount,
|
|
383
|
+
});
|
|
384
|
+
}
|
|
376
385
|
}
|
|
377
386
|
}
|
|
378
387
|
}
|
|
@@ -388,6 +397,9 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
388
397
|
runtime.rt.consecutiveErrors = 0; // S38.6: circuit-breaker reset on success
|
|
389
398
|
// R4: a successful assistant turn consumes any queued nudge.
|
|
390
399
|
runtime.rt.retryNudgePending = false;
|
|
400
|
+
// R10: reset outage advisory so a recovered-then-flapping
|
|
401
|
+
// provider re-advises once per episode.
|
|
402
|
+
runtime.rt.providerOutageAdvised = false;
|
|
391
403
|
}
|
|
392
404
|
else if (effectiveCategory === "compaction-noop") {
|
|
393
405
|
// (4) pi race / manual compact catch — NOT retryable. The compaction
|
|
@@ -487,18 +499,24 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
487
499
|
runtime.rt.consecutiveErrors++; // still counts toward the circuit breaker
|
|
488
500
|
runtime.rt.poisonedCount++; // R7: dashboard counter
|
|
489
501
|
const sig = errSig || "unknown";
|
|
490
|
-
// (a) dashboard + log
|
|
502
|
+
// (a) dashboard + log — R11: include signal + rawText for forensics
|
|
491
503
|
runtime.dashboard.event("poisoned_context", {
|
|
492
504
|
signature: sig,
|
|
493
505
|
repeatCount: runtime.rt.errorTextRepeatCount,
|
|
494
506
|
turnIndex: event.turnIndex,
|
|
495
507
|
sessionId: runtime.rt.sessionId,
|
|
508
|
+
signal: detail.signal,
|
|
509
|
+
rawText: (errSig || '').slice(0, 500),
|
|
510
|
+
...(detail.httpStatus !== undefined ? { httpStatus: detail.httpStatus } : {}),
|
|
496
511
|
});
|
|
497
512
|
runtime.logger.warn("poisoned-context", {
|
|
498
513
|
sessionId: runtime.rt.sessionId,
|
|
499
514
|
turnIndex: event.turnIndex,
|
|
500
515
|
signature: sig,
|
|
501
516
|
repeatCount: runtime.rt.errorTextRepeatCount,
|
|
517
|
+
signal: detail.signal,
|
|
518
|
+
rawText: (errSig || '').slice(0, 500),
|
|
519
|
+
...(detail.httpStatus !== undefined ? { httpStatus: detail.httpStatus } : {}),
|
|
502
520
|
});
|
|
503
521
|
// (b) one-per-session advise message (throttled by poisonedAdviseSent)
|
|
504
522
|
if (!runtime.rt.poisonedAdviseSent) {
|
|
@@ -552,6 +570,8 @@ export function registerAgentHandlers(pi, runtime, config) {
|
|
|
552
570
|
}
|
|
553
571
|
// S38.6: circuit-breaker — stop retrying after too many consecutive errors.
|
|
554
572
|
runtime.rt.consecutiveErrors++;
|
|
573
|
+
// R10: send calm "provider outage" advisory once per episode.
|
|
574
|
+
await maybeSendProviderOutageAdvisory(effectiveCategory, runtime, pi, config, { signal: detail.signal, rawText: (errSig || '').slice(0, 500) });
|
|
555
575
|
if (runtime.rt.consecutiveErrors > config.maxConsecutiveErrors) {
|
|
556
576
|
runtime.dashboard.event("error_retry_circuit_open", {
|
|
557
577
|
consecutive: runtime.rt.consecutiveErrors,
|
|
@@ -23,49 +23,48 @@
|
|
|
23
23
|
* - `etimedout` is Node's timeout errno lowercased — does NOT contain the
|
|
24
24
|
* substring "timeout".
|
|
25
25
|
* - `socket hang up` is Node's ECONNRESET *message*; the errno lives in
|
|
26
|
-
* error.code, which extractErrorSignature never sees.
|
|
27
|
-
|
|
26
|
+
* error.code, which extractErrorSignature never sees.
|
|
27
|
+
* - router phrasings (all targets failed / no healthy target / too many
|
|
28
|
+
* concurrent) -- pi's status prefix is console-only (2026-07-30 incident). */
|
|
29
|
+
export const KNOWN_RETRYABLE_TRANSIENT_PATTERN = /max(imum)? output token|rate[\s.-]?limit|429|too many requests|too many concurrent|overloaded|5\d\d|internal server|bad gateway|service unavailable|all targets failed|no healthy target|fetch failed|network|time(?:d[\s-]?out|out)|etimedout|econnreset|econnrefused|epipe|eai_again|socket|closed unexpectedly|premature close|other side closed|connection (lost|refused|reset|aborted?|was closed)|stream (interrupted|closed|ended|failed)|disconnected/;
|
|
28
30
|
/** R7: true when the error text carries a known-retryable transient marker.
|
|
29
31
|
* Defensively lowercases so callers can pass raw or normalized text. */
|
|
30
32
|
export function isKnownRetryableTransient(text) {
|
|
31
33
|
return KNOWN_RETRYABLE_TRANSIENT_PATTERN.test(text.toLowerCase());
|
|
32
34
|
}
|
|
33
|
-
/**
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
35
|
+
/** R8: best-effort extraction of HTTP status from pi AgentMessage shapes. */
|
|
36
|
+
function extractHttpStatus(m) {
|
|
37
|
+
for (const outer of [m.error, m]) {
|
|
38
|
+
if (!outer || typeof outer !== 'object')
|
|
39
|
+
continue;
|
|
40
|
+
for (const key of ['status', 'statusCode', 'code']) {
|
|
41
|
+
const v = outer[key];
|
|
42
|
+
if (typeof v === 'number' && v >= 100 && v <= 599)
|
|
43
|
+
return v;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
/** R11: classify a turn-end error/stop signal into a retry category with a
|
|
49
|
+
* diagnostic signal tag naming the rule that fired, plus the raw httpStatus
|
|
50
|
+
* when extractHttpStatus found one. */
|
|
51
|
+
export function classifyErrorDetailed(message) {
|
|
43
52
|
let text = '';
|
|
44
|
-
// R3: usage-token signal. Only when `usage` is explicitly present with
|
|
45
|
-
// inputTokens=0 AND outputTokens=0 do we know the turn never reached the
|
|
46
|
-
// model. An ABSENT usage field means "unknown" (the event didn't carry
|
|
47
|
-
// token counts) — we do NOT treat that as the 0-token poisoned signal,
|
|
48
|
-
// because mid-response stream deaths and partial-content turns routinely
|
|
49
|
-
// omit usage while still being transient (preserves all pre-R3 tests).
|
|
50
53
|
let usagePresent = false;
|
|
51
54
|
let totalTokens = 0;
|
|
55
|
+
let httpStatus;
|
|
52
56
|
if (typeof message === 'string') {
|
|
53
57
|
text = message;
|
|
54
58
|
}
|
|
55
59
|
else if (message && typeof message === 'object') {
|
|
56
60
|
const m = message;
|
|
57
61
|
const sr = typeof m.stopReason === 'string' ? m.stopReason : '';
|
|
58
|
-
// S28 guard: length stopReason is handled exclusively by the S28 path.
|
|
59
62
|
if (sr === 'length')
|
|
60
|
-
return null;
|
|
61
|
-
// User ESC / Ctrl-C abort — stopReason === 'aborted'. Not retryable:
|
|
62
|
-
// nudging would restart a task the user explicitly stopped.
|
|
63
|
+
return { category: null, signal: 'length-guard' };
|
|
63
64
|
if (sr === 'aborted')
|
|
64
|
-
return 'cancelled';
|
|
65
|
-
// Success / normal tool flow — not an error, nothing to retry.
|
|
65
|
+
return { category: 'cancelled', signal: 'cancelled' };
|
|
66
66
|
if (sr === 'stop' || sr === 'toolUse' || sr === 'tool_use')
|
|
67
|
-
return null;
|
|
68
|
-
// R3: extract usage tokens for the 0-token poisoned-context signal.
|
|
67
|
+
return { category: null, signal: 'success' };
|
|
69
68
|
const usage = m.usage;
|
|
70
69
|
usagePresent = usage != null && typeof usage === 'object';
|
|
71
70
|
if (usagePresent) {
|
|
@@ -74,6 +73,7 @@ export function classifyError(message) {
|
|
|
74
73
|
(typeof u.inputTokens === 'number' ? u.inputTokens : 0) +
|
|
75
74
|
(typeof u.outputTokens === 'number' ? u.outputTokens : 0);
|
|
76
75
|
}
|
|
76
|
+
httpStatus = extractHttpStatus(m);
|
|
77
77
|
const parts = [];
|
|
78
78
|
if (sr)
|
|
79
79
|
parts.push(sr);
|
|
@@ -88,128 +88,75 @@ export function classifyError(message) {
|
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
90
|
if (m.error) {
|
|
91
|
-
// Extract message from error objects for pattern matching.
|
|
92
91
|
const err = m.error;
|
|
93
|
-
if (typeof err === 'string')
|
|
92
|
+
if (typeof err === 'string')
|
|
94
93
|
parts.push(err);
|
|
95
|
-
}
|
|
96
94
|
else if (err && typeof err === 'object') {
|
|
97
95
|
const errObj = err;
|
|
98
|
-
if (typeof errObj.message === 'string')
|
|
96
|
+
if (typeof errObj.message === 'string')
|
|
99
97
|
parts.push(errObj.message);
|
|
100
|
-
|
|
101
|
-
else {
|
|
98
|
+
else
|
|
102
99
|
parts.push(JSON.stringify(err));
|
|
103
|
-
}
|
|
104
100
|
}
|
|
105
101
|
}
|
|
106
|
-
// S38: detect mid-response errors where the stream died without a
|
|
107
|
-
// proper stopReason (empty/undefined) — this catches provider failures
|
|
108
|
-
// that cut off the response mid-stream, INCLUDING the case where the
|
|
109
|
-
// provider emitted partial content before dying (a truncated response
|
|
110
|
-
// with no stop reason is a mid-stream death, not a success).
|
|
111
|
-
// A genuine success ALWAYS carries stop/tool_use/toolUse (which
|
|
112
|
-
// short-circuit to null at the top), so any message reaching here with a
|
|
113
|
-
// falsy stopReason is a stream failure → retryable.
|
|
114
102
|
if (!sr) {
|
|
115
|
-
|
|
103
|
+
const r = { category: 'transient', signal: 'stream-death-no-stopreason' };
|
|
104
|
+
if (httpStatus !== undefined)
|
|
105
|
+
r.httpStatus = httpStatus;
|
|
106
|
+
return r;
|
|
116
107
|
}
|
|
117
108
|
text = parts.join(' ');
|
|
118
109
|
}
|
|
119
110
|
if (!text)
|
|
120
|
-
return null;
|
|
111
|
+
return { category: null, signal: 'unknown' };
|
|
121
112
|
const s = text.toLowerCase();
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (/already compacted/.test(s))
|
|
126
|
-
return 'compaction-noop';
|
|
127
|
-
if (/compaction failed/.test(s))
|
|
128
|
-
return 'compaction-noop';
|
|
129
|
-
if (/nothing to compact/.test(s))
|
|
130
|
-
return 'compaction-noop';
|
|
131
|
-
if (/auto[\s-]?compaction failed/.test(s))
|
|
132
|
-
return 'compaction-noop';
|
|
133
|
-
// --- context-overflow (ORDER BEFORE generic transient!) ---
|
|
134
|
-
// A 400 from the model meaning "the prompt is bigger than the context window."
|
|
135
|
-
// Catches BOTH provider phrasings so the classification does not depend on
|
|
136
|
-
// which backend the router landed on:
|
|
137
|
-
// - pi/Anthropic wrapper: "too long for this model's context window even
|
|
138
|
-
// after compaction. Reduce the conversation length..." ->
|
|
139
|
-
// too long | context window | even after compaction | reduce the conversation
|
|
140
|
-
// - OpenAI/OpenRouter provider-side (often wrapped in "All targets failed:
|
|
141
|
-
// <model>. Last error: ..."): "This model's maximum context length is N
|
|
142
|
-
// tokens. Your request requires at least M tokens. Please reduce your
|
|
143
|
-
// input or max_tokens." -> maximum context length | context length
|
|
144
|
-
// exceeded | requires at least N tokens | reduce your input
|
|
145
|
-
// All of these carry `invalid_request_error` (UNDERSCORE, not 'invalid
|
|
146
|
-
// request' space) and would otherwise fall through to the generic
|
|
147
|
-
// `s.includes('error')` transient branch below, misclassifying as
|
|
148
|
-
// 'transient' and firing up to 5 blind retry nudges that re-submit the same
|
|
149
|
-
// oversized prompt -> re-400 -> busy-loop. 'context-overflow' instead forces
|
|
150
|
-
// ONE deferred re-compact (debounce-bypassed, race-guarded) and fires NO
|
|
151
|
-
// blind retry nudge. The forced re-compact is shaped by the existing
|
|
152
|
-
// session_before_compact durable trim (it cannot lower pi's firstKeptEntryId).
|
|
113
|
+
if (/already compacted|compaction failed|nothing to compact|auto[\s-]?compaction failed/.test(s)) {
|
|
114
|
+
return { category: 'compaction-noop', signal: 'compaction-noop' };
|
|
115
|
+
}
|
|
153
116
|
if (/too long|context window|maximum context length|context length exceeded|requires at least \d+ tokens|even after compaction|reduce the conversation|reduce your input/.test(s)) {
|
|
154
|
-
return 'context-overflow';
|
|
117
|
+
return { category: 'context-overflow', signal: 'context-overflow' };
|
|
118
|
+
}
|
|
119
|
+
if (isKnownRetryableTransient(s)) {
|
|
120
|
+
const r = { category: 'transient', signal: 'transient-marker' };
|
|
121
|
+
if (httpStatus !== undefined)
|
|
122
|
+
r.httpStatus = httpStatus;
|
|
123
|
+
return r;
|
|
124
|
+
}
|
|
125
|
+
if (httpStatus === 429 || (httpStatus !== undefined && httpStatus >= 500)) {
|
|
126
|
+
return { category: 'transient', signal: 'transient-status', httpStatus };
|
|
127
|
+
}
|
|
128
|
+
if (httpStatus === 401 || httpStatus === 403) {
|
|
129
|
+
return { category: 'permanent', signal: 'permanent-status', httpStatus };
|
|
155
130
|
}
|
|
156
|
-
// --- transient (known-retryable markers FIRST — these override poisoned signals) ---
|
|
157
|
-
// R3: network/throughput failures (timeout, ECONNRESET, 5xx, 429) MUST stay
|
|
158
|
-
// transient even when usage is 0 tokens, because they are retryable and
|
|
159
|
-
// /clear cannot fix them. 'connection aborted' is a network failure
|
|
160
|
-
// (ECONNABORTED), NOT a user ESC (stopReason 'aborted', early-return above).
|
|
161
|
-
// The marker set is shared with the agent-handlers R3 repeat-upgrade guard
|
|
162
|
-
// (isKnownRetryableTransient) so the two never drift.
|
|
163
|
-
if (isKnownRetryableTransient(s))
|
|
164
|
-
return 'transient';
|
|
165
|
-
// --- poisoned-context (R3: ORDER AFTER specific transient markers, BEFORE
|
|
166
|
-
// the generic 'error' transient fallthrough) ---
|
|
167
|
-
// A DETERMINISTIC request-rejection that retrying cannot fix. Re-submitting
|
|
168
|
-
// the same prompt re-triggers the same rejection. The 2026-07-28 incident
|
|
169
|
-
// was a provider returning 0-token "Request failed — please retry." turns
|
|
170
|
-
// for every request; classifyError mapped stopReason 'error' → 'transient'
|
|
171
|
-
// and the session emitted ~9 nudge bursts (~60 context-bloating messages)
|
|
172
|
-
// before stopping.
|
|
173
|
-
//
|
|
174
|
-
// Signal 1: provider request-validation 400 that is NOT context-overflow
|
|
175
|
-
// (context-overflow already returned above). Orphaned tool result / malformed
|
|
176
|
-
// message structure / unexpected role ordering / empty content. These were
|
|
177
|
-
// previously classified 'permanent' (max 1 retry) — but a single retry
|
|
178
|
-
// re-submits the SAME malformed structure and re-400s, so they are poisoned,
|
|
179
|
-
// not permanent. Auth/permission 400s stay permanent (separate check below).
|
|
180
131
|
if (/invalid_request_error|invalid request|malformed|bad request|orphaned tool|tool (result )?(without|for )|unexpected role|role (ordering|sequence)|empty (content|message|request)/.test(s)) {
|
|
181
|
-
return 'poisoned-context';
|
|
132
|
+
return { category: 'poisoned-context', signal: 'poisoned-invalid-request' };
|
|
182
133
|
}
|
|
183
|
-
// Signal 2: generic catch-all "request failed" / "request error" with no
|
|
184
|
-
// specific transient marker (transient markers already returned above).
|
|
185
|
-
// This is the exact phrasing the 2026-07-28 incident used. Without a
|
|
186
|
-
// specific retryable cause, treat the rejection as deterministic.
|
|
187
134
|
if (/request failed|request error/.test(s)) {
|
|
188
|
-
return 'poisoned-context';
|
|
135
|
+
return { category: 'poisoned-context', signal: 'poisoned-request-failed' };
|
|
189
136
|
}
|
|
190
|
-
// Signal 3: stopReason 'error' + usage explicitly 0 tokens (the turn never
|
|
191
|
-
// reached the model). Conservative: only when `usage` is PRESENT and 0 — an
|
|
192
|
-
// absent usage field means "unknown" and stays transient (mid-response
|
|
193
|
-
// deaths / partial-content turns often omit usage). The transient markers
|
|
194
|
-
// above already returned, so reaching here with 'error' text and 0 tokens
|
|
195
|
-
// means the request was rejected before any model work.
|
|
196
137
|
if (usagePresent && totalTokens === 0 && /error/.test(s)) {
|
|
197
|
-
return '
|
|
138
|
+
return { category: 'transient', signal: 'bare-0-token' };
|
|
198
139
|
}
|
|
199
|
-
// --- transient (generic 'error' stopReason fallthrough) ---
|
|
200
|
-
// Reaches here for a generic stopReason 'error' with no specific marker and
|
|
201
|
-
// no 0-token signal (usage absent or > 0). Stays transient — conservative.
|
|
202
140
|
if (s.includes('error') && !/\b(auth|unauthorized|invalid (api )?key|permission)\b/.test(s)) {
|
|
203
|
-
return 'transient'
|
|
141
|
+
return { category: 'transient', signal: 'generic-error' };
|
|
142
|
+
}
|
|
143
|
+
if (/auth|unauthorized|invalid (api )?key|permission/.test(s)) {
|
|
144
|
+
return { category: 'permanent', signal: 'permanent-auth' };
|
|
204
145
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
146
|
+
return { category: null, signal: 'unknown' };
|
|
147
|
+
}
|
|
148
|
+
/** S38.2: classify a turn-end error/stop signal into a retry category.
|
|
149
|
+
*
|
|
150
|
+
* `length` is returned as null — S28 owns the max-output-token length stopReason
|
|
151
|
+
* exclusively (its agent_end nudge path is separate and must not be doubled).
|
|
152
|
+
*
|
|
153
|
+
* Thin wrapper around classifyErrorDetailed — returns only the category.
|
|
154
|
+
*
|
|
155
|
+
* @param message the event.message (a pi AgentMessage) or an error string
|
|
156
|
+
* @returns 'transient' | 'permanent' | 'compaction-noop' | 'context-overflow' | 'cancelled' | 'poisoned-context' | null (success/unknown)
|
|
157
|
+
*/
|
|
158
|
+
export function classifyError(message) {
|
|
159
|
+
return classifyErrorDetailed(message).category;
|
|
213
160
|
}
|
|
214
161
|
/** S38.2: exponential backoff for error-retry nudges.
|
|
215
162
|
* count is 1-based (the retry about to fire). baseMs is the unit
|
|
@@ -230,10 +177,33 @@ export function errorRetryBackoffMs(count, baseMs = 5000) {
|
|
|
230
177
|
* 'error' for retryable turns) so two turns with the same error content share
|
|
231
178
|
* a signature. Empty string when there is no error text (e.g. a bare
|
|
232
179
|
* stopReason 'error' with no content) — the caller skips repeat detection in
|
|
233
|
-
* that case (the stateless 0-token signal handles bare errors).
|
|
180
|
+
* that case (the stateless 0-token signal handles bare errors).
|
|
181
|
+
*
|
|
182
|
+
* R9: when the joined text is empty AND stopReason is 'error' AND usage is
|
|
183
|
+
* present with 0 total tokens, returns the constant "bare-0-token-error" so
|
|
184
|
+
* the repeat detector can corroborate bare 0-token errors (first turn is
|
|
185
|
+
* transient; repeated occurrences upgrade to poisoned at threshold). Bare
|
|
186
|
+
* 'error' WITHOUT usage keeps returning '' (absent usage = unknown; mid-
|
|
187
|
+
* response deaths stay out of repeat tracking). */
|
|
188
|
+
/** R12: normalize volatile tokens in an already-lowercased error string so that
|
|
189
|
+
* equivalent errors with varied model aliases, IPs, hex request-ids, retry
|
|
190
|
+
* counts, and long numeric ids produce the same signature. Order matters.
|
|
191
|
+
* 3-digit HTTP status codes (429/500/502/etc.) survive — the long-number rule
|
|
192
|
+
* only covers 4+ digits. Invariant: normalizeVolatileTokens never empties a
|
|
193
|
+
* non-empty string (every regex replaces with a non-empty literal token). */
|
|
194
|
+
function normalizeVolatileTokens(s) {
|
|
195
|
+
let n = s;
|
|
196
|
+
n = n.replace(/\b\d{1,3}(\.\d{1,3}){3}(:\d+)?\b/g, '<ip>'); // 1. IP:port / bare IPv4
|
|
197
|
+
n = n.replace(/\b[\w-]+(\/[\w.:+-]+)+/g, '<model>'); // 2. slash-separated paths
|
|
198
|
+
n = n.replace(/\b[0-9a-f]{8,}\b/g, '<hex>'); // 3. hex ids (8+ chars)
|
|
199
|
+
n = n.replace(/\b\d{4,}\b/g, '<n>'); // 4. long numbers (4+ digits)
|
|
200
|
+
n = n.replace(/\bafter \d+ attempts?\b/g, 'after <n> attempts'); // 5. attempt phrasing
|
|
201
|
+
n = n.replace(/\s+/g, ' ').trim(); // 6. collapse whitespace
|
|
202
|
+
return n;
|
|
203
|
+
}
|
|
234
204
|
export function extractErrorSignature(message) {
|
|
235
205
|
if (typeof message === 'string')
|
|
236
|
-
return message.toLowerCase().trim();
|
|
206
|
+
return normalizeVolatileTokens(message.toLowerCase().trim());
|
|
237
207
|
if (!message || typeof message !== 'object')
|
|
238
208
|
return '';
|
|
239
209
|
const m = message;
|
|
@@ -260,5 +230,14 @@ export function extractErrorSignature(message) {
|
|
|
260
230
|
parts.push(JSON.stringify(err));
|
|
261
231
|
}
|
|
262
232
|
}
|
|
263
|
-
|
|
233
|
+
const joined = parts.join(' ').toLowerCase().trim();
|
|
234
|
+
const normalized = normalizeVolatileTokens(joined);
|
|
235
|
+
if (normalized === '' &&
|
|
236
|
+
m.stopReason === 'error' &&
|
|
237
|
+
m.usage &&
|
|
238
|
+
typeof m.usage === 'object' &&
|
|
239
|
+
(m.usage.inputTokens ?? 0) + (m.usage.outputTokens ?? 0) === 0) {
|
|
240
|
+
return 'bare-0-token-error';
|
|
241
|
+
}
|
|
242
|
+
return normalized;
|
|
264
243
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* outage-advisor.ts — R10: calm "provider outage" advisory.
|
|
3
|
+
*
|
|
4
|
+
* When a provider/router flaps (transient errors: timeouts, 5xx, 429s),
|
|
5
|
+
* the extension retries silently and eventually goes quiet after caps. This
|
|
6
|
+
* module sends ONE user-facing advisory per outage episode so the user knows
|
|
7
|
+
* to wait rather than running /clear.
|
|
8
|
+
*
|
|
9
|
+
* Distinct from the poisoned-context /clear advise: the outage advisory is
|
|
10
|
+
* for transient errors where the user's context is fine.
|
|
11
|
+
*
|
|
12
|
+
* PREVENT-PI-003: sends via safeSendUserMessage (user-role only).
|
|
13
|
+
* PREVENT-PI-004: local ctx call, no network.
|
|
14
|
+
*/
|
|
15
|
+
import { safeSendUserMessage } from "./send-safe.js";
|
|
16
|
+
/**
|
|
17
|
+
* Check the provider-outage advisory condition and fire once per episode.
|
|
18
|
+
*
|
|
19
|
+
* Called from the transient/permanent retry branch of the turn_end handler,
|
|
20
|
+
* AFTER consecutiveErrors is incremented.
|
|
21
|
+
*
|
|
22
|
+
* @param effectiveCategory — must be "transient" for the advisory to fire.
|
|
23
|
+
* @param runtime — the live MegaRuntime (mutated on advisory fire).
|
|
24
|
+
* @param pi — pi ExtensionAPI for safeSendUserMessage.
|
|
25
|
+
* @param config — providerOutageAdviseThreshold config.
|
|
26
|
+
* @param detail — R11: optional signal + rawText for forensics.
|
|
27
|
+
*/
|
|
28
|
+
export async function maybeSendProviderOutageAdvisory(effectiveCategory, runtime, pi, config, detail) {
|
|
29
|
+
if (effectiveCategory !== "transient")
|
|
30
|
+
return;
|
|
31
|
+
if (config.providerOutageAdviseThreshold <= 0)
|
|
32
|
+
return;
|
|
33
|
+
if (runtime.rt.providerOutageAdvised)
|
|
34
|
+
return;
|
|
35
|
+
if (runtime.rt.consecutiveErrors < config.providerOutageAdviseThreshold)
|
|
36
|
+
return;
|
|
37
|
+
runtime.rt.providerOutageAdvised = true;
|
|
38
|
+
runtime.dashboard.event("provider_outage_advised", {
|
|
39
|
+
consecutiveErrors: runtime.rt.consecutiveErrors,
|
|
40
|
+
turnIndex: runtime.currentTurn,
|
|
41
|
+
sessionId: runtime.rt.sessionId,
|
|
42
|
+
...(detail?.signal !== undefined ? { signal: detail.signal } : {}),
|
|
43
|
+
...(detail?.rawText !== undefined ? { rawText: detail.rawText } : {}),
|
|
44
|
+
});
|
|
45
|
+
runtime.logger.info("provider-outage-advised", {
|
|
46
|
+
sessionId: runtime.rt.sessionId,
|
|
47
|
+
consecutiveErrors: runtime.rt.consecutiveErrors,
|
|
48
|
+
turnIndex: runtime.currentTurn,
|
|
49
|
+
...(detail?.signal !== undefined ? { signal: detail.signal } : {}),
|
|
50
|
+
...(detail?.rawText !== undefined ? { rawText: detail.rawText } : {}),
|
|
51
|
+
});
|
|
52
|
+
await safeSendUserMessage(pi, `[mega-compact] the provider is having issues (${runtime.rt.consecutiveErrors} consecutive failures — timeouts/5xx/rate-limits). Retries are bounded and continue automatically; your context is fine — do NOT clear or reset it. Work resumes as soon as the provider recovers.`);
|
|
53
|
+
}
|