pi-mega-compact 0.21.10 → 0.21.12

package/README.md

@@ -7,6 +7,7 @@ A local-first context compressor for the [pi coding agent](https://github.com/ea
 ## Features
 
 - **Auto-compaction** — the store watches context pressure and compacts quietly in the background. You'll notice when a long session just stays long while the token gauge rests comfortably far from the ceiling.
+- **Small-context models are the point** — models with 32k windows (GLM-4.7 etc.) are a first-class case, not an afterthought. The gate accounts for the provider's full declared output reserve, and the live-trim budgets the tail so `input + reserve + margin <= window` — no truncation loops at the overflow edge. Token accounting counts everything the provider actually receives (thinking blocks, tool-call arguments included), not just visible text.
 - **Two-layer compaction** — every LLM call sees a live trim of the context window, and every trim is checkpointed to SQLite so a crash or a `/clear` never loses the work.
 - **Semantic dedup, three layers deep** — exact hash (L0) -> MinHash/LSH (L1) -> cosine over trigram embeddings (L2). The dedup audit log records per-tier decisions with similarity scores for tuning.
 - **RAPTOR memory hierarchy** — decisions you made an hour ago don't scroll off; they get packed up as hierarchical checkpoints and re-inlined the moment your next session asks for them. Multi-level retrieval (leaves + summary clusters) is on by default. Since v0.11.10, RAPTOR tree updates are incremental (no full rebuild) — enabled by default.
@@ -94,6 +95,11 @@ Set env vars before starting pi. Defaults are in `src/config/dedup.ts`.
 | `MEGACOMPACT_NEW_UI` | `true` | Use the new Tailwind/shadcn dashboard shell |
 | `MEGACOMPACT_COST_API_ENABLED` | `false` | Opt-in: fetch model pricing from an external API (PREVENT-PI-004 applies to defaults; opt-in features are exempt). Enriches dashboard cost data for models not in the local pricing table |
 | `MEGACOMPACT_COST_API_URL` | _(unset)_ | OpenRouter-compatible model pricing endpoint (e.g. `https://openrouter.ai/api/v1/models`). Only contacted when `MEGACOMPACT_COST_API_ENABLED=true` |
+| `MEGACOMPACT_OVERFLOW_HEADROOM` | `true` | Fire compaction before `input + output reserve + margin` exceeds the window (prevents provider 400s on small-context models) |
+| `MEGACOMPACT_OUTPUT_RESERVE_PCT` | `0.30` | Fallback output reserve as a fraction of the window when the model's declared maxTokens is missing or implausible |
+| `MEGACOMPACT_OUTPUT_ERROR_COMPACT` | `true` | One-shot force-compact when a response truncates mid-output (`stopReason: length`) |
+| `MEGACOMPACT_WIRE_OVERHEAD` | `true` | Add the provider's invisible request overhead H (system prompt + tool definitions + extension systemPrompt prepends — never in the stored transcript) back into the token estimate for the headroom gate and tail cap; H is a per-model EMA of observed wire samples, else `MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT` × window. Closes the small-context-model 400 loop (attempt #9). OFF = byte-identical v0.21.11 |
+| `MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT` | `0.15` | Fraction of the context window used as the overhead H when no wire sample has been observed yet for the model (clamped 0–0.85). Percent-based: identical math at every window size |
 
 Full config reference: [`docs/CONFIGURATION.md`](docs/CONFIGURATION.md)
 
@@ -116,7 +122,7 @@ Detailed architecture: [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)
 
 ```bash
 npm run build     # TypeScript compile
-npm test          # Build + 1197 tests
+npm test          # Build + 4400+ tests
 npm run lint      # Type check + guardrails scan
 ```
 

package/dist/extensions/dashboard-server/routes-rag-settings-compaction.js

@@ -38,5 +38,7 @@ export const COMPACTION_SETTINGS = {
         boolDirect("MEGACOMPACT_OUTPUT_ERROR_COMPACT", "Output-Error Compact", "When a model response is truncated mid-output (stopReason: 'length'), trip a one-shot forced compaction to free input headroom. Closes the small-context deadlock where the model truncates below the input threshold.", true),
         boolDirect("MEGACOMPACT_OVERFLOW_HEADROOM", "Overflow Headroom Gate", "Fire compaction BEFORE the request overflows the model window — when input tokens + the output reserve + safety margin would exceed the context window — instead of waiting for the percent fire point (which judges only INPUT and never trips on small-window models whose output budget is a large fraction of the window). Percent-based: the reserve scales with the model's own window, so the math holds at every window size (32k…5M). OFF disables this pre-fire check (the gate reverts to input-only judgment); the pair-safe tail-cap hardenings are unconditional safety fixes and remain active.", true),
         num("MEGACOMPACT_OUTPUT_RESERVE_PCT", "Output Reserve %", "FALLBACK output reserve as a fraction of the context window, used only when the model's declared maxTokens is absent or implausible (0, or a models.json sentinel like 1e9/1e38, or >= the window). When maxTokens IS plausible the declared value wins — vLLM-style backends reserve the FULL declared maxTokens. Default 0.30 (30%), clamped 0.10–0.95.", 0.3, 0.1, 0.95),
+        boolDirect("MEGACOMPACT_WIRE_OVERHEAD", "Invisible-Overhead Calibration", "Add the provider's fixed request overhead H (system prompt + tool definitions + extension systemPrompt prepends — everything pi adds at request time that NEVER appears in the stored transcript) back into the token estimate for the headroom gate + tail cap. H is a per-model EMA of observed wire samples, else the Wire-Overhead Default fraction of the window. Closes the 32k overflow loop (attempt #9). OFF = byte-identical v0.21.11.", true),
+        num("MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT", "Wire-Overhead Default %", "Fraction of the context window used as the overhead H when no wire sample has been observed yet for the model. Once a sample lands, the per-model EMA wins. Clamped 0–0.85, default 0.15 (15%). Percent-based: identical math at every window size.", 0.15, 0, 0.85),
     ],
 };

package/dist/extensions/mega-config.js

@@ -259,6 +259,16 @@ export function loadConfig() {
         contextHealthOutputQuality: envBool("MEGACOMPACT_CONTEXT_HEALTH_OUTPUT_QUALITY", true),
         contextHealthCachePoison: envBool("MEGACOMPACT_CONTEXT_HEALTH_CACHE_POISON", true),
         contextHealthMitigate: envBool("MEGACOMPACT_CONTEXT_HEALTH_MITIGATE", false),
+        // v0.21.12: invisible-overhead calibration — add the provider's fixed
+        // request overhead H (system+tools+extension prepends, never in the
+        // transcript) back into the token estimate for the headroom gate / tail
+        // cap. H is an EMA of observed wire samples per model, else
+        // wireOverheadDefaultPct × window. Default ON; OFF = byte-identical
+        // v0.21.11 (every H term is 0). Closes attempt #9 of the 32k overflow loop.
+        wireOverhead: envBool("MEGACOMPACT_WIRE_OVERHEAD", true),
+        // v0.21.12: fallback H as a fraction of the window when no EMA sample
+        // exists yet. Clamped [0, 0.85]; default 0.15. Percent-based.
+        wireOverheadDefaultPct: clamp(envFlag("MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT", 0.15), 0, 0.85),
         // D.1: env-overridable recompact delta (minimum context growth % before
         // re-compacting instead of replaying the cached live trim). Default 50.
         recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),

package/dist/extensions/mega-events/context-handler/headroom.js

@@ -1,5 +1,60 @@
-import { estimateBlockTokens, estimateMessageTokens } from "../../../src/tokens.js";
+import { estimateBlockTokens } from "../../../src/tokens.js";
 import { messageContentText } from "./messageText.js";
+/**
+ * Full-surface AgentMessage token estimate for BUDGET arithmetic (tail cap).
+ *
+ * convertToLlm (pi dist/core/messages.js) ships assistant/toolResult messages
+ * VERBATIM — every content block goes over the wire: text, thinking, toolCall
+ * (name + full `arguments` JSON), toolResult output, role wrappers. The text
+ * extractor (messageContentText) is lossy-on-purpose for analytics, and using
+ * it here made a GLM-4.7-style assistant message with ~11.6k bytes of toolCall
+ * arguments register as ~77 tokens — a 30k-token tail passed an 11.9k budget,
+ * the model overflowed, and pi's one-shot compact-and-retry failed
+ * ("Context overflow recovery failed", 2026-08-20 incident).
+ *
+ * Counts every byte the provider actually receives. Still a heuristic (len/4
+ * + 1 per block, like estimateBlockTokens) — just no longer lossy. Never
+ * throws: unknown block shapes fall back to their JSON serialization length,
+ * and a non-array/string content is counted as its serialization.
+ */
+export function estimateAgentMessageBudgetTokens(m) {
+    try {
+        const c = m.content;
+        let bytes = 0;
+        if (typeof c === "string") {
+            bytes += c.length;
+        }
+        else if (Array.isArray(c)) {
+            for (const b of c) {
+                if (b == null || typeof b !== "object")
+                    continue;
+                const o = b;
+                if (typeof o.text === "string")
+                    bytes += o.text.length;
+                if (typeof o.thinking === "string")
+                    bytes += o.thinking.length;
+                if (typeof o.name === "string")
+                    bytes += o.name.length;
+                if (o.arguments != null)
+                    bytes += JSON.stringify(o.arguments).length;
+                if (typeof o.output === "string")
+                    bytes += o.output.length;
+                // Per-block envelope overhead (role/type markers), matching the
+                // len/4+1 block accounting in estimateBlockTokens.
+                bytes += 4;
+            }
+        }
+        else if (c != null) {
+            bytes += JSON.stringify(c).length;
+        }
+        return estimateBlockTokens(" ".repeat(Math.max(0, bytes)));
+    }
+    catch {
+        // non-fatal: fall back to the legacy text-only estimate rather than
+        // disable the cap on a pathological message.
+        return estimateBlockTokens(messageContentText(m));
+    }
+}
 /**
  * The model's declared maxTokens is only trusted as the output budget when it
  * is plausible. models.json carries sentinel junk for some entries (1e9,
@@ -84,14 +139,21 @@ export function applyTailCap(opts) {
     // <= 95% of the window) plus margin + summary can still exceed the window
     // on tiny summaries-free edges; the floor keeps the cap alive with a small
     // positive budget instead of disabling it (pre-v0.21.9 behavior).
-    const budget = Math.max(1, ctxWindow - reserveTokens - safetyMargin - Math.max(0, summaryTokens));
+    // v0.21.12: also subtract the invisible overhead H so the cap bounds the
+    // ACTUAL wire prompt (messages + H), breaking the 400 loop when the estimate
+    // undercounts by the system-prompt/tool-definition overhead.
+    const budget = Math.max(1, ctxWindow -
+        reserveTokens -
+        safetyMargin -
+        Math.max(0, summaryTokens) -
+        Math.max(0, opts.overheadTokens ?? 0));
     let start = 0;
     let tailTokens = 0;
     for (let i = recentRaw.length - 1; i >= 0; i--) {
         tailTokens +=
             msgTokens != null
                 ? Math.max(0, msgTokens[i])
-                : estimateMessageTokens({ text: messageContentText(recentRaw[i]) });
+                : estimateAgentMessageBudgetTokens(recentRaw[i]);
         if (tailTokens > budget) {
             // Keep from i+1 onward; never drop below the FINAL message.
             start = Math.min(i + 1, recentRaw.length - 1);
@@ -119,10 +181,11 @@ export function applyTailCap(opts) {
 export function recapReplayedTail(opts) {
     return applyTailCap({
         recentRaw: opts.recentRaw,
-        summaryTokens: estimateBlockTokens(messageContentText(opts.summaryAgentMsg)),
+        summaryTokens: estimateAgentMessageBudgetTokens(opts.summaryAgentMsg),
         ctxWindow: opts.ctxWindow,
         maxOutputTokens: opts.maxOutputTokens,
         outputReservePct: opts.outputReservePct,
         safetyMarginPct: opts.safetyMarginPct,
+        overheadTokens: opts.overheadTokens ?? 0,
     });
 }

package/dist/extensions/mega-events/context-handler/liveTrim.js

@@ -6,7 +6,7 @@ import { applyTailCap } from "./headroom.js";
  * call. Returns the tailed view, or undefined when no trim is safe this call.
  */
 export function buildLiveTrimView(runtime, config, ctx, opts) {
-    const { messages, view, pct, currentTokens, usageTokens, pressure, ran, perModelThreshold, tailResult, } = opts;
+    const { messages, view, pct, currentTokens, usageTokens, pressure, ran, perModelThreshold, tailResult, overheadTokens = 0, } = opts;
     // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
     // Non-destructive: pi keeps the real transcript; only this LLM call sees the
     // trimmed window. We compute the cut on the engine view (pure, tested) then
@@ -52,6 +52,37 @@ export function buildLiveTrimView(runtime, config, ctx, opts) {
                 anchorUserMessages,
                 criticalOver: (pct ?? 0) >= 90,
             });
+            // v0.21.12: CAP THE SKIP PATH. When computeLiveTrimCut returns null
+            // (anchor floor blocked cutting a fat recent tool pair, or the
+            // criticalOver hatch stayed closed because estimated pressure ≈80%),
+            // the pre-v0.21.12 code shipped the RAW untrimmed view — which, with
+            // the invisible overhead H uncounted, overflowed the window → 400
+            // again, forever (the entire v0.21.11 blind spot). This is the
+            // invariant "the trim path never ships a view the budget wouldn't
+            // allow": even when we cannot summarize, we still front-drop
+            // OLDEST messages until the RAW tail + overhead fits the budget, so
+            // the model is never fed a prompt that exceeds its window. Flag OFF
+            // ⇒ byte-identical to v0.21.11 (return raw, no cap).
+            if (config.wireOverhead && runtime.lastCtxWindow > 0) {
+                const { recent, dropped } = applyTailCap({
+                    recentRaw: messages,
+                    summaryTokens: 0,
+                    ctxWindow: runtime.lastCtxWindow,
+                    maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
+                    outputReservePct: config.outputReservePct,
+                    safetyMarginPct: perModelThreshold.safetyMarginPct,
+                    overheadTokens,
+                });
+                if (dropped > 0) {
+                    runtime.diagCtxSkipCapped++;
+                    runtime.logger.info("skip_cap_applied", {
+                        sessionId: runtime.rt.sessionId,
+                        dropped,
+                        ctxWindow: runtime.lastCtxWindow,
+                    });
+                    return tailResult(recent) ?? { messages: recent };
+                }
+            }
             return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
         }
         const summaryMsg = liveTrimSummaryMessage({
@@ -98,6 +129,7 @@ export function buildLiveTrimView(runtime, config, ctx, opts) {
             maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
             outputReservePct: config.outputReservePct,
             safetyMarginPct: modelThreshold.safetyMarginPct,
+            overheadTokens,
         });
         if (dropped > 0) {
             runtime.logger.warn("live-trim-tail-cap", {

package/dist/extensions/mega-events/context-handler/pipelineRun.js

@@ -85,6 +85,7 @@ export function invokePipeline(pi, runtime, config, ctx, opts) {
                 maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
                 outputReservePct: config.outputReservePct,
                 safetyMarginPct: runtime.trimCache.safetyMarginPct,
+                overheadTokens: opts.overheadTokens ?? 0,
             });
             runtime.diagLiveTrimFires++;
             runtime.diagLiveTrimReplays++;

package/dist/extensions/mega-events/context-handler/wireTruth.js

@@ -0,0 +1,128 @@
+/**
+ * context-handler/wireTruth.ts — invisible-overhead calibration + wire-truth parse.
+ *
+ * Attempt #9 on the small-context-model overflow loop (2026-08-19 incident,
+ * 8 prior attempts). ROOT CAUSE: pi adds a FIXED OVERHEAD H at request time
+ * (system prompt + tool definitions + extension systemPrompt prepends) that
+ * NEVER appears in the stored transcript. Neither pi's estimateContextTokens nor
+ * our estimateSessionTokens/applyTailCap count H. So when the provider 400s with
+ * "request (39048 tokens) exceeds the available context size (32768 tokens)" we
+ * have been estimating ~18–26k and judging headroom against that — undercounting
+ * by ~50%, so the gate never trips correctly and a RAW uncapped view ships → 400
+ * again, forever (pi's one-shot overflow recovery only resets on user
+ * message_start, so auto-retries burn it permanently).
+ *
+ * Two mechanisms here:
+ *  - parseWireTruth: regex the provider's 400 text into ground-truth
+ *    request/available token counts. Pure + unit-tested.
+ *  - Per-model overhead EMA: calibrate H from observed samples so the gate and
+ *    tail cap can ADD it back. Persisted in the SQLite meta table (copy the
+ *    thrashGuard.ts pattern) so it survives restarts and travels with the repo.
+ *
+ * Non-fatal EVERYWHERE: every store read/write is best-effort, swallowed on
+ * failure. Structured JSON logging only (logger + emit dual-sink, mirroring
+ * armThrashGuard's 3WF-5 pattern). No console.*, no network, no mocks.
+ */
+import { getMetaNumber, setMetaNumber } from "../../../src/store/sqlite.js";
+/** EMA smoothing factor for the overhead calibration (fixed, no count needed). */
+export const OVERHEAD_EMA_ALPHA = 0.4;
+/** Safety clamp: an overhead sample may never exceed this fraction of the window. */
+export const OVERHEAD_CLAMP_FRACTION = 0.85;
+/** Meta key prefix; the model id is appended (meta stores integers only). */
+export const OVERHEAD_META_PREFIX = "wire.overhead_ema.";
+/**
+ * Parse the provider's overflow error text into ground-truth token counts.
+ *
+ * Matches strings like:
+ *   "request (39048 tokens) exceeds the available context size (32768 tokens)"
+ *   "request (39,048 tokens) exceeds the available context size (32,768 tokens)"
+ *
+ * Pure + side-effect free — trivially unit-testable. Returns null when the text
+ * does not match the expected shape (e.g. an unrelated error message).
+ */
+export function parseWireTruth(text) {
+    if (typeof text !== "string" || text.length === 0)
+        return null;
+    const m = text.match(/request\s*\((\d[\d,]*)\s*tokens\)\s*exceeds the available context size\s*\((\d[\d,]*)\s*tokens\)/i);
+    if (!m)
+        return null;
+    const requestTokens = Number(m[1]?.replace(/,/g, ""));
+    const availableTokens = Number(m[2]?.replace(/,/g, ""));
+    if (!Number.isFinite(requestTokens) || !Number.isFinite(availableTokens))
+        return null;
+    if (requestTokens <= 0 || availableTokens <= 0)
+        return null;
+    return { requestTokens, availableTokens };
+}
+/** Build the meta key for a model's overhead EMA (stored ×100, integer). */
+function overheadKey(modelId) {
+    return OVERHEAD_META_PREFIX + modelId;
+}
+/**
+ * Read the calibrated overhead (tokens) for a model. Returns 0 when absent or
+ * unreadable — non-fatal everywhere. When the context window is known, clamps the
+ * value into [0, 0.85 × ctxWindow] as a safety against a runaway EMA.
+ */
+export function readWireOverhead(modelId, stateDir, ctxWindow = 0) {
+    if (!modelId)
+        return 0;
+    try {
+        const stored = getMetaNumber(overheadKey(modelId), stateDir);
+        if (!Number.isFinite(stored) || stored <= 0)
+            return 0;
+        const frac = stored / 100;
+        let overhead = frac * (ctxWindow > 0 ? ctxWindow : 1);
+        if (ctxWindow > 0) {
+            overhead = Math.min(overhead, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+        }
+        return Math.max(0, overhead);
+    }
+    catch {
+        return 0; // non-fatal: never fail the agent loop on a store read
+    }
+}
+/**
+ * Fold a new overhead sample into the per-model EMA and persist it. Returns the
+ * new EMA in tokens (0 when the sample is invalid). First sample initializes the
+ * EMA directly (no warm-up). Best-effort: never throws; a store failure is
+ * swallowed and the in-memory EMA is still returned.
+ *
+ * Clamped into [0, 0.85 × ctxWindow] when ctxWindow is known.
+ */
+export function sampleWireOverhead(modelId, stateDir, sample, ctxWindow = 0) {
+    if (!modelId)
+        return 0;
+    if (!Number.isFinite(sample) || sample <= 0)
+        return 0;
+    let ema = sample;
+    try {
+        const stored = getMetaNumber(overheadKey(modelId), stateDir);
+        if (Number.isFinite(stored) && stored > 0) {
+            // prev is stored as a fraction ×100; convert back to token space
+            // against the window before blending so the EMA is dimensionally
+            // consistent (token space in, token space out).
+            const prevTokens = (stored / 100) * (ctxWindow > 0 ? ctxWindow : sample);
+            ema = OVERHEAD_EMA_ALPHA * sample + (1 - OVERHEAD_EMA_ALPHA) * prevTokens;
+        }
+        // Convert EMA to a fraction of the window (so the meta value is window-
+        // independent + percent-based). When the window is unknown, clamp the
+        // stored fraction to a sane [0, OVERHEAD_CLAMP_FRACTION] band so a stale
+        // 0-window sample cannot poison a later windowed read.
+        let frac = ctxWindow > 0 ? ema / ctxWindow : ema;
+        if (ctxWindow > 0) {
+            frac = Math.min(frac, OVERHEAD_CLAMP_FRACTION);
+        }
+        else {
+            frac = Math.min(Math.max(frac, 0), OVERHEAD_CLAMP_FRACTION);
+        }
+        setMetaNumber(overheadKey(modelId), Math.round(frac * 100), stateDir);
+    }
+    catch {
+        /* non-fatal: best-effort meta write */
+    }
+    // Return the token-space EMA (clamped) regardless of whether the persist landed.
+    let out = ema;
+    if (ctxWindow > 0)
+        out = Math.min(out, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+    return Math.max(0, out);
+}

package/dist/extensions/mega-events/context-handler/wireTruthApply.js

@@ -0,0 +1,121 @@
+import { parseWireTruth, sampleWireOverhead, readWireOverhead } from "./wireTruth.js";
+/**
+ * Calibrate the overhead EMA from a usage-bearing context event. Called once per
+ * event (from the handler) when usage is finite + wireOverhead is ON.
+ * `estimateTokens` MUST be the REAL message-list estimate (the engineView
+ * estimate), never the pct-derived fallback — pi's percent reconstructs
+ * usage.tokens exactly, so using it would force hSample ≈ 0 and erase the EMA
+ * (0.6^5 retained after five healthy turns).
+ */
+export function sampleWireOverheadFromUsage(opts) {
+    const { runtime, config, modelId, usageTokens, resolvedWindow, estimateTokens } = opts;
+    if (!config.wireOverhead)
+        return; // byte-identical when OFF
+    if (modelId === "" || usageTokens == null || !Number.isFinite(usageTokens) || resolvedWindow <= 0)
+        return;
+    const hSample = Math.max(0, usageTokens - estimateTokens);
+    try {
+        sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+    }
+    catch {
+        /* non-fatal */
+    }
+}
+/**
+ * Resolve the invisible overhead H (tokens) the handler feeds to the tail-cap
+ * budget at the fire/replay paths. Returns the calibrated EMA (or the
+ * wireOverheadDefaultPct × window fallback when no sample exists yet). 0 when
+ * wireOverhead is OFF or no model/window is known — byte-identical to v0.21.11.
+ * Computed once so every call site passes the SAME H the gate used.
+ */
+export function resolveOverheadTokens(opts) {
+    const { config, modelId, resolvedWindow, stateDir } = opts;
+    if (!config.wireOverhead || modelId === "" || resolvedWindow <= 0)
+        return 0;
+    const e = readWireOverhead(modelId, stateDir, resolvedWindow);
+    return e > 0 ? e : config.wireOverheadDefaultPct * resolvedWindow;
+}
+/**
+ * v0.21.12: invisible-overhead correction of the ESTIMATE-path token count. When
+ * the token estimate came from the message list (not provider usage) AND
+ * wireOverhead is ON, add H so the gate/thrash/tail-cap see the REAL request
+ * size. H defaults to wireOverheadDefaultPct × window until a wire sample
+ * calibrates it. Flag OFF ⇒ H = 0 (byte-identical to v0.21.11). The provider
+ * usage path is ground truth for the message-list size and is never corrected.
+ */
+export function correctEstimateWithOverhead(opts) {
+    const { config, tokenSource, rawTokens, modelId, resolvedWindow, stateDir } = opts;
+    if (!config.wireOverhead || tokenSource !== "estimate" || resolvedWindow <= 0)
+        return rawTokens;
+    const h = modelId !== "" ? readWireOverhead(modelId, stateDir, resolvedWindow) : 0;
+    const H = h > 0 ? h : config.wireOverheadDefaultPct * resolvedWindow;
+    return rawTokens + H;
+}
+/**
+ * Apply the wire-truth gate override for THIS event. Returns the (possibly
+ * corrected) currentTokens. When the last assistant message is an error
+ * (stopReason "error" or an errorMessage field) and its text matches the
+ * provider's overflow error shape, the parsed requestTokens become
+ * ground-truth currentTokens — even when our estimate reads far below every
+ * threshold. Also feeds the EMA and prefers the parsed availableTokens over
+ * runtime.lastCtxWindow when they differ. No-op (returns currentTokens
+ * unchanged) when wireOverhead is OFF or no error/parse matched.
+ */
+export function applyWireTruthOverride(opts) {
+    const { runtime, config, messages, modelId, resolvedWindow, estimateTokens, currentTokens } = opts;
+    if (!config.wireOverhead)
+        return currentTokens; // byte-identical when OFF
+    try {
+        const last = messages[messages.length - 1];
+        const lastText = typeof last?.errorMessage === "string"
+            ? last.errorMessage
+            : typeof last?.content === "string"
+                ? last.content
+                : Array.isArray(last?.content)
+                    ? (last.content
+                        .map((b) => b?.text ?? "")
+                        .join(""))
+                    : "";
+        const isError = last?.stopReason === "error" || typeof last?.errorMessage === "string";
+        if (!isError || lastText.length === 0)
+            return currentTokens;
+        const parsed = parseWireTruth(lastText);
+        if (parsed == null)
+            return currentTokens;
+        const wireTokens = parsed.requestTokens;
+        runtime.lastCtxTokens = wireTokens;
+        // EMA: the gap between the wire prompt and our message estimate (usage is
+        // absent in the 400 case, so estimateTokens is the true message-list estimate).
+        if (modelId !== "" && resolvedWindow > 0) {
+            const hSample = Math.max(0, wireTokens - estimateTokens);
+            sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+        }
+        // Prefer the provider's own available size for this event's math.
+        if (parsed.availableTokens > 0 &&
+            Math.abs(parsed.availableTokens - runtime.lastCtxWindow) > 0) {
+            runtime.lastCtxWindow = parsed.availableTokens;
+        }
+        runtime.diagCtxWireTruth++;
+        runtime.logger.info("wire_truth_parse", {
+            sessionId: runtime.rt.sessionId,
+            requestTokens: parsed.requestTokens,
+            availableTokens: parsed.availableTokens,
+            estimateTokens,
+        });
+        try {
+            runtime.appendEvent("wire_truth_parse", {
+                requestTokens: parsed.requestTokens,
+                availableTokens: parsed.availableTokens,
+                estimateTokens,
+            });
+        }
+        catch {
+            /* non-fatal */
+        }
+        return wireTokens;
+    }
+    catch {
+        /* non-fatal */
+        return currentTokens;
+    }
+}

package/dist/extensions/mega-events/context-handler.js

@@ -10,6 +10,7 @@ import { markCompactionFired, evaluatePendingReduction, } from "./context-handle
 import { invokePipeline } from "./context-handler/pipelineRun.js";
 import { buildLiveTrimView } from "./context-handler/liveTrim.js";
 import { recapReplayedTail } from "./context-handler/headroom.js";
+import { sampleWireOverheadFromUsage, applyWireTruthOverride, resolveOverheadTokens, correctEstimateWithOverhead, } from "./context-handler/wireTruthApply.js";
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(pi, runtime, config) {
     // ---- Auto-trigger: live trim (compact and continue) + native durable ----
@@ -71,13 +72,44 @@ export function registerContextHandler(pi, runtime, config) {
         // show empty/zero. Compute view lazily only when the fallback is needed
         // (at most one engineView call per context event; when auto is on and
         // usage.tokens is present, view is computed once below via reuse).
-        const viewForFallback = usage?.tokens == null ? runtime.engineView(messages) : null;
-        const currentTokens = usage?.tokens ??
-            (viewForFallback != null
-                ? estimateSessionTokens(viewForFallback)
-                : null) ??
-            Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+        // v0.21.12: build the engineView whenever wireOverhead is ON (not only when
+        // usage is absent) so the EMA sampling + wire-truth blocks measure the REAL
+        // message-list estimate. Flag OFF keeps the v0.21.11 lazy path. Without
+        // this, estimateTokens falls back to the pct-derived value, which
+        // reconstructs usage.tokens exactly → hSample ≈ 0 → EMA trains to nothing.
+        const viewForFallback = usage?.tokens == null || config.wireOverhead ? runtime.engineView(messages) : null;
+        // v0.21.12: track WHICH source produced currentTokens so the invisible-
+        // overhead correction (H) is only applied to the ESTIMATE path. The
+        // provider-reported usage is ground truth for the message-list size; H is
+        // the gap between that and the wire prompt (system+tools+prepends).
+        const tokenSource = usage?.tokens != null
+            ? "usage"
+            : viewForFallback != null
+                ? "estimate"
+                : "pct";
+        const estimateTokens = viewForFallback != null
+            ? estimateSessionTokens(viewForFallback)
+            : Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+        const rawTokens = usage?.tokens ?? estimateTokens;
+        // v0.21.12: invisible-overhead correction of the ESTIMATE-path count (see
+        // wireTruthApply.ts). modelId/resolvedWindow feed the EMA + tail-cap helpers.
+        const modelId = runtime.currentModel?.modelId ?? "";
+        const resolvedWindow = usage?.contextWindow ?? (runtime.currentModel?.contextWindow ?? 0);
+        let currentTokens = correctEstimateWithOverhead({
+            config, tokenSource, rawTokens, modelId, resolvedWindow,
+            stateDir: runtime.currentStateDir,
+        });
         runtime.lastCtxTokens = currentTokens ?? null;
+        // v0.21.12: the invisible overhead H to feed the tail-cap budget at the
+        // fire/replay paths below (resolved once via the wireTruthApply helper so
+        // every call site passes the SAME H the gate used). 0 when the flag is OFF
+        // (byte-identical to v0.21.11) or no model/window is known.
+        const overheadTokens = resolveOverheadTokens({
+            config,
+            modelId,
+            resolvedWindow,
+            stateDir: runtime.currentStateDir,
+        });
         // 3WF-2: consume a pending live-window delta from a prior compaction. If a
         // compaction fired on the previous context event and the live window did
         // not shrink, this arms the ThrashGuard (meta). No-op when none pending.
@@ -103,6 +135,18 @@ export function registerContextHandler(pi, runtime, config) {
             reportedWindow > 0
                 ? reportedWindow
                 : (runtime.currentModel?.contextWindow ?? 0);
+        // v0.21.12: calibrate the invisible overhead H from EVERY context event that
+        // carries finite usage. estimateTokens is the REAL message-list estimate
+        // (engineView is built when wireOverhead is ON), so hSample = the true
+        // overhead (system+tools+prepends), not ≈0. Non-fatal; never throws.
+        sampleWireOverheadFromUsage({
+            runtime,
+            config,
+            modelId,
+            usageTokens: usage?.tokens,
+            resolvedWindow,
+            estimateTokens,
+        });
         runtime.snapshot(ctx);
         if (!config.auto) {
             const tailed = tailResult();
@@ -114,6 +158,21 @@ export function registerContextHandler(pi, runtime, config) {
         // S27 DB-mirror + VC1B ledger append. Runs BEFORE the fast-gate so every
         // message is captured, even if we don't compact this turn. Non-fatal.
         appendMirrorAndLedger(runtime, config, messages);
+        // v0.21.12: WIRED-TRUTH gate override (extracted to wireTruthApply.ts).
+        // When the last assistant message is an error whose text matches the
+        // provider's 400 shape, the parsed requestTokens become ground-truth
+        // currentTokens for THIS event's gate — breaking the 400 loop when the
+        // estimate path undercounts by ~50% (the v0.21.11 blind spot). No-op when
+        // wireOverhead is OFF (byte-identical).
+        currentTokens = applyWireTruthOverride({
+            runtime,
+            config,
+            messages,
+            modelId,
+            resolvedWindow,
+            estimateTokens,
+            currentTokens,
+        });
         // S29 FAST GATE: drive the auto-trigger off the context percent (see
         // gateCheck.ts). Returns a tailed view when the gate does not pass.
         const gate = evaluateGate(runtime, config, { pct, currentTokens, tailResult });
@@ -156,6 +215,7 @@ export function registerContextHandler(pi, runtime, config) {
                     maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
                     outputReservePct: config.outputReservePct,
                     safetyMarginPct: runtime.trimCache.safetyMarginPct,
+                    overheadTokens,
                 });
                 runtime.diagLiveTrimFires++; // trim view returned this call (replay counts as a fire)
                 runtime.diagLiveTrimReplays++;
@@ -203,6 +263,7 @@ export function registerContextHandler(pi, runtime, config) {
             pct,
             currentTokens,
             tailResult,
+            overheadTokens,
         });
         if (pipeline.kind === "return")
             return pipeline.view;
@@ -286,6 +347,7 @@ export function registerContextHandler(pi, runtime, config) {
             ran: pipeline.ran,
             perModelThreshold: gate.perModelThreshold,
             tailResult,
+            overheadTokens,
         });
     });
 }

package/dist/extensions/mega-runtime/runtime-instrumentation.js

@@ -24,6 +24,8 @@ export class RuntimeInstrumentation {
     diagCtxThrown = 0; // live-trim try threw (caught)
     diagCtxOutputErrorTrip = 0; // Phase H: output-error catch tripped a forced compaction
     diagCtxHeadroomTrip = 0; // v0.21.9: output-headroom gate tripped a pre-overflow compaction
+    diagCtxWireTruth = 0; // v0.21.12: a provider 400 text was parsed into ground-truth tokens
+    diagCtxSkipCapped = 0; // v0.21.12: the live-trim skip path was tail-capped to fit the budget
     // Context health instrumentation (v0.12): rolling ring buffers for
     // drift detection + cache poison Layer 1 hash baseline.
     recentTurnEmbeddings = [];

package/extensions/dashboard-server/routes-rag-settings-compaction.ts

@@ -87,5 +87,19 @@ export const COMPACTION_SETTINGS: SettingGroup = {
 			0.1,
 			0.95,
 		),
+		boolDirect(
+			"MEGACOMPACT_WIRE_OVERHEAD",
+			"Invisible-Overhead Calibration",
+			"Add the provider's fixed request overhead H (system prompt + tool definitions + extension systemPrompt prepends — everything pi adds at request time that NEVER appears in the stored transcript) back into the token estimate for the headroom gate + tail cap. H is a per-model EMA of observed wire samples, else the Wire-Overhead Default fraction of the window. Closes the 32k overflow loop (attempt #9). OFF = byte-identical v0.21.11.",
+			true,
+		),
+		num(
+			"MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT",
+			"Wire-Overhead Default %",
+			"Fraction of the context window used as the overhead H when no wire sample has been observed yet for the model. Once a sample lands, the per-model EMA wins. Clamped 0–0.85, default 0.15 (15%). Percent-based: identical math at every window size.",
+			0.15,
+			0,
+			0.85,
+		),
 	],
 };

package/extensions/mega-config-types.ts

@@ -233,4 +233,18 @@ export interface MegaConfig {
 	contextHealthCachePoison: boolean;
 	/** v0.12: KV cache poison mitigation — inject prefix break on mismatch. Default OFF. */
 	contextHealthMitigate: boolean;
+	/** v0.21.12: invisible-overhead calibration. When ON, the handler adds the
+	 *  provider's fixed request overhead H (system prompt + tool definitions +
+	 *  extension systemPrompt prepends — everything pi adds at request time that
+	 *  NEVER appears in the stored transcript) back into the token estimate used
+	 *  by the headroom gate, thrash consult, and tail-cap budget. H is calibrated
+	 *  per-model from observed wire samples (EMA) when available, else a
+	 *  configurable fraction of the window (wireOverheadDefaultPct). OFF =
+	 *  byte-identical to v0.21.11 (every H term is 0; no code path changes). */
+	wireOverhead: boolean;
+	/** v0.21.12: fraction of the context window used as H when no EMA sample
+	 *  exists yet for the model. Clamped [0, 0.85]; default 0.15. Percent-based by
+	 *  design (LTS invariant): identical math at every window size. Only the
+	 *  no-sample fallback — once a wire sample lands, the EMA wins. */
+	wireOverheadDefaultPct: number;
 }