pi-mega-compact 0.21.11 → 0.21.12

package/README.md

@@ -7,6 +7,7 @@ A local-first context compressor for the [pi coding agent](https://github.com/ea
 ## Features
 
 - **Auto-compaction** — the store watches context pressure and compacts quietly in the background. You'll notice when a long session just stays long while the token gauge rests comfortably far from the ceiling.
+- **Small-context models are the point** — models with 32k windows (GLM-4.7 etc.) are a first-class case, not an afterthought. The gate accounts for the provider's full declared output reserve, and the live-trim budgets the tail so `input + reserve + margin <= window` — no truncation loops at the overflow edge. Token accounting counts everything the provider actually receives (thinking blocks, tool-call arguments included), not just visible text.
 - **Two-layer compaction** — every LLM call sees a live trim of the context window, and every trim is checkpointed to SQLite so a crash or a `/clear` never loses the work.
 - **Semantic dedup, three layers deep** — exact hash (L0) -> MinHash/LSH (L1) -> cosine over trigram embeddings (L2). The dedup audit log records per-tier decisions with similarity scores for tuning.
 - **RAPTOR memory hierarchy** — decisions you made an hour ago don't scroll off; they get packed up as hierarchical checkpoints and re-inlined the moment your next session asks for them. Multi-level retrieval (leaves + summary clusters) is on by default. Since v0.11.10, RAPTOR tree updates are incremental (no full rebuild) — enabled by default.
@@ -94,6 +95,11 @@ Set env vars before starting pi. Defaults are in `src/config/dedup.ts`.
 | `MEGACOMPACT_NEW_UI` | `true` | Use the new Tailwind/shadcn dashboard shell |
 | `MEGACOMPACT_COST_API_ENABLED` | `false` | Opt-in: fetch model pricing from an external API (PREVENT-PI-004 applies to defaults; opt-in features are exempt). Enriches dashboard cost data for models not in the local pricing table |
 | `MEGACOMPACT_COST_API_URL` | _(unset)_ | OpenRouter-compatible model pricing endpoint (e.g. `https://openrouter.ai/api/v1/models`). Only contacted when `MEGACOMPACT_COST_API_ENABLED=true` |
+| `MEGACOMPACT_OVERFLOW_HEADROOM` | `true` | Fire compaction before `input + output reserve + margin` exceeds the window (prevents provider 400s on small-context models) |
+| `MEGACOMPACT_OUTPUT_RESERVE_PCT` | `0.30` | Fallback output reserve as a fraction of the window when the model's declared maxTokens is missing or implausible |
+| `MEGACOMPACT_OUTPUT_ERROR_COMPACT` | `true` | One-shot force-compact when a response truncates mid-output (`stopReason: length`) |
+| `MEGACOMPACT_WIRE_OVERHEAD` | `true` | Add the provider's invisible request overhead H (system prompt + tool definitions + extension systemPrompt prepends — never in the stored transcript) back into the token estimate for the headroom gate and tail cap; H is a per-model EMA of observed wire samples, else `MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT` × window. Closes the small-context-model 400 loop (attempt #9). OFF = byte-identical v0.21.11 |
+| `MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT` | `0.15` | Fraction of the context window used as the overhead H when no wire sample has been observed yet for the model (clamped 0–0.85). Percent-based: identical math at every window size |
 
 Full config reference: [`docs/CONFIGURATION.md`](docs/CONFIGURATION.md)
 
@@ -116,7 +122,7 @@ Detailed architecture: [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)
 
 ```bash
 npm run build     # TypeScript compile
-npm test          # Build + 1197 tests
+npm test          # Build + 4400+ tests
 npm run lint      # Type check + guardrails scan
 ```
 

package/dist/extensions/dashboard-server/routes-rag-settings-compaction.js

@@ -38,5 +38,7 @@ export const COMPACTION_SETTINGS = {
         boolDirect("MEGACOMPACT_OUTPUT_ERROR_COMPACT", "Output-Error Compact", "When a model response is truncated mid-output (stopReason: 'length'), trip a one-shot forced compaction to free input headroom. Closes the small-context deadlock where the model truncates below the input threshold.", true),
         boolDirect("MEGACOMPACT_OVERFLOW_HEADROOM", "Overflow Headroom Gate", "Fire compaction BEFORE the request overflows the model window — when input tokens + the output reserve + safety margin would exceed the context window — instead of waiting for the percent fire point (which judges only INPUT and never trips on small-window models whose output budget is a large fraction of the window). Percent-based: the reserve scales with the model's own window, so the math holds at every window size (32k…5M). OFF disables this pre-fire check (the gate reverts to input-only judgment); the pair-safe tail-cap hardenings are unconditional safety fixes and remain active.", true),
         num("MEGACOMPACT_OUTPUT_RESERVE_PCT", "Output Reserve %", "FALLBACK output reserve as a fraction of the context window, used only when the model's declared maxTokens is absent or implausible (0, or a models.json sentinel like 1e9/1e38, or >= the window). When maxTokens IS plausible the declared value wins — vLLM-style backends reserve the FULL declared maxTokens. Default 0.30 (30%), clamped 0.10–0.95.", 0.3, 0.1, 0.95),
+        boolDirect("MEGACOMPACT_WIRE_OVERHEAD", "Invisible-Overhead Calibration", "Add the provider's fixed request overhead H (system prompt + tool definitions + extension systemPrompt prepends — everything pi adds at request time that NEVER appears in the stored transcript) back into the token estimate for the headroom gate + tail cap. H is a per-model EMA of observed wire samples, else the Wire-Overhead Default fraction of the window. Closes the 32k overflow loop (attempt #9). OFF = byte-identical v0.21.11.", true),
+        num("MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT", "Wire-Overhead Default %", "Fraction of the context window used as the overhead H when no wire sample has been observed yet for the model. Once a sample lands, the per-model EMA wins. Clamped 0–0.85, default 0.15 (15%). Percent-based: identical math at every window size.", 0.15, 0, 0.85),
     ],
 };

package/dist/extensions/mega-config.js

@@ -259,6 +259,16 @@ export function loadConfig() {
         contextHealthOutputQuality: envBool("MEGACOMPACT_CONTEXT_HEALTH_OUTPUT_QUALITY", true),
         contextHealthCachePoison: envBool("MEGACOMPACT_CONTEXT_HEALTH_CACHE_POISON", true),
         contextHealthMitigate: envBool("MEGACOMPACT_CONTEXT_HEALTH_MITIGATE", false),
+        // v0.21.12: invisible-overhead calibration — add the provider's fixed
+        // request overhead H (system+tools+extension prepends, never in the
+        // transcript) back into the token estimate for the headroom gate / tail
+        // cap. H is an EMA of observed wire samples per model, else
+        // wireOverheadDefaultPct × window. Default ON; OFF = byte-identical
+        // v0.21.11 (every H term is 0). Closes attempt #9 of the 32k overflow loop.
+        wireOverhead: envBool("MEGACOMPACT_WIRE_OVERHEAD", true),
+        // v0.21.12: fallback H as a fraction of the window when no EMA sample
+        // exists yet. Clamped [0, 0.85]; default 0.15. Percent-based.
+        wireOverheadDefaultPct: clamp(envFlag("MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT", 0.15), 0, 0.85),
         // D.1: env-overridable recompact delta (minimum context growth % before
         // re-compacting instead of replaying the cached live trim). Default 50.
         recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),

package/dist/extensions/mega-events/context-handler/headroom.js

@@ -139,7 +139,14 @@ export function applyTailCap(opts) {
     // <= 95% of the window) plus margin + summary can still exceed the window
     // on tiny summaries-free edges; the floor keeps the cap alive with a small
     // positive budget instead of disabling it (pre-v0.21.9 behavior).
-    const budget = Math.max(1, ctxWindow - reserveTokens - safetyMargin - Math.max(0, summaryTokens));
+    // v0.21.12: also subtract the invisible overhead H so the cap bounds the
+    // ACTUAL wire prompt (messages + H), breaking the 400 loop when the estimate
+    // undercounts by the system-prompt/tool-definition overhead.
+    const budget = Math.max(1, ctxWindow -
+        reserveTokens -
+        safetyMargin -
+        Math.max(0, summaryTokens) -
+        Math.max(0, opts.overheadTokens ?? 0));
     let start = 0;
     let tailTokens = 0;
     for (let i = recentRaw.length - 1; i >= 0; i--) {
@@ -179,5 +186,6 @@ export function recapReplayedTail(opts) {
         maxOutputTokens: opts.maxOutputTokens,
         outputReservePct: opts.outputReservePct,
         safetyMarginPct: opts.safetyMarginPct,
+        overheadTokens: opts.overheadTokens ?? 0,
     });
 }

package/dist/extensions/mega-events/context-handler/liveTrim.js

@@ -6,7 +6,7 @@ import { applyTailCap } from "./headroom.js";
  * call. Returns the tailed view, or undefined when no trim is safe this call.
  */
 export function buildLiveTrimView(runtime, config, ctx, opts) {
-    const { messages, view, pct, currentTokens, usageTokens, pressure, ran, perModelThreshold, tailResult, } = opts;
+    const { messages, view, pct, currentTokens, usageTokens, pressure, ran, perModelThreshold, tailResult, overheadTokens = 0, } = opts;
     // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
     // Non-destructive: pi keeps the real transcript; only this LLM call sees the
     // trimmed window. We compute the cut on the engine view (pure, tested) then
@@ -52,6 +52,37 @@ export function buildLiveTrimView(runtime, config, ctx, opts) {
                 anchorUserMessages,
                 criticalOver: (pct ?? 0) >= 90,
             });
+            // v0.21.12: CAP THE SKIP PATH. When computeLiveTrimCut returns null
+            // (anchor floor blocked cutting a fat recent tool pair, or the
+            // criticalOver hatch stayed closed because estimated pressure ≈80%),
+            // the pre-v0.21.12 code shipped the RAW untrimmed view — which, with
+            // the invisible overhead H uncounted, overflowed the window → 400
+            // again, forever (the entire v0.21.11 blind spot). This is the
+            // invariant "the trim path never ships a view the budget wouldn't
+            // allow": even when we cannot summarize, we still front-drop
+            // OLDEST messages until the RAW tail + overhead fits the budget, so
+            // the model is never fed a prompt that exceeds its window. Flag OFF
+            // ⇒ byte-identical to v0.21.11 (return raw, no cap).
+            if (config.wireOverhead && runtime.lastCtxWindow > 0) {
+                const { recent, dropped } = applyTailCap({
+                    recentRaw: messages,
+                    summaryTokens: 0,
+                    ctxWindow: runtime.lastCtxWindow,
+                    maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
+                    outputReservePct: config.outputReservePct,
+                    safetyMarginPct: perModelThreshold.safetyMarginPct,
+                    overheadTokens,
+                });
+                if (dropped > 0) {
+                    runtime.diagCtxSkipCapped++;
+                    runtime.logger.info("skip_cap_applied", {
+                        sessionId: runtime.rt.sessionId,
+                        dropped,
+                        ctxWindow: runtime.lastCtxWindow,
+                    });
+                    return tailResult(recent) ?? { messages: recent };
+                }
+            }
             return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
         }
         const summaryMsg = liveTrimSummaryMessage({
@@ -98,6 +129,7 @@ export function buildLiveTrimView(runtime, config, ctx, opts) {
             maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
             outputReservePct: config.outputReservePct,
             safetyMarginPct: modelThreshold.safetyMarginPct,
+            overheadTokens,
         });
         if (dropped > 0) {
             runtime.logger.warn("live-trim-tail-cap", {

package/dist/extensions/mega-events/context-handler/pipelineRun.js

@@ -85,6 +85,7 @@ export function invokePipeline(pi, runtime, config, ctx, opts) {
                 maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
                 outputReservePct: config.outputReservePct,
                 safetyMarginPct: runtime.trimCache.safetyMarginPct,
+                overheadTokens: opts.overheadTokens ?? 0,
             });
             runtime.diagLiveTrimFires++;
             runtime.diagLiveTrimReplays++;

package/dist/extensions/mega-events/context-handler/wireTruth.js

@@ -0,0 +1,128 @@
+/**
+ * context-handler/wireTruth.ts — invisible-overhead calibration + wire-truth parse.
+ *
+ * Attempt #9 on the small-context-model overflow loop (2026-08-19 incident,
+ * 8 prior attempts). ROOT CAUSE: pi adds a FIXED OVERHEAD H at request time
+ * (system prompt + tool definitions + extension systemPrompt prepends) that
+ * NEVER appears in the stored transcript. Neither pi's estimateContextTokens nor
+ * our estimateSessionTokens/applyTailCap count H. So when the provider 400s with
+ * "request (39048 tokens) exceeds the available context size (32768 tokens)" we
+ * have been estimating ~18–26k and judging headroom against that — undercounting
+ * by ~50%, so the gate never trips correctly and a RAW uncapped view ships → 400
+ * again, forever (pi's one-shot overflow recovery only resets on user
+ * message_start, so auto-retries burn it permanently).
+ *
+ * Two mechanisms here:
+ *  - parseWireTruth: regex the provider's 400 text into ground-truth
+ *    request/available token counts. Pure + unit-tested.
+ *  - Per-model overhead EMA: calibrate H from observed samples so the gate and
+ *    tail cap can ADD it back. Persisted in the SQLite meta table (copy the
+ *    thrashGuard.ts pattern) so it survives restarts and travels with the repo.
+ *
+ * Non-fatal EVERYWHERE: every store read/write is best-effort, swallowed on
+ * failure. Structured JSON logging only (logger + emit dual-sink, mirroring
+ * armThrashGuard's 3WF-5 pattern). No console.*, no network, no mocks.
+ */
+import { getMetaNumber, setMetaNumber } from "../../../src/store/sqlite.js";
+/** EMA smoothing factor for the overhead calibration (fixed, no count needed). */
+export const OVERHEAD_EMA_ALPHA = 0.4;
+/** Safety clamp: an overhead sample may never exceed this fraction of the window. */
+export const OVERHEAD_CLAMP_FRACTION = 0.85;
+/** Meta key prefix; the model id is appended (meta stores integers only). */
+export const OVERHEAD_META_PREFIX = "wire.overhead_ema.";
+/**
+ * Parse the provider's overflow error text into ground-truth token counts.
+ *
+ * Matches strings like:
+ *   "request (39048 tokens) exceeds the available context size (32768 tokens)"
+ *   "request (39,048 tokens) exceeds the available context size (32,768 tokens)"
+ *
+ * Pure + side-effect free — trivially unit-testable. Returns null when the text
+ * does not match the expected shape (e.g. an unrelated error message).
+ */
+export function parseWireTruth(text) {
+    if (typeof text !== "string" || text.length === 0)
+        return null;
+    const m = text.match(/request\s*\((\d[\d,]*)\s*tokens\)\s*exceeds the available context size\s*\((\d[\d,]*)\s*tokens\)/i);
+    if (!m)
+        return null;
+    const requestTokens = Number(m[1]?.replace(/,/g, ""));
+    const availableTokens = Number(m[2]?.replace(/,/g, ""));
+    if (!Number.isFinite(requestTokens) || !Number.isFinite(availableTokens))
+        return null;
+    if (requestTokens <= 0 || availableTokens <= 0)
+        return null;
+    return { requestTokens, availableTokens };
+}
+/** Build the meta key for a model's overhead EMA (stored ×100, integer). */
+function overheadKey(modelId) {
+    return OVERHEAD_META_PREFIX + modelId;
+}
+/**
+ * Read the calibrated overhead (tokens) for a model. Returns 0 when absent or
+ * unreadable — non-fatal everywhere. When the context window is known, clamps the
+ * value into [0, 0.85 × ctxWindow] as a safety against a runaway EMA.
+ */
+export function readWireOverhead(modelId, stateDir, ctxWindow = 0) {
+    if (!modelId)
+        return 0;
+    try {
+        const stored = getMetaNumber(overheadKey(modelId), stateDir);
+        if (!Number.isFinite(stored) || stored <= 0)
+            return 0;
+        const frac = stored / 100;
+        let overhead = frac * (ctxWindow > 0 ? ctxWindow : 1);
+        if (ctxWindow > 0) {
+            overhead = Math.min(overhead, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+        }
+        return Math.max(0, overhead);
+    }
+    catch {
+        return 0; // non-fatal: never fail the agent loop on a store read
+    }
+}
+/**
+ * Fold a new overhead sample into the per-model EMA and persist it. Returns the
+ * new EMA in tokens (0 when the sample is invalid). First sample initializes the
+ * EMA directly (no warm-up). Best-effort: never throws; a store failure is
+ * swallowed and the in-memory EMA is still returned.
+ *
+ * Clamped into [0, 0.85 × ctxWindow] when ctxWindow is known.
+ */
+export function sampleWireOverhead(modelId, stateDir, sample, ctxWindow = 0) {
+    if (!modelId)
+        return 0;
+    if (!Number.isFinite(sample) || sample <= 0)
+        return 0;
+    let ema = sample;
+    try {
+        const stored = getMetaNumber(overheadKey(modelId), stateDir);
+        if (Number.isFinite(stored) && stored > 0) {
+            // prev is stored as a fraction ×100; convert back to token space
+            // against the window before blending so the EMA is dimensionally
+            // consistent (token space in, token space out).
+            const prevTokens = (stored / 100) * (ctxWindow > 0 ? ctxWindow : sample);
+            ema = OVERHEAD_EMA_ALPHA * sample + (1 - OVERHEAD_EMA_ALPHA) * prevTokens;
+        }
+        // Convert EMA to a fraction of the window (so the meta value is window-
+        // independent + percent-based). When the window is unknown, clamp the
+        // stored fraction to a sane [0, OVERHEAD_CLAMP_FRACTION] band so a stale
+        // 0-window sample cannot poison a later windowed read.
+        let frac = ctxWindow > 0 ? ema / ctxWindow : ema;
+        if (ctxWindow > 0) {
+            frac = Math.min(frac, OVERHEAD_CLAMP_FRACTION);
+        }
+        else {
+            frac = Math.min(Math.max(frac, 0), OVERHEAD_CLAMP_FRACTION);
+        }
+        setMetaNumber(overheadKey(modelId), Math.round(frac * 100), stateDir);
+    }
+    catch {
+        /* non-fatal: best-effort meta write */
+    }
+    // Return the token-space EMA (clamped) regardless of whether the persist landed.
+    let out = ema;
+    if (ctxWindow > 0)
+        out = Math.min(out, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+    return Math.max(0, out);
+}

package/dist/extensions/mega-events/context-handler/wireTruthApply.js

@@ -0,0 +1,121 @@
+import { parseWireTruth, sampleWireOverhead, readWireOverhead } from "./wireTruth.js";
+/**
+ * Calibrate the overhead EMA from a usage-bearing context event. Called once per
+ * event (from the handler) when usage is finite + wireOverhead is ON.
+ * `estimateTokens` MUST be the REAL message-list estimate (the engineView
+ * estimate), never the pct-derived fallback — pi's percent reconstructs
+ * usage.tokens exactly, so using it would force hSample ≈ 0 and erase the EMA
+ * (0.6^5 retained after five healthy turns).
+ */
+export function sampleWireOverheadFromUsage(opts) {
+    const { runtime, config, modelId, usageTokens, resolvedWindow, estimateTokens } = opts;
+    if (!config.wireOverhead)
+        return; // byte-identical when OFF
+    if (modelId === "" || usageTokens == null || !Number.isFinite(usageTokens) || resolvedWindow <= 0)
+        return;
+    const hSample = Math.max(0, usageTokens - estimateTokens);
+    try {
+        sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+    }
+    catch {
+        /* non-fatal */
+    }
+}
+/**
+ * Resolve the invisible overhead H (tokens) the handler feeds to the tail-cap
+ * budget at the fire/replay paths. Returns the calibrated EMA (or the
+ * wireOverheadDefaultPct × window fallback when no sample exists yet). 0 when
+ * wireOverhead is OFF or no model/window is known — byte-identical to v0.21.11.
+ * Computed once so every call site passes the SAME H the gate used.
+ */
+export function resolveOverheadTokens(opts) {
+    const { config, modelId, resolvedWindow, stateDir } = opts;
+    if (!config.wireOverhead || modelId === "" || resolvedWindow <= 0)
+        return 0;
+    const e = readWireOverhead(modelId, stateDir, resolvedWindow);
+    return e > 0 ? e : config.wireOverheadDefaultPct * resolvedWindow;
+}
+/**
+ * v0.21.12: invisible-overhead correction of the ESTIMATE-path token count. When
+ * the token estimate came from the message list (not provider usage) AND
+ * wireOverhead is ON, add H so the gate/thrash/tail-cap see the REAL request
+ * size. H defaults to wireOverheadDefaultPct × window until a wire sample
+ * calibrates it. Flag OFF ⇒ H = 0 (byte-identical to v0.21.11). The provider
+ * usage path is ground truth for the message-list size and is never corrected.
+ */
+export function correctEstimateWithOverhead(opts) {
+    const { config, tokenSource, rawTokens, modelId, resolvedWindow, stateDir } = opts;
+    if (!config.wireOverhead || tokenSource !== "estimate" || resolvedWindow <= 0)
+        return rawTokens;
+    const h = modelId !== "" ? readWireOverhead(modelId, stateDir, resolvedWindow) : 0;
+    const H = h > 0 ? h : config.wireOverheadDefaultPct * resolvedWindow;
+    return rawTokens + H;
+}
+/**
+ * Apply the wire-truth gate override for THIS event. Returns the (possibly
+ * corrected) currentTokens. When the last assistant message is an error
+ * (stopReason "error" or an errorMessage field) and its text matches the
+ * provider's overflow error shape, the parsed requestTokens become
+ * ground-truth currentTokens — even when our estimate reads far below every
+ * threshold. Also feeds the EMA and prefers the parsed availableTokens over
+ * runtime.lastCtxWindow when they differ. No-op (returns currentTokens
+ * unchanged) when wireOverhead is OFF or no error/parse matched.
+ */
+export function applyWireTruthOverride(opts) {
+    const { runtime, config, messages, modelId, resolvedWindow, estimateTokens, currentTokens } = opts;
+    if (!config.wireOverhead)
+        return currentTokens; // byte-identical when OFF
+    try {
+        const last = messages[messages.length - 1];
+        const lastText = typeof last?.errorMessage === "string"
+            ? last.errorMessage
+            : typeof last?.content === "string"
+                ? last.content
+                : Array.isArray(last?.content)
+                    ? (last.content
+                        .map((b) => b?.text ?? "")
+                        .join(""))
+                    : "";
+        const isError = last?.stopReason === "error" || typeof last?.errorMessage === "string";
+        if (!isError || lastText.length === 0)
+            return currentTokens;
+        const parsed = parseWireTruth(lastText);
+        if (parsed == null)
+            return currentTokens;
+        const wireTokens = parsed.requestTokens;
+        runtime.lastCtxTokens = wireTokens;
+        // EMA: the gap between the wire prompt and our message estimate (usage is
+        // absent in the 400 case, so estimateTokens is the true message-list estimate).
+        if (modelId !== "" && resolvedWindow > 0) {
+            const hSample = Math.max(0, wireTokens - estimateTokens);
+            sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+        }
+        // Prefer the provider's own available size for this event's math.
+        if (parsed.availableTokens > 0 &&
+            Math.abs(parsed.availableTokens - runtime.lastCtxWindow) > 0) {
+            runtime.lastCtxWindow = parsed.availableTokens;
+        }
+        runtime.diagCtxWireTruth++;
+        runtime.logger.info("wire_truth_parse", {
+            sessionId: runtime.rt.sessionId,
+            requestTokens: parsed.requestTokens,
+            availableTokens: parsed.availableTokens,
+            estimateTokens,
+        });
+        try {
+            runtime.appendEvent("wire_truth_parse", {
+                requestTokens: parsed.requestTokens,
+                availableTokens: parsed.availableTokens,
+                estimateTokens,
+            });
+        }
+        catch {
+            /* non-fatal */
+        }
+        return wireTokens;
+    }
+    catch {
+        /* non-fatal */
+        return currentTokens;
+    }
+}

package/dist/extensions/mega-events/context-handler.js

@@ -10,6 +10,7 @@ import { markCompactionFired, evaluatePendingReduction, } from "./context-handle
 import { invokePipeline } from "./context-handler/pipelineRun.js";
 import { buildLiveTrimView } from "./context-handler/liveTrim.js";
 import { recapReplayedTail } from "./context-handler/headroom.js";
+import { sampleWireOverheadFromUsage, applyWireTruthOverride, resolveOverheadTokens, correctEstimateWithOverhead, } from "./context-handler/wireTruthApply.js";
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(pi, runtime, config) {
     // ---- Auto-trigger: live trim (compact and continue) + native durable ----
@@ -71,13 +72,44 @@ export function registerContextHandler(pi, runtime, config) {
         // show empty/zero. Compute view lazily only when the fallback is needed
         // (at most one engineView call per context event; when auto is on and
         // usage.tokens is present, view is computed once below via reuse).
-        const viewForFallback = usage?.tokens == null ? runtime.engineView(messages) : null;
-        const currentTokens = usage?.tokens ??
-            (viewForFallback != null
-                ? estimateSessionTokens(viewForFallback)
-                : null) ??
-            Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+        // v0.21.12: build the engineView whenever wireOverhead is ON (not only when
+        // usage is absent) so the EMA sampling + wire-truth blocks measure the REAL
+        // message-list estimate. Flag OFF keeps the v0.21.11 lazy path. Without
+        // this, estimateTokens falls back to the pct-derived value, which
+        // reconstructs usage.tokens exactly → hSample ≈ 0 → EMA trains to nothing.
+        const viewForFallback = usage?.tokens == null || config.wireOverhead ? runtime.engineView(messages) : null;
+        // v0.21.12: track WHICH source produced currentTokens so the invisible-
+        // overhead correction (H) is only applied to the ESTIMATE path. The
+        // provider-reported usage is ground truth for the message-list size; H is
+        // the gap between that and the wire prompt (system+tools+prepends).
+        const tokenSource = usage?.tokens != null
+            ? "usage"
+            : viewForFallback != null
+                ? "estimate"
+                : "pct";
+        const estimateTokens = viewForFallback != null
+            ? estimateSessionTokens(viewForFallback)
+            : Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+        const rawTokens = usage?.tokens ?? estimateTokens;
+        // v0.21.12: invisible-overhead correction of the ESTIMATE-path count (see
+        // wireTruthApply.ts). modelId/resolvedWindow feed the EMA + tail-cap helpers.
+        const modelId = runtime.currentModel?.modelId ?? "";
+        const resolvedWindow = usage?.contextWindow ?? (runtime.currentModel?.contextWindow ?? 0);
+        let currentTokens = correctEstimateWithOverhead({
+            config, tokenSource, rawTokens, modelId, resolvedWindow,
+            stateDir: runtime.currentStateDir,
+        });
         runtime.lastCtxTokens = currentTokens ?? null;
+        // v0.21.12: the invisible overhead H to feed the tail-cap budget at the
+        // fire/replay paths below (resolved once via the wireTruthApply helper so
+        // every call site passes the SAME H the gate used). 0 when the flag is OFF
+        // (byte-identical to v0.21.11) or no model/window is known.
+        const overheadTokens = resolveOverheadTokens({
+            config,
+            modelId,
+            resolvedWindow,
+            stateDir: runtime.currentStateDir,
+        });
         // 3WF-2: consume a pending live-window delta from a prior compaction. If a
         // compaction fired on the previous context event and the live window did
         // not shrink, this arms the ThrashGuard (meta). No-op when none pending.
@@ -103,6 +135,18 @@ export function registerContextHandler(pi, runtime, config) {
             reportedWindow > 0
                 ? reportedWindow
                 : (runtime.currentModel?.contextWindow ?? 0);
+        // v0.21.12: calibrate the invisible overhead H from EVERY context event that
+        // carries finite usage. estimateTokens is the REAL message-list estimate
+        // (engineView is built when wireOverhead is ON), so hSample = the true
+        // overhead (system+tools+prepends), not ≈0. Non-fatal; never throws.
+        sampleWireOverheadFromUsage({
+            runtime,
+            config,
+            modelId,
+            usageTokens: usage?.tokens,
+            resolvedWindow,
+            estimateTokens,
+        });
         runtime.snapshot(ctx);
         if (!config.auto) {
             const tailed = tailResult();
@@ -114,6 +158,21 @@ export function registerContextHandler(pi, runtime, config) {
         // S27 DB-mirror + VC1B ledger append. Runs BEFORE the fast-gate so every
         // message is captured, even if we don't compact this turn. Non-fatal.
         appendMirrorAndLedger(runtime, config, messages);
+        // v0.21.12: WIRED-TRUTH gate override (extracted to wireTruthApply.ts).
+        // When the last assistant message is an error whose text matches the
+        // provider's 400 shape, the parsed requestTokens become ground-truth
+        // currentTokens for THIS event's gate — breaking the 400 loop when the
+        // estimate path undercounts by ~50% (the v0.21.11 blind spot). No-op when
+        // wireOverhead is OFF (byte-identical).
+        currentTokens = applyWireTruthOverride({
+            runtime,
+            config,
+            messages,
+            modelId,
+            resolvedWindow,
+            estimateTokens,
+            currentTokens,
+        });
         // S29 FAST GATE: drive the auto-trigger off the context percent (see
         // gateCheck.ts). Returns a tailed view when the gate does not pass.
         const gate = evaluateGate(runtime, config, { pct, currentTokens, tailResult });
@@ -156,6 +215,7 @@ export function registerContextHandler(pi, runtime, config) {
                     maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
                     outputReservePct: config.outputReservePct,
                     safetyMarginPct: runtime.trimCache.safetyMarginPct,
+                    overheadTokens,
                 });
                 runtime.diagLiveTrimFires++; // trim view returned this call (replay counts as a fire)
                 runtime.diagLiveTrimReplays++;
@@ -203,6 +263,7 @@ export function registerContextHandler(pi, runtime, config) {
             pct,
             currentTokens,
             tailResult,
+            overheadTokens,
         });
         if (pipeline.kind === "return")
             return pipeline.view;
@@ -286,6 +347,7 @@ export function registerContextHandler(pi, runtime, config) {
             ran: pipeline.ran,
             perModelThreshold: gate.perModelThreshold,
             tailResult,
+            overheadTokens,
         });
     });
 }

package/dist/extensions/mega-runtime/runtime-instrumentation.js

@@ -24,6 +24,8 @@ export class RuntimeInstrumentation {
     diagCtxThrown = 0; // live-trim try threw (caught)
     diagCtxOutputErrorTrip = 0; // Phase H: output-error catch tripped a forced compaction
     diagCtxHeadroomTrip = 0; // v0.21.9: output-headroom gate tripped a pre-overflow compaction
+    diagCtxWireTruth = 0; // v0.21.12: a provider 400 text was parsed into ground-truth tokens
+    diagCtxSkipCapped = 0; // v0.21.12: the live-trim skip path was tail-capped to fit the budget
     // Context health instrumentation (v0.12): rolling ring buffers for
     // drift detection + cache poison Layer 1 hash baseline.
     recentTurnEmbeddings = [];

package/extensions/dashboard-server/routes-rag-settings-compaction.ts

@@ -87,5 +87,19 @@ export const COMPACTION_SETTINGS: SettingGroup = {
 			0.1,
 			0.95,
 		),
+		boolDirect(
+			"MEGACOMPACT_WIRE_OVERHEAD",
+			"Invisible-Overhead Calibration",
+			"Add the provider's fixed request overhead H (system prompt + tool definitions + extension systemPrompt prepends — everything pi adds at request time that NEVER appears in the stored transcript) back into the token estimate for the headroom gate + tail cap. H is a per-model EMA of observed wire samples, else the Wire-Overhead Default fraction of the window. Closes the 32k overflow loop (attempt #9). OFF = byte-identical v0.21.11.",
+			true,
+		),
+		num(
+			"MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT",
+			"Wire-Overhead Default %",
+			"Fraction of the context window used as the overhead H when no wire sample has been observed yet for the model. Once a sample lands, the per-model EMA wins. Clamped 0–0.85, default 0.15 (15%). Percent-based: identical math at every window size.",
+			0.15,
+			0,
+			0.85,
+		),
 	],
 };

package/extensions/mega-config-types.ts

@@ -233,4 +233,18 @@ export interface MegaConfig {
 	contextHealthCachePoison: boolean;
 	/** v0.12: KV cache poison mitigation — inject prefix break on mismatch. Default OFF. */
 	contextHealthMitigate: boolean;
+	/** v0.21.12: invisible-overhead calibration. When ON, the handler adds the
+	 *  provider's fixed request overhead H (system prompt + tool definitions +
+	 *  extension systemPrompt prepends — everything pi adds at request time that
+	 *  NEVER appears in the stored transcript) back into the token estimate used
+	 *  by the headroom gate, thrash consult, and tail-cap budget. H is calibrated
+	 *  per-model from observed wire samples (EMA) when available, else a
+	 *  configurable fraction of the window (wireOverheadDefaultPct). OFF =
+	 *  byte-identical to v0.21.11 (every H term is 0; no code path changes). */
+	wireOverhead: boolean;
+	/** v0.21.12: fraction of the context window used as H when no EMA sample
+	 *  exists yet for the model. Clamped [0, 0.85]; default 0.15. Percent-based by
+	 *  design (LTS invariant): identical math at every window size. Only the
+	 *  no-sample fallback — once a wire sample lands, the EMA wins. */
+	wireOverheadDefaultPct: number;
 }

package/extensions/mega-config.ts

@@ -302,6 +302,16 @@ export function loadConfig(): MegaConfig {
 		contextHealthOutputQuality: envBool("MEGACOMPACT_CONTEXT_HEALTH_OUTPUT_QUALITY", true),
 		contextHealthCachePoison: envBool("MEGACOMPACT_CONTEXT_HEALTH_CACHE_POISON", true),
 		contextHealthMitigate: envBool("MEGACOMPACT_CONTEXT_HEALTH_MITIGATE", false),
+		// v0.21.12: invisible-overhead calibration — add the provider's fixed
+		// request overhead H (system+tools+extension prepends, never in the
+		// transcript) back into the token estimate for the headroom gate / tail
+		// cap. H is an EMA of observed wire samples per model, else
+		// wireOverheadDefaultPct × window. Default ON; OFF = byte-identical
+		// v0.21.11 (every H term is 0). Closes attempt #9 of the 32k overflow loop.
+		wireOverhead: envBool("MEGACOMPACT_WIRE_OVERHEAD", true),
+		// v0.21.12: fallback H as a fraction of the window when no EMA sample
+		// exists yet. Clamped [0, 0.85]; default 0.15. Percent-based.
+		wireOverheadDefaultPct: clamp(envFlag("MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT", 0.15), 0, 0.85),
 		// D.1: env-overridable recompact delta (minimum context growth % before
 		// re-compacting instead of replaying the cached live trim). Default 50.
 		recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),

package/extensions/mega-events/context-handler/headroom.ts

@@ -159,6 +159,15 @@ export function applyTailCap(opts: {
 	 * otherwise each message's tokens are estimated from its text content.
 	 */
 	messageTokens?: readonly number[];
+	/**
+	 * v0.21.12: the provider's invisible overhead H (system prompt + tool
+	 * definitions + extension systemPrompt prepends — everything pi adds at
+	 * request time that NEVER appears in the stored transcript). Subtracted from
+	 * the budget so the cap accounts for the REAL wire prompt, not just the
+	 * counted messages. Default 0 (no H) — identical behavior to v0.21.11 when
+	 * the wireOverhead flag is OFF. Clamped to >= 0.
+	 */
+	overheadTokens?: number;
 }): { recent: AgentMessage[]; dropped: number } {
 	const { recentRaw, summaryTokens, ctxWindow, outputReservePct } = opts;
 	if (ctxWindow <= 0 || recentRaw.length <= 1) {
@@ -180,9 +189,16 @@ export function applyTailCap(opts: {
 	// <= 95% of the window) plus margin + summary can still exceed the window
 	// on tiny summaries-free edges; the floor keeps the cap alive with a small
 	// positive budget instead of disabling it (pre-v0.21.9 behavior).
+	// v0.21.12: also subtract the invisible overhead H so the cap bounds the
+	// ACTUAL wire prompt (messages + H), breaking the 400 loop when the estimate
+	// undercounts by the system-prompt/tool-definition overhead.
 	const budget = Math.max(
 		1,
-		ctxWindow - reserveTokens - safetyMargin - Math.max(0, summaryTokens),
+		ctxWindow -
+			reserveTokens -
+			safetyMargin -
+			Math.max(0, summaryTokens) -
+			Math.max(0, opts.overheadTokens ?? 0),
 	);
 	let start = 0;
 	let tailTokens = 0;
@@ -225,6 +241,8 @@ export function recapReplayedTail(opts: {
 	maxOutputTokens: number;
 	outputReservePct: number;
 	safetyMarginPct: number;
+	/** v0.21.12: invisible overhead H (see applyTailCap). Default 0. */
+	overheadTokens?: number;
 }): { recent: AgentMessage[]; dropped: number } {
 	return applyTailCap({
 		recentRaw: opts.recentRaw,
@@ -233,5 +251,6 @@ export function recapReplayedTail(opts: {
 		maxOutputTokens: opts.maxOutputTokens,
 		outputReservePct: opts.outputReservePct,
 		safetyMarginPct: opts.safetyMarginPct,
+		overheadTokens: opts.overheadTokens ?? 0,
 	});
 }

package/extensions/mega-events/context-handler/liveTrim.ts

@@ -45,6 +45,10 @@ export function buildLiveTrimView(
 		ran: { result: CompactResult };
 		perModelThreshold: { safetyMarginPct: number; firePointPct: number };
 		tailResult: TailResultFn;
+		/** v0.21.12: the provider's invisible overhead H (system+tools+prepends),
+		 *  handler-resolved. Subtracted from the tail-cap budget so the cap bounds
+		 *  the REAL wire prompt. 0 when the flag is OFF (byte-identical v0.21.11). */
+		overheadTokens?: number;
 	},
 ): { messages: AgentMessage[] } | undefined {
 	const {
@@ -57,6 +61,7 @@ export function buildLiveTrimView(
 		ran,
 		perModelThreshold,
 		tailResult,
+		overheadTokens = 0,
 	} = opts;
 
 	// S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
@@ -105,6 +110,37 @@ export function buildLiveTrimView(
 				anchorUserMessages,
 				criticalOver: (pct ?? 0) >= 90,
 			});
+			// v0.21.12: CAP THE SKIP PATH. When computeLiveTrimCut returns null
+			// (anchor floor blocked cutting a fat recent tool pair, or the
+			// criticalOver hatch stayed closed because estimated pressure ≈80%),
+			// the pre-v0.21.12 code shipped the RAW untrimmed view — which, with
+			// the invisible overhead H uncounted, overflowed the window → 400
+			// again, forever (the entire v0.21.11 blind spot). This is the
+			// invariant "the trim path never ships a view the budget wouldn't
+			// allow": even when we cannot summarize, we still front-drop
+			// OLDEST messages until the RAW tail + overhead fits the budget, so
+			// the model is never fed a prompt that exceeds its window. Flag OFF
+			// ⇒ byte-identical to v0.21.11 (return raw, no cap).
+			if (config.wireOverhead && runtime.lastCtxWindow > 0) {
+				const { recent, dropped } = applyTailCap({
+					recentRaw: messages,
+					summaryTokens: 0,
+					ctxWindow: runtime.lastCtxWindow,
+					maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
+					outputReservePct: config.outputReservePct,
+					safetyMarginPct: perModelThreshold.safetyMarginPct,
+					overheadTokens,
+				});
+				if (dropped > 0) {
+					runtime.diagCtxSkipCapped++;
+					runtime.logger.info("skip_cap_applied", {
+						sessionId: runtime.rt.sessionId,
+						dropped,
+						ctxWindow: runtime.lastCtxWindow,
+					});
+					return tailResult(recent) ?? { messages: recent };
+				}
+			}
 			return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
 		}
 		const summaryMsg = liveTrimSummaryMessage({
@@ -152,6 +188,7 @@ export function buildLiveTrimView(
 			maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
 			outputReservePct: config.outputReservePct,
 			safetyMarginPct: modelThreshold.safetyMarginPct,
+			overheadTokens,
 		});
 		if (dropped > 0) {
 			runtime.logger.warn("live-trim-tail-cap", {

package/extensions/mega-events/context-handler/pipelineRun.ts

@@ -48,6 +48,7 @@ export function invokePipeline(
 		pct: number | null | undefined;
 		currentTokens: number;
 		tailResult: TailResultFn;
+		overheadTokens?: number;
 	},
 ): PipelineOutcome {
 	// VC5C: emit the rollout decision per compact event (observability seam).
@@ -133,6 +134,7 @@ export function invokePipeline(
 				maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
 				outputReservePct: config.outputReservePct,
 				safetyMarginPct: runtime.trimCache.safetyMarginPct,
+				overheadTokens: opts.overheadTokens ?? 0,
 			});
 			runtime.diagLiveTrimFires++;
 			runtime.diagLiveTrimReplays++;

package/extensions/mega-events/context-handler/wireTruth.ts

@@ -0,0 +1,132 @@
+/**
+ * context-handler/wireTruth.ts — invisible-overhead calibration + wire-truth parse.
+ *
+ * Attempt #9 on the small-context-model overflow loop (2026-08-19 incident,
+ * 8 prior attempts). ROOT CAUSE: pi adds a FIXED OVERHEAD H at request time
+ * (system prompt + tool definitions + extension systemPrompt prepends) that
+ * NEVER appears in the stored transcript. Neither pi's estimateContextTokens nor
+ * our estimateSessionTokens/applyTailCap count H. So when the provider 400s with
+ * "request (39048 tokens) exceeds the available context size (32768 tokens)" we
+ * have been estimating ~18–26k and judging headroom against that — undercounting
+ * by ~50%, so the gate never trips correctly and a RAW uncapped view ships → 400
+ * again, forever (pi's one-shot overflow recovery only resets on user
+ * message_start, so auto-retries burn it permanently).
+ *
+ * Two mechanisms here:
+ *  - parseWireTruth: regex the provider's 400 text into ground-truth
+ *    request/available token counts. Pure + unit-tested.
+ *  - Per-model overhead EMA: calibrate H from observed samples so the gate and
+ *    tail cap can ADD it back. Persisted in the SQLite meta table (copy the
+ *    thrashGuard.ts pattern) so it survives restarts and travels with the repo.
+ *
+ * Non-fatal EVERYWHERE: every store read/write is best-effort, swallowed on
+ * failure. Structured JSON logging only (logger + emit dual-sink, mirroring
+ * armThrashGuard's 3WF-5 pattern). No console.*, no network, no mocks.
+ */
+import { getMetaNumber, setMetaNumber } from "../../../src/store/sqlite.js";
+
+/** EMA smoothing factor for the overhead calibration (fixed, no count needed). */
+export const OVERHEAD_EMA_ALPHA = 0.4;
+
+/** Safety clamp: an overhead sample may never exceed this fraction of the window. */
+export const OVERHEAD_CLAMP_FRACTION = 0.85;
+
+/** Meta key prefix; the model id is appended (meta stores integers only). */
+export const OVERHEAD_META_PREFIX = "wire.overhead_ema.";
+
+/**
+ * Parse the provider's overflow error text into ground-truth token counts.
+ *
+ * Matches strings like:
+ *   "request (39048 tokens) exceeds the available context size (32768 tokens)"
+ *   "request (39,048 tokens) exceeds the available context size (32,768 tokens)"
+ *
+ * Pure + side-effect free — trivially unit-testable. Returns null when the text
+ * does not match the expected shape (e.g. an unrelated error message).
+ */
+export function parseWireTruth(
+	text: string,
+): { requestTokens: number; availableTokens: number } | null {
+	if (typeof text !== "string" || text.length === 0) return null;
+	const m = text.match(
+		/request\s*\((\d[\d,]*)\s*tokens\)\s*exceeds the available context size\s*\((\d[\d,]*)\s*tokens\)/i,
+	);
+	if (!m) return null;
+	const requestTokens = Number(m[1]?.replace(/,/g, ""));
+	const availableTokens = Number(m[2]?.replace(/,/g, ""));
+	if (!Number.isFinite(requestTokens) || !Number.isFinite(availableTokens)) return null;
+	if (requestTokens <= 0 || availableTokens <= 0) return null;
+	return { requestTokens, availableTokens };
+}
+
+/** Build the meta key for a model's overhead EMA (stored ×100, integer). */
+function overheadKey(modelId: string): string {
+	return OVERHEAD_META_PREFIX + modelId;
+}
+
+/**
+ * Read the calibrated overhead (tokens) for a model. Returns 0 when absent or
+ * unreadable — non-fatal everywhere. When the context window is known, clamps the
+ * value into [0, 0.85 × ctxWindow] as a safety against a runaway EMA.
+ */
+export function readWireOverhead(modelId: string, stateDir: string, ctxWindow = 0): number {
+	if (!modelId) return 0;
+	try {
+		const stored = getMetaNumber(overheadKey(modelId), stateDir);
+		if (!Number.isFinite(stored) || stored <= 0) return 0;
+		const frac = stored / 100;
+		let overhead = frac * (ctxWindow > 0 ? ctxWindow : 1);
+		if (ctxWindow > 0) {
+			overhead = Math.min(overhead, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+		}
+		return Math.max(0, overhead);
+	} catch {
+		return 0; // non-fatal: never fail the agent loop on a store read
+	}
+}
+
+/**
+ * Fold a new overhead sample into the per-model EMA and persist it. Returns the
+ * new EMA in tokens (0 when the sample is invalid). First sample initializes the
+ * EMA directly (no warm-up). Best-effort: never throws; a store failure is
+ * swallowed and the in-memory EMA is still returned.
+ *
+ * Clamped into [0, 0.85 × ctxWindow] when ctxWindow is known.
+ */
+export function sampleWireOverhead(
+	modelId: string,
+	stateDir: string,
+	sample: number,
+	ctxWindow = 0,
+): number {
+	if (!modelId) return 0;
+	if (!Number.isFinite(sample) || sample <= 0) return 0;
+	let ema = sample;
+	try {
+		const stored = getMetaNumber(overheadKey(modelId), stateDir);
+		if (Number.isFinite(stored) && stored > 0) {
+			// prev is stored as a fraction ×100; convert back to token space
+			// against the window before blending so the EMA is dimensionally
+			// consistent (token space in, token space out).
+			const prevTokens = (stored / 100) * (ctxWindow > 0 ? ctxWindow : sample);
+			ema = OVERHEAD_EMA_ALPHA * sample + (1 - OVERHEAD_EMA_ALPHA) * prevTokens;
+		}
+		// Convert EMA to a fraction of the window (so the meta value is window-
+		// independent + percent-based). When the window is unknown, clamp the
+		// stored fraction to a sane [0, OVERHEAD_CLAMP_FRACTION] band so a stale
+		// 0-window sample cannot poison a later windowed read.
+		let frac = ctxWindow > 0 ? ema / ctxWindow : ema;
+		if (ctxWindow > 0) {
+			frac = Math.min(frac, OVERHEAD_CLAMP_FRACTION);
+		} else {
+			frac = Math.min(Math.max(frac, 0), OVERHEAD_CLAMP_FRACTION);
+		}
+		setMetaNumber(overheadKey(modelId), Math.round(frac * 100), stateDir);
+	} catch {
+		/* non-fatal: best-effort meta write */
+	}
+	// Return the token-space EMA (clamped) regardless of whether the persist landed.
+	let out = ema;
+	if (ctxWindow > 0) out = Math.min(out, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+	return Math.max(0, out);
+}

package/extensions/mega-events/context-handler/wireTruthApply.ts

@@ -0,0 +1,168 @@
+/**
+ * context-handler/wireTruthApply.ts — v0.21.12 wired-overhead runtime seam.
+ *
+ * Delegate-shell sibling extracted from context-handler.ts (extensions/ 400-line
+ * soft limit). Holds the two wire-overhead call-site blocks that don't fit the
+ * handler's budget:
+ *  - sampleWireOverheadFromUsage: calibrate the per-model overhead EMA from a
+ *    healthy usage-bearing context event (H_sample = usage.tokens − the REAL
+ *    message-list estimate). MUST use the true estimate, not the pct-derived
+ *    fallback — otherwise hSample ≈ 0 and the EMA trains itself to nothing.
+ *  - applyWireTruthOverride: when the last assistant message is an error whose
+ *    text matches the provider's 400 shape, treat the parsed requestTokens as
+ *    ground-truth currentTokens for THIS event's gate, feed the EMA, and prefer
+ *    the parsed availableTokens over runtime.lastCtxWindow.
+ *
+ * Both are no-op / byte-identical when config.wireOverhead is OFF. Non-fatal
+ * everywhere; never throw on the agent loop. Structured logging only.
+ */
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { MegaRuntime } from "../../mega-runtime.js";
+import type { MegaConfig } from "../../mega-config.js";
+import { parseWireTruth, sampleWireOverhead, readWireOverhead } from "./wireTruth.js";
+
+/**
+ * Calibrate the overhead EMA from a usage-bearing context event. Called once per
+ * event (from the handler) when usage is finite + wireOverhead is ON.
+ * `estimateTokens` MUST be the REAL message-list estimate (the engineView
+ * estimate), never the pct-derived fallback — pi's percent reconstructs
+ * usage.tokens exactly, so using it would force hSample ≈ 0 and erase the EMA
+ * (0.6^5 retained after five healthy turns).
+ */
+export function sampleWireOverheadFromUsage(opts: {
+	runtime: MegaRuntime;
+	config: MegaConfig;
+	modelId: string;
+	usageTokens: number | null | undefined;
+	resolvedWindow: number;
+	estimateTokens: number;
+}): void {
+	const { runtime, config, modelId, usageTokens, resolvedWindow, estimateTokens } = opts;
+	if (!config.wireOverhead) return; // byte-identical when OFF
+	if (modelId === "" || usageTokens == null || !Number.isFinite(usageTokens) || resolvedWindow <= 0)
+		return;
+	const hSample = Math.max(0, usageTokens - estimateTokens);
+	try {
+		sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+	} catch {
+		/* non-fatal */
+	}
+}
+
+/**
+ * Resolve the invisible overhead H (tokens) the handler feeds to the tail-cap
+ * budget at the fire/replay paths. Returns the calibrated EMA (or the
+ * wireOverheadDefaultPct × window fallback when no sample exists yet). 0 when
+ * wireOverhead is OFF or no model/window is known — byte-identical to v0.21.11.
+ * Computed once so every call site passes the SAME H the gate used.
+ */
+export function resolveOverheadTokens(opts: {
+	config: MegaConfig;
+	modelId: string;
+	resolvedWindow: number;
+	stateDir: string;
+}): number {
+	const { config, modelId, resolvedWindow, stateDir } = opts;
+	if (!config.wireOverhead || modelId === "" || resolvedWindow <= 0) return 0;
+	const e = readWireOverhead(modelId, stateDir, resolvedWindow);
+	return e > 0 ? e : config.wireOverheadDefaultPct * resolvedWindow;
+}
+
+/**
+ * v0.21.12: invisible-overhead correction of the ESTIMATE-path token count. When
+ * the token estimate came from the message list (not provider usage) AND
+ * wireOverhead is ON, add H so the gate/thrash/tail-cap see the REAL request
+ * size. H defaults to wireOverheadDefaultPct × window until a wire sample
+ * calibrates it. Flag OFF ⇒ H = 0 (byte-identical to v0.21.11). The provider
+ * usage path is ground truth for the message-list size and is never corrected.
+ */
+export function correctEstimateWithOverhead(opts: {
+	config: MegaConfig;
+	tokenSource: "usage" | "estimate" | "pct";
+	rawTokens: number;
+	modelId: string;
+	resolvedWindow: number;
+	stateDir: string;
+}): number {
+	const { config, tokenSource, rawTokens, modelId, resolvedWindow, stateDir } = opts;
+	if (!config.wireOverhead || tokenSource !== "estimate" || resolvedWindow <= 0) return rawTokens;
+	const h = modelId !== "" ? readWireOverhead(modelId, stateDir, resolvedWindow) : 0;
+	const H = h > 0 ? h : config.wireOverheadDefaultPct * resolvedWindow;
+	return rawTokens + H;
+}
+
+/**
+ * Apply the wire-truth gate override for THIS event. Returns the (possibly
+ * corrected) currentTokens. When the last assistant message is an error
+ * (stopReason "error" or an errorMessage field) and its text matches the
+ * provider's overflow error shape, the parsed requestTokens become
+ * ground-truth currentTokens — even when our estimate reads far below every
+ * threshold. Also feeds the EMA and prefers the parsed availableTokens over
+ * runtime.lastCtxWindow when they differ. No-op (returns currentTokens
+ * unchanged) when wireOverhead is OFF or no error/parse matched.
+ */
+export function applyWireTruthOverride(opts: {
+	runtime: MegaRuntime;
+	config: MegaConfig;
+	messages: readonly AgentMessage[];
+	modelId: string;
+	resolvedWindow: number;
+	estimateTokens: number;
+	currentTokens: number;
+}): number {
+	const { runtime, config, messages, modelId, resolvedWindow, estimateTokens, currentTokens } = opts;
+	if (!config.wireOverhead) return currentTokens; // byte-identical when OFF
+	try {
+		const last = messages[messages.length - 1] as
+			| { role?: string; stopReason?: string; errorMessage?: string; content?: unknown }
+			| undefined;
+		const lastText =
+			typeof last?.errorMessage === "string"
+				? last.errorMessage
+				: typeof last?.content === "string"
+					? last.content
+					: Array.isArray(last?.content)
+						? ((last.content as Array<{ text?: string }>)
+							.map((b) => b?.text ?? "")
+							.join(""))
+						: "";
+		const isError =
+			last?.stopReason === "error" || typeof last?.errorMessage === "string";
+		if (!isError || lastText.length === 0) return currentTokens;
+		const parsed = parseWireTruth(lastText);
+		if (parsed == null) return currentTokens;
+		const wireTokens = parsed.requestTokens;
+		runtime.lastCtxTokens = wireTokens;
+		// EMA: the gap between the wire prompt and our message estimate (usage is
+		// absent in the 400 case, so estimateTokens is the true message-list estimate).
+		if (modelId !== "" && resolvedWindow > 0) {
+			const hSample = Math.max(0, wireTokens - estimateTokens);
+			sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+		}
+		// Prefer the provider's own available size for this event's math.
+		if (parsed.availableTokens > 0 &&
+			Math.abs(parsed.availableTokens - runtime.lastCtxWindow) > 0) {
+			runtime.lastCtxWindow = parsed.availableTokens;
+		}
+		runtime.diagCtxWireTruth++;
+		runtime.logger.info("wire_truth_parse", {
+			sessionId: runtime.rt.sessionId,
+			requestTokens: parsed.requestTokens,
+			availableTokens: parsed.availableTokens,
+			estimateTokens,
+		});
+		try {
+			runtime.appendEvent("wire_truth_parse", {
+				requestTokens: parsed.requestTokens,
+				availableTokens: parsed.availableTokens,
+				estimateTokens,
+			});
+		} catch {
+			/* non-fatal */
+		}
+		return wireTokens;
+	} catch {
+		/* non-fatal */
+		return currentTokens;
+	}
+}

package/extensions/mega-events/context-handler.ts

@@ -35,6 +35,12 @@ import {
 import { invokePipeline } from "./context-handler/pipelineRun.js";
 import { buildLiveTrimView } from "./context-handler/liveTrim.js";
 import { recapReplayedTail } from "./context-handler/headroom.js";
+import {
+	sampleWireOverheadFromUsage,
+	applyWireTruthOverride,
+	resolveOverheadTokens,
+	correctEstimateWithOverhead,
+} from "./context-handler/wireTruthApply.js";
 
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(
@@ -99,15 +105,48 @@ export function registerContextHandler(
 		// show empty/zero. Compute view lazily only when the fallback is needed
 		// (at most one engineView call per context event; when auto is on and
 		// usage.tokens is present, view is computed once below via reuse).
+		// v0.21.12: build the engineView whenever wireOverhead is ON (not only when
+		// usage is absent) so the EMA sampling + wire-truth blocks measure the REAL
+		// message-list estimate. Flag OFF keeps the v0.21.11 lazy path. Without
+		// this, estimateTokens falls back to the pct-derived value, which
+		// reconstructs usage.tokens exactly → hSample ≈ 0 → EMA trains to nothing.
 		const viewForFallback =
-			usage?.tokens == null ? runtime.engineView(messages) : null;
-		const currentTokens =
-			usage?.tokens ??
-			(viewForFallback != null
+			usage?.tokens == null || config.wireOverhead ? runtime.engineView(messages) : null;
+		// v0.21.12: track WHICH source produced currentTokens so the invisible-
+		// overhead correction (H) is only applied to the ESTIMATE path. The
+		// provider-reported usage is ground truth for the message-list size; H is
+		// the gap between that and the wire prompt (system+tools+prepends).
+		const tokenSource: "usage" | "estimate" | "pct" =
+			usage?.tokens != null
+				? "usage"
+				: viewForFallback != null
+					? "estimate"
+					: "pct";
+		const estimateTokens =
+			viewForFallback != null
 				? estimateSessionTokens(viewForFallback)
-				: null) ??
-			Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+				: Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+		const rawTokens = usage?.tokens ?? estimateTokens;
+		// v0.21.12: invisible-overhead correction of the ESTIMATE-path count (see
+		// wireTruthApply.ts). modelId/resolvedWindow feed the EMA + tail-cap helpers.
+		const modelId = runtime.currentModel?.modelId ?? "";
+		const resolvedWindow =
+			usage?.contextWindow ?? (runtime.currentModel?.contextWindow ?? 0);
+		let currentTokens = correctEstimateWithOverhead({
+			config, tokenSource, rawTokens, modelId, resolvedWindow,
+			stateDir: runtime.currentStateDir,
+		});
 		runtime.lastCtxTokens = currentTokens ?? null;
+		// v0.21.12: the invisible overhead H to feed the tail-cap budget at the
+		// fire/replay paths below (resolved once via the wireTruthApply helper so
+		// every call site passes the SAME H the gate used). 0 when the flag is OFF
+		// (byte-identical to v0.21.11) or no model/window is known.
+		const overheadTokens = resolveOverheadTokens({
+			config,
+			modelId,
+			resolvedWindow,
+			stateDir: runtime.currentStateDir,
+		});
 		// 3WF-2: consume a pending live-window delta from a prior compaction. If a
 		// compaction fired on the previous context event and the live window did
 		// not shrink, this arms the ThrashGuard (meta). No-op when none pending.
@@ -132,6 +171,18 @@ export function registerContextHandler(
 			reportedWindow > 0
 				? reportedWindow
 				: (runtime.currentModel?.contextWindow ?? 0);
+		// v0.21.12: calibrate the invisible overhead H from EVERY context event that
+		// carries finite usage. estimateTokens is the REAL message-list estimate
+		// (engineView is built when wireOverhead is ON), so hSample = the true
+		// overhead (system+tools+prepends), not ≈0. Non-fatal; never throws.
+		sampleWireOverheadFromUsage({
+			runtime,
+			config,
+			modelId,
+			usageTokens: usage?.tokens,
+			resolvedWindow,
+			estimateTokens,
+		});
 		runtime.snapshot(ctx);
 		if (!config.auto) {
 			const tailed = tailResult();
@@ -145,6 +196,22 @@ export function registerContextHandler(
 		// message is captured, even if we don't compact this turn. Non-fatal.
 		appendMirrorAndLedger(runtime, config, messages);
 
+		// v0.21.12: WIRED-TRUTH gate override (extracted to wireTruthApply.ts).
+		// When the last assistant message is an error whose text matches the
+		// provider's 400 shape, the parsed requestTokens become ground-truth
+		// currentTokens for THIS event's gate — breaking the 400 loop when the
+		// estimate path undercounts by ~50% (the v0.21.11 blind spot). No-op when
+		// wireOverhead is OFF (byte-identical).
+		currentTokens = applyWireTruthOverride({
+			runtime,
+			config,
+			messages,
+			modelId,
+			resolvedWindow,
+			estimateTokens,
+			currentTokens,
+		});
+
 		// S29 FAST GATE: drive the auto-trigger off the context percent (see
 		// gateCheck.ts). Returns a tailed view when the gate does not pass.
 		const gate = evaluateGate(runtime, config, { pct, currentTokens, tailResult });
@@ -190,6 +257,7 @@ export function registerContextHandler(
 					maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
 					outputReservePct: config.outputReservePct,
 					safetyMarginPct: runtime.trimCache.safetyMarginPct,
+					overheadTokens,
 				});
 				runtime.diagLiveTrimFires++; // trim view returned this call (replay counts as a fire)
 				runtime.diagLiveTrimReplays++;
@@ -240,6 +308,7 @@ export function registerContextHandler(
 			pct,
 			currentTokens,
 			tailResult,
+			overheadTokens,
 		});
 		if (pipeline.kind === "return") return pipeline.view;
 
@@ -321,6 +390,7 @@ export function registerContextHandler(
 			ran: pipeline.ran,
 			perModelThreshold: gate.perModelThreshold,
 			tailResult,
+			overheadTokens,
 		});
 	});
 }

package/extensions/mega-runtime/runtime-instrumentation.ts

@@ -40,6 +40,8 @@ export class RuntimeInstrumentation {
 	diagCtxThrown = 0; // live-trim try threw (caught)
 	diagCtxOutputErrorTrip = 0; // Phase H: output-error catch tripped a forced compaction
 	diagCtxHeadroomTrip = 0; // v0.21.9: output-headroom gate tripped a pre-overflow compaction
+	diagCtxWireTruth = 0; // v0.21.12: a provider 400 text was parsed into ground-truth tokens
+	diagCtxSkipCapped = 0; // v0.21.12: the live-trim skip path was tail-capped to fit the budget
 
 	// Context health instrumentation (v0.12): rolling ring buffers for
 	// drift detection + cache poison Layer 1 hash baseline.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.21.11",
+  "version": "0.21.12",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",