pi-mega-compact 0.21.10 → 0.21.12

package/extensions/mega-config.ts

@@ -302,6 +302,16 @@ export function loadConfig(): MegaConfig {
 		contextHealthOutputQuality: envBool("MEGACOMPACT_CONTEXT_HEALTH_OUTPUT_QUALITY", true),
 		contextHealthCachePoison: envBool("MEGACOMPACT_CONTEXT_HEALTH_CACHE_POISON", true),
 		contextHealthMitigate: envBool("MEGACOMPACT_CONTEXT_HEALTH_MITIGATE", false),
+		// v0.21.12: invisible-overhead calibration — add the provider's fixed
+		// request overhead H (system+tools+extension prepends, never in the
+		// transcript) back into the token estimate for the headroom gate / tail
+		// cap. H is an EMA of observed wire samples per model, else
+		// wireOverheadDefaultPct × window. Default ON; OFF = byte-identical
+		// v0.21.11 (every H term is 0). Closes attempt #9 of the 32k overflow loop.
+		wireOverhead: envBool("MEGACOMPACT_WIRE_OVERHEAD", true),
+		// v0.21.12: fallback H as a fraction of the window when no EMA sample
+		// exists yet. Clamped [0, 0.85]; default 0.15. Percent-based.
+		wireOverheadDefaultPct: clamp(envFlag("MEGACOMPACT_WIRE_OVERHEAD_DEFAULT_PCT", 0.15), 0, 0.85),
 		// D.1: env-overridable recompact delta (minimum context growth % before
 		// re-compacting instead of replaying the cached live trim). Default 50.
 		recompactPctDelta: envFlag("MEGACOMPACT_RECOMPACT_PCT_DELTA", 50),

package/extensions/mega-events/context-handler/headroom.ts

@@ -16,9 +16,56 @@
  * Pure functions, no runtime dependency — trivially unit-testable headlessly.
  */
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
-import { estimateBlockTokens, estimateMessageTokens } from "../../../src/tokens.js";
+import { estimateBlockTokens } from "../../../src/tokens.js";
 import { messageContentText } from "./messageText.js";
 
+/**
+ * Full-surface AgentMessage token estimate for BUDGET arithmetic (tail cap).
+ *
+ * convertToLlm (pi dist/core/messages.js) ships assistant/toolResult messages
+ * VERBATIM — every content block goes over the wire: text, thinking, toolCall
+ * (name + full `arguments` JSON), toolResult output, role wrappers. The text
+ * extractor (messageContentText) is lossy-on-purpose for analytics, and using
+ * it here made a GLM-4.7-style assistant message with ~11.6k bytes of toolCall
+ * arguments register as ~77 tokens — a 30k-token tail passed an 11.9k budget,
+ * the model overflowed, and pi's one-shot compact-and-retry failed
+ * ("Context overflow recovery failed", 2026-08-20 incident).
+ *
+ * Counts every byte the provider actually receives. Still a heuristic (len/4
+ * + 1 per block, like estimateBlockTokens) — just no longer lossy. Never
+ * throws: unknown block shapes fall back to their JSON serialization length,
+ * and a non-array/string content is counted as its serialization.
+ */
+export function estimateAgentMessageBudgetTokens(m: AgentMessage): number {
+	try {
+		const c = (m as { content?: unknown }).content;
+		let bytes = 0;
+		if (typeof c === "string") {
+			bytes += c.length;
+		} else if (Array.isArray(c)) {
+			for (const b of c) {
+				if (b == null || typeof b !== "object") continue;
+				const o = b as Record<string, unknown>;
+				if (typeof o.text === "string") bytes += o.text.length;
+				if (typeof o.thinking === "string") bytes += o.thinking.length;
+				if (typeof o.name === "string") bytes += o.name.length;
+				if (o.arguments != null) bytes += JSON.stringify(o.arguments).length;
+				if (typeof o.output === "string") bytes += o.output.length;
+				// Per-block envelope overhead (role/type markers), matching the
+				// len/4+1 block accounting in estimateBlockTokens.
+				bytes += 4;
+			}
+		} else if (c != null) {
+			bytes += JSON.stringify(c).length;
+		}
+		return estimateBlockTokens(" ".repeat(Math.max(0, bytes)));
+	} catch {
+		// non-fatal: fall back to the legacy text-only estimate rather than
+		// disable the cap on a pathological message.
+		return estimateBlockTokens(messageContentText(m));
+	}
+}
+
 /**
  * The model's declared maxTokens is only trusted as the output budget when it
  * is plausible. models.json carries sentinel junk for some entries (1e9,
@@ -112,6 +159,15 @@ export function applyTailCap(opts: {
 	 * otherwise each message's tokens are estimated from its text content.
 	 */
 	messageTokens?: readonly number[];
+	/**
+	 * v0.21.12: the provider's invisible overhead H (system prompt + tool
+	 * definitions + extension systemPrompt prepends — everything pi adds at
+	 * request time that NEVER appears in the stored transcript). Subtracted from
+	 * the budget so the cap accounts for the REAL wire prompt, not just the
+	 * counted messages. Default 0 (no H) — identical behavior to v0.21.11 when
+	 * the wireOverhead flag is OFF. Clamped to >= 0.
+	 */
+	overheadTokens?: number;
 }): { recent: AgentMessage[]; dropped: number } {
 	const { recentRaw, summaryTokens, ctxWindow, outputReservePct } = opts;
 	if (ctxWindow <= 0 || recentRaw.length <= 1) {
@@ -133,9 +189,16 @@ export function applyTailCap(opts: {
 	// <= 95% of the window) plus margin + summary can still exceed the window
 	// on tiny summaries-free edges; the floor keeps the cap alive with a small
 	// positive budget instead of disabling it (pre-v0.21.9 behavior).
+	// v0.21.12: also subtract the invisible overhead H so the cap bounds the
+	// ACTUAL wire prompt (messages + H), breaking the 400 loop when the estimate
+	// undercounts by the system-prompt/tool-definition overhead.
 	const budget = Math.max(
 		1,
-		ctxWindow - reserveTokens - safetyMargin - Math.max(0, summaryTokens),
+		ctxWindow -
+			reserveTokens -
+			safetyMargin -
+			Math.max(0, summaryTokens) -
+			Math.max(0, opts.overheadTokens ?? 0),
 	);
 	let start = 0;
 	let tailTokens = 0;
@@ -143,7 +206,7 @@ export function applyTailCap(opts: {
 		tailTokens +=
 			msgTokens != null
 				? Math.max(0, msgTokens[i])
-				: estimateMessageTokens({ text: messageContentText(recentRaw[i]) });
+				: estimateAgentMessageBudgetTokens(recentRaw[i]);
 		if (tailTokens > budget) {
 			// Keep from i+1 onward; never drop below the FINAL message.
 			start = Math.min(i + 1, recentRaw.length - 1);
@@ -178,13 +241,16 @@ export function recapReplayedTail(opts: {
 	maxOutputTokens: number;
 	outputReservePct: number;
 	safetyMarginPct: number;
+	/** v0.21.12: invisible overhead H (see applyTailCap). Default 0. */
+	overheadTokens?: number;
 }): { recent: AgentMessage[]; dropped: number } {
 	return applyTailCap({
 		recentRaw: opts.recentRaw,
-		summaryTokens: estimateBlockTokens(messageContentText(opts.summaryAgentMsg)),
+		summaryTokens: estimateAgentMessageBudgetTokens(opts.summaryAgentMsg),
 		ctxWindow: opts.ctxWindow,
 		maxOutputTokens: opts.maxOutputTokens,
 		outputReservePct: opts.outputReservePct,
 		safetyMarginPct: opts.safetyMarginPct,
+		overheadTokens: opts.overheadTokens ?? 0,
 	});
 }

package/extensions/mega-events/context-handler/liveTrim.ts

@@ -45,6 +45,10 @@ export function buildLiveTrimView(
 		ran: { result: CompactResult };
 		perModelThreshold: { safetyMarginPct: number; firePointPct: number };
 		tailResult: TailResultFn;
+		/** v0.21.12: the provider's invisible overhead H (system+tools+prepends),
+		 *  handler-resolved. Subtracted from the tail-cap budget so the cap bounds
+		 *  the REAL wire prompt. 0 when the flag is OFF (byte-identical v0.21.11). */
+		overheadTokens?: number;
 	},
 ): { messages: AgentMessage[] } | undefined {
 	const {
@@ -57,6 +61,7 @@ export function buildLiveTrimView(
 		ran,
 		perModelThreshold,
 		tailResult,
+		overheadTokens = 0,
 	} = opts;
 
 	// S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
@@ -105,6 +110,37 @@ export function buildLiveTrimView(
 				anchorUserMessages,
 				criticalOver: (pct ?? 0) >= 90,
 			});
+			// v0.21.12: CAP THE SKIP PATH. When computeLiveTrimCut returns null
+			// (anchor floor blocked cutting a fat recent tool pair, or the
+			// criticalOver hatch stayed closed because estimated pressure ≈80%),
+			// the pre-v0.21.12 code shipped the RAW untrimmed view — which, with
+			// the invisible overhead H uncounted, overflowed the window → 400
+			// again, forever (the entire v0.21.11 blind spot). This is the
+			// invariant "the trim path never ships a view the budget wouldn't
+			// allow": even when we cannot summarize, we still front-drop
+			// OLDEST messages until the RAW tail + overhead fits the budget, so
+			// the model is never fed a prompt that exceeds its window. Flag OFF
+			// ⇒ byte-identical to v0.21.11 (return raw, no cap).
+			if (config.wireOverhead && runtime.lastCtxWindow > 0) {
+				const { recent, dropped } = applyTailCap({
+					recentRaw: messages,
+					summaryTokens: 0,
+					ctxWindow: runtime.lastCtxWindow,
+					maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
+					outputReservePct: config.outputReservePct,
+					safetyMarginPct: perModelThreshold.safetyMarginPct,
+					overheadTokens,
+				});
+				if (dropped > 0) {
+					runtime.diagCtxSkipCapped++;
+					runtime.logger.info("skip_cap_applied", {
+						sessionId: runtime.rt.sessionId,
+						dropped,
+						ctxWindow: runtime.lastCtxWindow,
+					});
+					return tailResult(recent) ?? { messages: recent };
+				}
+			}
 			return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
 		}
 		const summaryMsg = liveTrimSummaryMessage({
@@ -152,6 +188,7 @@ export function buildLiveTrimView(
 			maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
 			outputReservePct: config.outputReservePct,
 			safetyMarginPct: modelThreshold.safetyMarginPct,
+			overheadTokens,
 		});
 		if (dropped > 0) {
 			runtime.logger.warn("live-trim-tail-cap", {

package/extensions/mega-events/context-handler/pipelineRun.ts

@@ -48,6 +48,7 @@ export function invokePipeline(
 		pct: number | null | undefined;
 		currentTokens: number;
 		tailResult: TailResultFn;
+		overheadTokens?: number;
 	},
 ): PipelineOutcome {
 	// VC5C: emit the rollout decision per compact event (observability seam).
@@ -133,6 +134,7 @@ export function invokePipeline(
 				maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
 				outputReservePct: config.outputReservePct,
 				safetyMarginPct: runtime.trimCache.safetyMarginPct,
+				overheadTokens: opts.overheadTokens ?? 0,
 			});
 			runtime.diagLiveTrimFires++;
 			runtime.diagLiveTrimReplays++;

package/extensions/mega-events/context-handler/wireTruth.ts

@@ -0,0 +1,132 @@
+/**
+ * context-handler/wireTruth.ts — invisible-overhead calibration + wire-truth parse.
+ *
+ * Attempt #9 on the small-context-model overflow loop (2026-08-19 incident,
+ * 8 prior attempts). ROOT CAUSE: pi adds a FIXED OVERHEAD H at request time
+ * (system prompt + tool definitions + extension systemPrompt prepends) that
+ * NEVER appears in the stored transcript. Neither pi's estimateContextTokens nor
+ * our estimateSessionTokens/applyTailCap count H. So when the provider 400s with
+ * "request (39048 tokens) exceeds the available context size (32768 tokens)" we
+ * have been estimating ~18–26k and judging headroom against that — undercounting
+ * by ~50%, so the gate never trips correctly and a RAW uncapped view ships → 400
+ * again, forever (pi's one-shot overflow recovery only resets on user
+ * message_start, so auto-retries burn it permanently).
+ *
+ * Two mechanisms here:
+ *  - parseWireTruth: regex the provider's 400 text into ground-truth
+ *    request/available token counts. Pure + unit-tested.
+ *  - Per-model overhead EMA: calibrate H from observed samples so the gate and
+ *    tail cap can ADD it back. Persisted in the SQLite meta table (copy the
+ *    thrashGuard.ts pattern) so it survives restarts and travels with the repo.
+ *
+ * Non-fatal EVERYWHERE: every store read/write is best-effort, swallowed on
+ * failure. Structured JSON logging only (logger + emit dual-sink, mirroring
+ * armThrashGuard's 3WF-5 pattern). No console.*, no network, no mocks.
+ */
+import { getMetaNumber, setMetaNumber } from "../../../src/store/sqlite.js";
+
+/** EMA smoothing factor for the overhead calibration (fixed, no count needed). */
+export const OVERHEAD_EMA_ALPHA = 0.4;
+
+/** Safety clamp: an overhead sample may never exceed this fraction of the window. */
+export const OVERHEAD_CLAMP_FRACTION = 0.85;
+
+/** Meta key prefix; the model id is appended (meta stores integers only). */
+export const OVERHEAD_META_PREFIX = "wire.overhead_ema.";
+
+/**
+ * Parse the provider's overflow error text into ground-truth token counts.
+ *
+ * Matches strings like:
+ *   "request (39048 tokens) exceeds the available context size (32768 tokens)"
+ *   "request (39,048 tokens) exceeds the available context size (32,768 tokens)"
+ *
+ * Pure + side-effect free — trivially unit-testable. Returns null when the text
+ * does not match the expected shape (e.g. an unrelated error message).
+ */
+export function parseWireTruth(
+	text: string,
+): { requestTokens: number; availableTokens: number } | null {
+	if (typeof text !== "string" || text.length === 0) return null;
+	const m = text.match(
+		/request\s*\((\d[\d,]*)\s*tokens\)\s*exceeds the available context size\s*\((\d[\d,]*)\s*tokens\)/i,
+	);
+	if (!m) return null;
+	const requestTokens = Number(m[1]?.replace(/,/g, ""));
+	const availableTokens = Number(m[2]?.replace(/,/g, ""));
+	if (!Number.isFinite(requestTokens) || !Number.isFinite(availableTokens)) return null;
+	if (requestTokens <= 0 || availableTokens <= 0) return null;
+	return { requestTokens, availableTokens };
+}
+
+/** Build the meta key for a model's overhead EMA (stored ×100, integer). */
+function overheadKey(modelId: string): string {
+	return OVERHEAD_META_PREFIX + modelId;
+}
+
+/**
+ * Read the calibrated overhead (tokens) for a model. Returns 0 when absent or
+ * unreadable — non-fatal everywhere. When the context window is known, clamps the
+ * value into [0, 0.85 × ctxWindow] as a safety against a runaway EMA.
+ */
+export function readWireOverhead(modelId: string, stateDir: string, ctxWindow = 0): number {
+	if (!modelId) return 0;
+	try {
+		const stored = getMetaNumber(overheadKey(modelId), stateDir);
+		if (!Number.isFinite(stored) || stored <= 0) return 0;
+		const frac = stored / 100;
+		let overhead = frac * (ctxWindow > 0 ? ctxWindow : 1);
+		if (ctxWindow > 0) {
+			overhead = Math.min(overhead, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+		}
+		return Math.max(0, overhead);
+	} catch {
+		return 0; // non-fatal: never fail the agent loop on a store read
+	}
+}
+
+/**
+ * Fold a new overhead sample into the per-model EMA and persist it. Returns the
+ * new EMA in tokens (0 when the sample is invalid). First sample initializes the
+ * EMA directly (no warm-up). Best-effort: never throws; a store failure is
+ * swallowed and the in-memory EMA is still returned.
+ *
+ * Clamped into [0, 0.85 × ctxWindow] when ctxWindow is known.
+ */
+export function sampleWireOverhead(
+	modelId: string,
+	stateDir: string,
+	sample: number,
+	ctxWindow = 0,
+): number {
+	if (!modelId) return 0;
+	if (!Number.isFinite(sample) || sample <= 0) return 0;
+	let ema = sample;
+	try {
+		const stored = getMetaNumber(overheadKey(modelId), stateDir);
+		if (Number.isFinite(stored) && stored > 0) {
+			// prev is stored as a fraction ×100; convert back to token space
+			// against the window before blending so the EMA is dimensionally
+			// consistent (token space in, token space out).
+			const prevTokens = (stored / 100) * (ctxWindow > 0 ? ctxWindow : sample);
+			ema = OVERHEAD_EMA_ALPHA * sample + (1 - OVERHEAD_EMA_ALPHA) * prevTokens;
+		}
+		// Convert EMA to a fraction of the window (so the meta value is window-
+		// independent + percent-based). When the window is unknown, clamp the
+		// stored fraction to a sane [0, OVERHEAD_CLAMP_FRACTION] band so a stale
+		// 0-window sample cannot poison a later windowed read.
+		let frac = ctxWindow > 0 ? ema / ctxWindow : ema;
+		if (ctxWindow > 0) {
+			frac = Math.min(frac, OVERHEAD_CLAMP_FRACTION);
+		} else {
+			frac = Math.min(Math.max(frac, 0), OVERHEAD_CLAMP_FRACTION);
+		}
+		setMetaNumber(overheadKey(modelId), Math.round(frac * 100), stateDir);
+	} catch {
+		/* non-fatal: best-effort meta write */
+	}
+	// Return the token-space EMA (clamped) regardless of whether the persist landed.
+	let out = ema;
+	if (ctxWindow > 0) out = Math.min(out, ctxWindow * OVERHEAD_CLAMP_FRACTION);
+	return Math.max(0, out);
+}

package/extensions/mega-events/context-handler/wireTruthApply.ts

@@ -0,0 +1,168 @@
+/**
+ * context-handler/wireTruthApply.ts — v0.21.12 wired-overhead runtime seam.
+ *
+ * Delegate-shell sibling extracted from context-handler.ts (extensions/ 400-line
+ * soft limit). Holds the two wire-overhead call-site blocks that don't fit the
+ * handler's budget:
+ *  - sampleWireOverheadFromUsage: calibrate the per-model overhead EMA from a
+ *    healthy usage-bearing context event (H_sample = usage.tokens − the REAL
+ *    message-list estimate). MUST use the true estimate, not the pct-derived
+ *    fallback — otherwise hSample ≈ 0 and the EMA trains itself to nothing.
+ *  - applyWireTruthOverride: when the last assistant message is an error whose
+ *    text matches the provider's 400 shape, treat the parsed requestTokens as
+ *    ground-truth currentTokens for THIS event's gate, feed the EMA, and prefer
+ *    the parsed availableTokens over runtime.lastCtxWindow.
+ *
+ * Both are no-op / byte-identical when config.wireOverhead is OFF. Non-fatal
+ * everywhere; never throw on the agent loop. Structured logging only.
+ */
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { MegaRuntime } from "../../mega-runtime.js";
+import type { MegaConfig } from "../../mega-config.js";
+import { parseWireTruth, sampleWireOverhead, readWireOverhead } from "./wireTruth.js";
+
+/**
+ * Calibrate the overhead EMA from a usage-bearing context event. Called once per
+ * event (from the handler) when usage is finite + wireOverhead is ON.
+ * `estimateTokens` MUST be the REAL message-list estimate (the engineView
+ * estimate), never the pct-derived fallback — pi's percent reconstructs
+ * usage.tokens exactly, so using it would force hSample ≈ 0 and erase the EMA
+ * (0.6^5 retained after five healthy turns).
+ */
+export function sampleWireOverheadFromUsage(opts: {
+	runtime: MegaRuntime;
+	config: MegaConfig;
+	modelId: string;
+	usageTokens: number | null | undefined;
+	resolvedWindow: number;
+	estimateTokens: number;
+}): void {
+	const { runtime, config, modelId, usageTokens, resolvedWindow, estimateTokens } = opts;
+	if (!config.wireOverhead) return; // byte-identical when OFF
+	if (modelId === "" || usageTokens == null || !Number.isFinite(usageTokens) || resolvedWindow <= 0)
+		return;
+	const hSample = Math.max(0, usageTokens - estimateTokens);
+	try {
+		sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+	} catch {
+		/* non-fatal */
+	}
+}
+
+/**
+ * Resolve the invisible overhead H (tokens) the handler feeds to the tail-cap
+ * budget at the fire/replay paths. Returns the calibrated EMA (or the
+ * wireOverheadDefaultPct × window fallback when no sample exists yet). 0 when
+ * wireOverhead is OFF or no model/window is known — byte-identical to v0.21.11.
+ * Computed once so every call site passes the SAME H the gate used.
+ */
+export function resolveOverheadTokens(opts: {
+	config: MegaConfig;
+	modelId: string;
+	resolvedWindow: number;
+	stateDir: string;
+}): number {
+	const { config, modelId, resolvedWindow, stateDir } = opts;
+	if (!config.wireOverhead || modelId === "" || resolvedWindow <= 0) return 0;
+	const e = readWireOverhead(modelId, stateDir, resolvedWindow);
+	return e > 0 ? e : config.wireOverheadDefaultPct * resolvedWindow;
+}
+
+/**
+ * v0.21.12: invisible-overhead correction of the ESTIMATE-path token count. When
+ * the token estimate came from the message list (not provider usage) AND
+ * wireOverhead is ON, add H so the gate/thrash/tail-cap see the REAL request
+ * size. H defaults to wireOverheadDefaultPct × window until a wire sample
+ * calibrates it. Flag OFF ⇒ H = 0 (byte-identical to v0.21.11). The provider
+ * usage path is ground truth for the message-list size and is never corrected.
+ */
+export function correctEstimateWithOverhead(opts: {
+	config: MegaConfig;
+	tokenSource: "usage" | "estimate" | "pct";
+	rawTokens: number;
+	modelId: string;
+	resolvedWindow: number;
+	stateDir: string;
+}): number {
+	const { config, tokenSource, rawTokens, modelId, resolvedWindow, stateDir } = opts;
+	if (!config.wireOverhead || tokenSource !== "estimate" || resolvedWindow <= 0) return rawTokens;
+	const h = modelId !== "" ? readWireOverhead(modelId, stateDir, resolvedWindow) : 0;
+	const H = h > 0 ? h : config.wireOverheadDefaultPct * resolvedWindow;
+	return rawTokens + H;
+}
+
+/**
+ * Apply the wire-truth gate override for THIS event. Returns the (possibly
+ * corrected) currentTokens. When the last assistant message is an error
+ * (stopReason "error" or an errorMessage field) and its text matches the
+ * provider's overflow error shape, the parsed requestTokens become
+ * ground-truth currentTokens — even when our estimate reads far below every
+ * threshold. Also feeds the EMA and prefers the parsed availableTokens over
+ * runtime.lastCtxWindow when they differ. No-op (returns currentTokens
+ * unchanged) when wireOverhead is OFF or no error/parse matched.
+ */
+export function applyWireTruthOverride(opts: {
+	runtime: MegaRuntime;
+	config: MegaConfig;
+	messages: readonly AgentMessage[];
+	modelId: string;
+	resolvedWindow: number;
+	estimateTokens: number;
+	currentTokens: number;
+}): number {
+	const { runtime, config, messages, modelId, resolvedWindow, estimateTokens, currentTokens } = opts;
+	if (!config.wireOverhead) return currentTokens; // byte-identical when OFF
+	try {
+		const last = messages[messages.length - 1] as
+			| { role?: string; stopReason?: string; errorMessage?: string; content?: unknown }
+			| undefined;
+		const lastText =
+			typeof last?.errorMessage === "string"
+				? last.errorMessage
+				: typeof last?.content === "string"
+					? last.content
+					: Array.isArray(last?.content)
+						? ((last.content as Array<{ text?: string }>)
+							.map((b) => b?.text ?? "")
+							.join(""))
+						: "";
+		const isError =
+			last?.stopReason === "error" || typeof last?.errorMessage === "string";
+		if (!isError || lastText.length === 0) return currentTokens;
+		const parsed = parseWireTruth(lastText);
+		if (parsed == null) return currentTokens;
+		const wireTokens = parsed.requestTokens;
+		runtime.lastCtxTokens = wireTokens;
+		// EMA: the gap between the wire prompt and our message estimate (usage is
+		// absent in the 400 case, so estimateTokens is the true message-list estimate).
+		if (modelId !== "" && resolvedWindow > 0) {
+			const hSample = Math.max(0, wireTokens - estimateTokens);
+			sampleWireOverhead(modelId, runtime.currentStateDir, hSample, resolvedWindow);
+		}
+		// Prefer the provider's own available size for this event's math.
+		if (parsed.availableTokens > 0 &&
+			Math.abs(parsed.availableTokens - runtime.lastCtxWindow) > 0) {
+			runtime.lastCtxWindow = parsed.availableTokens;
+		}
+		runtime.diagCtxWireTruth++;
+		runtime.logger.info("wire_truth_parse", {
+			sessionId: runtime.rt.sessionId,
+			requestTokens: parsed.requestTokens,
+			availableTokens: parsed.availableTokens,
+			estimateTokens,
+		});
+		try {
+			runtime.appendEvent("wire_truth_parse", {
+				requestTokens: parsed.requestTokens,
+				availableTokens: parsed.availableTokens,
+				estimateTokens,
+			});
+		} catch {
+			/* non-fatal */
+		}
+		return wireTokens;
+	} catch {
+		/* non-fatal */
+		return currentTokens;
+	}
+}

package/extensions/mega-events/context-handler.ts

@@ -35,6 +35,12 @@ import {
 import { invokePipeline } from "./context-handler/pipelineRun.js";
 import { buildLiveTrimView } from "./context-handler/liveTrim.js";
 import { recapReplayedTail } from "./context-handler/headroom.js";
+import {
+	sampleWireOverheadFromUsage,
+	applyWireTruthOverride,
+	resolveOverheadTokens,
+	correctEstimateWithOverhead,
+} from "./context-handler/wireTruthApply.js";
 
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(
@@ -99,15 +105,48 @@ export function registerContextHandler(
 		// show empty/zero. Compute view lazily only when the fallback is needed
 		// (at most one engineView call per context event; when auto is on and
 		// usage.tokens is present, view is computed once below via reuse).
+		// v0.21.12: build the engineView whenever wireOverhead is ON (not only when
+		// usage is absent) so the EMA sampling + wire-truth blocks measure the REAL
+		// message-list estimate. Flag OFF keeps the v0.21.11 lazy path. Without
+		// this, estimateTokens falls back to the pct-derived value, which
+		// reconstructs usage.tokens exactly → hSample ≈ 0 → EMA trains to nothing.
 		const viewForFallback =
-			usage?.tokens == null ? runtime.engineView(messages) : null;
-		const currentTokens =
-			usage?.tokens ??
-			(viewForFallback != null
+			usage?.tokens == null || config.wireOverhead ? runtime.engineView(messages) : null;
+		// v0.21.12: track WHICH source produced currentTokens so the invisible-
+		// overhead correction (H) is only applied to the ESTIMATE path. The
+		// provider-reported usage is ground truth for the message-list size; H is
+		// the gap between that and the wire prompt (system+tools+prepends).
+		const tokenSource: "usage" | "estimate" | "pct" =
+			usage?.tokens != null
+				? "usage"
+				: viewForFallback != null
+					? "estimate"
+					: "pct";
+		const estimateTokens =
+			viewForFallback != null
 				? estimateSessionTokens(viewForFallback)
-				: null) ??
-			Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+				: Math.round(((pct ?? 0) / 100) * (usage?.contextWindow ?? 0));
+		const rawTokens = usage?.tokens ?? estimateTokens;
+		// v0.21.12: invisible-overhead correction of the ESTIMATE-path count (see
+		// wireTruthApply.ts). modelId/resolvedWindow feed the EMA + tail-cap helpers.
+		const modelId = runtime.currentModel?.modelId ?? "";
+		const resolvedWindow =
+			usage?.contextWindow ?? (runtime.currentModel?.contextWindow ?? 0);
+		let currentTokens = correctEstimateWithOverhead({
+			config, tokenSource, rawTokens, modelId, resolvedWindow,
+			stateDir: runtime.currentStateDir,
+		});
 		runtime.lastCtxTokens = currentTokens ?? null;
+		// v0.21.12: the invisible overhead H to feed the tail-cap budget at the
+		// fire/replay paths below (resolved once via the wireTruthApply helper so
+		// every call site passes the SAME H the gate used). 0 when the flag is OFF
+		// (byte-identical to v0.21.11) or no model/window is known.
+		const overheadTokens = resolveOverheadTokens({
+			config,
+			modelId,
+			resolvedWindow,
+			stateDir: runtime.currentStateDir,
+		});
 		// 3WF-2: consume a pending live-window delta from a prior compaction. If a
 		// compaction fired on the previous context event and the live window did
 		// not shrink, this arms the ThrashGuard (meta). No-op when none pending.
@@ -132,6 +171,18 @@ export function registerContextHandler(
 			reportedWindow > 0
 				? reportedWindow
 				: (runtime.currentModel?.contextWindow ?? 0);
+		// v0.21.12: calibrate the invisible overhead H from EVERY context event that
+		// carries finite usage. estimateTokens is the REAL message-list estimate
+		// (engineView is built when wireOverhead is ON), so hSample = the true
+		// overhead (system+tools+prepends), not ≈0. Non-fatal; never throws.
+		sampleWireOverheadFromUsage({
+			runtime,
+			config,
+			modelId,
+			usageTokens: usage?.tokens,
+			resolvedWindow,
+			estimateTokens,
+		});
 		runtime.snapshot(ctx);
 		if (!config.auto) {
 			const tailed = tailResult();
@@ -145,6 +196,22 @@ export function registerContextHandler(
 		// message is captured, even if we don't compact this turn. Non-fatal.
 		appendMirrorAndLedger(runtime, config, messages);
 
+		// v0.21.12: WIRED-TRUTH gate override (extracted to wireTruthApply.ts).
+		// When the last assistant message is an error whose text matches the
+		// provider's 400 shape, the parsed requestTokens become ground-truth
+		// currentTokens for THIS event's gate — breaking the 400 loop when the
+		// estimate path undercounts by ~50% (the v0.21.11 blind spot). No-op when
+		// wireOverhead is OFF (byte-identical).
+		currentTokens = applyWireTruthOverride({
+			runtime,
+			config,
+			messages,
+			modelId,
+			resolvedWindow,
+			estimateTokens,
+			currentTokens,
+		});
+
 		// S29 FAST GATE: drive the auto-trigger off the context percent (see
 		// gateCheck.ts). Returns a tailed view when the gate does not pass.
 		const gate = evaluateGate(runtime, config, { pct, currentTokens, tailResult });
@@ -190,6 +257,7 @@ export function registerContextHandler(
 					maxOutputTokens: runtime.currentModel?.maxTokens ?? 0,
 					outputReservePct: config.outputReservePct,
 					safetyMarginPct: runtime.trimCache.safetyMarginPct,
+					overheadTokens,
 				});
 				runtime.diagLiveTrimFires++; // trim view returned this call (replay counts as a fire)
 				runtime.diagLiveTrimReplays++;
@@ -240,6 +308,7 @@ export function registerContextHandler(
 			pct,
 			currentTokens,
 			tailResult,
+			overheadTokens,
 		});
 		if (pipeline.kind === "return") return pipeline.view;
 
@@ -321,6 +390,7 @@ export function registerContextHandler(
 			ran: pipeline.ran,
 			perModelThreshold: gate.perModelThreshold,
 			tailResult,
+			overheadTokens,
 		});
 	});
 }

package/extensions/mega-runtime/runtime-instrumentation.ts

@@ -40,6 +40,8 @@ export class RuntimeInstrumentation {
 	diagCtxThrown = 0; // live-trim try threw (caught)
 	diagCtxOutputErrorTrip = 0; // Phase H: output-error catch tripped a forced compaction
 	diagCtxHeadroomTrip = 0; // v0.21.9: output-headroom gate tripped a pre-overflow compaction
+	diagCtxWireTruth = 0; // v0.21.12: a provider 400 text was parsed into ground-truth tokens
+	diagCtxSkipCapped = 0; // v0.21.12: the live-trim skip path was tail-capped to fit the budget
 
 	// Context health instrumentation (v0.12): rolling ring buffers for
 	// drift detection + cache poison Layer 1 hash baseline.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.21.10",
+  "version": "0.21.12",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",