pi-mega-compact 0.13.0 → 0.13.2

package/extensions/mega-compact-s38.test/_helpers.ts

@@ -0,0 +1,201 @@
+/**
+ * Shared harness for the mega-compact-s38 split test files.
+ *
+ * Extracted from the top of mega-compact-s38.test.ts: the mock-pi harness that
+ * loads the compiled extension, the eventTypes/eventPayloads log readers, the
+ * s38TurnEnd helpers, and the classifyError fixtures.
+ *
+ * Every split file imports this module, which also registers an `after()` hook
+ * that force-exits the test subprocess. Each harness() creates a MegaRuntime
+ * with an fs.watch game-state watcher that is never disposed (no
+ * session_shutdown in tests); those handles keep the event loop alive
+ * indefinitely, so without a force-exit `node --test` (without
+ * --test-force-exit) would hang. The hook drains stdout/stderr then defers the
+ * exit — a bare process.exit() discards unflushed pipe buffers (observed
+ * 2026-07-30: piped test results silently vanished).
+ *
+ * Relative paths are one directory deeper than the original file, hence
+ * `../mega-compact.js` / `../mega-events.js`.
+ */
+import { mkdtempSync, rmSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createRequire } from "node:module";
+import { after } from "node:test";
+import assert from "node:assert/strict";
+import { closeVectorIndex } from "../../src/store/vectorIndex.js";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+
+export const require = createRequire(import.meta.url);
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-s38-"));
+export const baseTmpDir = baseTmp;
+export { closeVectorIndex };
+// Isolate the machine-wide repo index so test runs never pollute the real one.
+process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
+let counter = 0;
+
+after(async () => {
+	// PGlite WASM close can hang; race with a timeout to prevent 40-min hangs.
+	try {
+		await Promise.race([closeVectorIndex(), new Promise((r) => setTimeout(r, 3000))]);
+	} catch { /* ignore */ }
+	if (existsSync(baseTmp)) rmSync(baseTmp, { recursive: true, force: true });
+	// Drain stdout/stderr, then defer the exit so the final test report flushes.
+	await new Promise((r) => process.stdout.write("", r));
+	await new Promise((r) => process.stderr.write("", r));
+	await new Promise((r) => setTimeout(r, 1500));
+	process.exit(0);
+});
+
+/** Read events.log and return an array of event type strings. */
+export function eventTypes(stateDir: string): string[] {
+	const { readFileSync: rf, existsSync: ex } =
+		require("node:fs") as typeof import("node:fs");
+	const { join: j } = require("node:path") as typeof import("node:path");
+	const logPath = j(stateDir, "events.log");
+	if (!ex(logPath)) return [];
+	const content = rf(logPath, "utf-8").trim();
+	if (content.length === 0) return [];
+	return content
+		.split("\n")
+		.map((line) => { try { return JSON.parse(line).type; } catch { return undefined; } })
+		.filter((t): t is string => typeof t === "string");
+}
+
+/** R11: read events.log and return full JSON payloads for a given type. */
+export function eventPayloads(stateDir: string, type: string): Record<string, unknown>[] {
+	const { readFileSync: rf, existsSync: ex } =
+		require("node:fs") as typeof import("node:fs");
+	const { join: j } = require("node:path") as typeof import("node:path");
+	const logPath = j(stateDir, "events.log");
+	if (!ex(logPath)) return [];
+	const content = rf(logPath, "utf-8").trim();
+	if (content.length === 0) return [];
+	return content
+		.split("\n")
+		.map((line) => { try { return JSON.parse(line); } catch { return undefined; } })
+		.filter((p): p is Record<string, unknown> => typeof p === "object" && p !== null && p.type === type);
+}
+
+/** Build a mock pi + ctx and load the extension into them. */
+export function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
+	const stateDir = join(baseTmp, `run-${counter++}`);
+	process.env.MEGACOMPACT_STATE_DIR = stateDir;
+	process.env.MEGACOMPACT_DEBUG = "true";
+	if (!opts.keepThreshold) process.env.MEGACOMPACT_THRESHOLD_TOKENS = "50";
+	if (!opts.keepTier) delete process.env.MEGACOMPACT_TIER;
+	process.env.MEGACOMPACT_FAST_GATE_PCT = "1";
+
+	const handlers: Record<string, Function[]> = {};
+	const appended: any[] = [];
+	const sendUserMessages: string[] = [];
+	const compactCalls: any[] = [];
+
+	const session: AgentMessage[] = [
+		{ role: "user", content: "read src/vec.ts and understand the index", timestamp: 0 } as unknown as AgentMessage,
+		{ role: "assistant", content: [{ type: "toolCall", name: "Read", id: "c1", arguments: {} }], api: "anthropic-messages", provider: "anthropic", model: "m", usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 }, stopReason: "tool_use", timestamp: 0 } as unknown as AgentMessage,
+		{ role: "user", content: "edit src/vec.ts to add a cosine helper", timestamp: 0 } as unknown as AgentMessage,
+		{ role: "assistant", content: [{ type: "toolCall", name: "Edit", id: "c1", arguments: {} }], api: "anthropic-messages", provider: "anthropic", model: "m", usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 }, stopReason: "tool_use", timestamp: 0 } as unknown as AgentMessage,
+	];
+	const toEntry = (m: AgentMessage, i: number): any => ({ type: "message", id: `e${i}`, parentId: null, timestamp: String(i), message: m });
+	const sessionManager = { getSessionId: () => "sess_ext_001", getEntries: () => session.map(toEntry), getBranch: () => session.map(toEntry) };
+
+	function makeCtx(over: Partial<any> = {}) {
+		return {
+			ui: { setStatus: () => {}, notify: () => {}, select: () => {}, confirm: async () => true, input: async () => "", setWidget: () => {} },
+			mode: "tui" as any, hasUI: true, cwd: stateDir, sessionManager,
+			modelRegistry: {} as any, model: undefined,
+			isIdle: () => true, isProjectTrusted: () => true,
+			signal: undefined, abort: () => {}, hasPendingMessages: () => false, shutdown: () => {},
+			getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
+			compact: (opts?: any) => {
+				compactCalls.push(opts);
+				const _sbc = handlers["session_before_compact"];
+				if (_sbc && _sbc.length) return _sbc[0]({ type: "session_before_compact", reason: "threshold", willRetry: false, signal: undefined, preparation: { firstKeptEntryId: "e2", messagesToSummarize: session.slice(0, 2), tokensBefore: 500 } } as any, makeCtx());
+				return undefined;
+			},
+			getSystemPrompt: () => "system base", ...over,
+		} as any;
+	}
+	const pi = {
+		on: (ev: string, h: Function) => { if (!handlers[ev]) handlers[ev] = []; handlers[ev].push(h); },
+		registerCommand: () => {}, registerTool: () => {}, registerShortcut: () => {},
+		registerFlag: () => {}, getFlag: () => undefined,
+		registerMessageRenderer: () => {}, registerEntryRenderer: () => {},
+		sendMessage: () => {}, sendUserMessage: (m: string) => { sendUserMessages.push(m); },
+		appendEntry: (t: string, d: any) => appended.push({ t, d }),
+		setSessionName: () => {}, getSessionName: () => undefined, setLabel: () => {},
+		exec: async () => ({ stdout: "", stderr: "", code: 0 }),
+		getActiveTools: () => [], getAllTools: () => {}, setActiveTools: () => {},
+		getCommands: () => [], setModel: async () => false,
+		getThinkingLevel: () => "off" as any, setThinkingLevel: () => {},
+	} as any;
+	const mod = require("../mega-compact.js") as { default: (p: any) => void };
+	mod.default(pi);
+	return {
+		stateDir, handlers, appended, compactCalls, sendUserMessages,
+		fire: async (ev: string, event: any, ctx: any) => { let r: any; for (const h of handlers[ev] || []) r = await h(event, ctx); return r; },
+		ctx: makeCtx, session,
+	};
+}
+
+export type Harness = ReturnType<typeof harness>;
+
+export const { classifyError: classifyErrorFn, classifyErrorDetailed: classifyErrorDetailedFn, extractErrorSignature: extractErrorSignatureFn } =
+	require("../mega-events.js") as { classifyError: typeof import("../mega-events.js").classifyError; classifyErrorDetailed: typeof import("../mega-events.js").classifyErrorDetailed; extractErrorSignature: typeof import("../mega-events.js").extractErrorSignature };
+
+/** S38 helper: fire a turn_end with a given stopReason + optional text, using a
+ *  low-pressure ctx so the durable-trim branch (ctx.compact) does NOT fire. */
+export async function s38TurnEnd(h: Harness, stopReason: string | undefined, text?: string) {
+	const lowCtx = h.ctx({ isIdle: () => true, hasPendingMessages: () => false, getContextUsage: () => ({ tokens: 100, contextWindow: 200000, percent: 0 }) });
+	const message: any = { role: "assistant" };
+	if (stopReason !== undefined) message.stopReason = stopReason;
+	if (text) message.content = text;
+	await h.fire("turn_end", { type: "turn_end", turnIndex: 1, message }, lowCtx);
+}
+
+/** R3 helper: like s38TurnEnd but attaches a `usage` object so the classifier's
+ *  0-token poisoned-context signal is exercised (usage PRESENT with 0 tokens).
+ *  Pass `tokens` > 0 to simulate a turn that reached the model. */
+export async function s38TurnEndUsage(h: Harness, stopReason: string | undefined, text: string | undefined, tokens: number) {
+	const lowCtx = h.ctx({ isIdle: () => true, hasPendingMessages: () => false, getContextUsage: () => ({ tokens: 100, contextWindow: 200000, percent: 0 }) });
+	const message: any = { role: "assistant", usage: { inputTokens: tokens, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } };
+	if (stopReason !== undefined) message.stopReason = stopReason;
+	if (text) message.content = text;
+	await h.fire("turn_end", { type: "turn_end", turnIndex: 1, message }, lowCtx);
+}
+
+/** R7 helper: fire the same transient error text `count` times with turn_start
+ *  + elapsed backoff between turns so each turn's nudge can fire. */
+export async function r7RepeatTurns(h: Harness, text: string, count: number) {
+	for (let i = 0; i < count; i++) {
+		await s38TurnEnd(h, "error", text);
+		await h.fire("turn_start", { type: "turn_start", turnIndex: i + 2 }, h.ctx());
+		await new Promise((r) => setTimeout(r, 3));
+	}
+}
+
+/** R7 helper: shared assertions for "stays transient" — no poisoned upgrade,
+ *  no /clear advise, and the transient retry path actually ran. */
+export function assertStaysTransient(h: Harness, label: string) {
+	assert.ok(
+		!eventTypes(h.stateDir).includes("poisoned_context"),
+		`${label}: no poisoned_context event (network errors never upgrade)`,
+	);
+	assert.ok(
+		!h.sendUserMessages.some((m) => m.includes("/clear")),
+		`${label}: no /clear advise message`,
+	);
+	assert.ok(
+		eventTypes(h.stateDir).includes("error_retry"),
+		`${label}: transient retry path ran (error_retry event)`,
+	);
+}
+
+/** The exact error bodies from the 2026-07-30 incident (GLM router flapping). */
+export const R8_NO_HEALTHY_TARGET =
+	'{"message":"No healthy target selected for alias \'hf:zai-org/GLM-4.7\'","type":"api_error"}';
+export const R8_SOCKET_CLOSED =
+	"All targets failed: modal/zai-org/GLM-5.1-FP8. Last error: The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()"; // guardrails-allow PREVENT-PI-004: verbatim 2026-07-30 incident error text (string fixture, not a network call)
+export const R8_TOO_MANY_CONCURRENT =
+	'All targets failed: modal/zai-org/GLM-5.1-FP8. Last error: {"error": "Too many concurrent requests for this model"}';

package/extensions/mega-compact.test/_helpers.ts

@@ -0,0 +1,237 @@
+/**
+ * Shared harness for the mega-compact split test files.
+ *
+ * Extracted from the top of mega-compact.test.ts: the mock-pi harness that loads
+ * the compiled extension and lets tests fire event/command handlers against it.
+ * Each split test file imports `harness()` (and `require`/`baseTmp` where
+ * needed) so the per-test isolated state-dir environment is preserved.
+ *
+ * Note: relative paths are one directory deeper than the original file, hence
+ * `../mega-compact.js` for the extension entry and `../../src/...` for src.
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createRequire } from "node:module";
+import { closeVectorIndex } from "../../src/store/vectorIndex.js";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+
+export const require = createRequire(import.meta.url);
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-ext-"));
+// Isolate the machine-wide repo index so test runs (which call bindRepo ->
+// upsertRepoRegistry) never pollute the developer's real ~/.mega-compact-index.
+process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
+// This end-to-end extension test drives the compact/recall flow through the
+// synchronous node:sqlite store (the authoritative path). The redundant PGlite
+// HNSW cross-repo index is additive and exercised in its own dedicated test
+// (store/vectorIndex.test.ts); starting the WASM worker here only risks the
+// teardown stall that hangs node --test on some machines. Disable it for this
+// suite — the extension behavior under test is identical with it off.
+process.env.MEGACOMPACT_PGLITE_DISABLED = "1";
+export const baseTmpDir = baseTmp;
+export { closeVectorIndex };
+let counter = 0;
+
+/** Build a mock pi + ctx and load the extension into them. */
+export function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
+	const stateDir = join(baseTmp, `run-${counter++}`);
+	process.env.MEGACOMPACT_STATE_DIR = stateDir;
+	process.env.MEGACOMPACT_DEBUG = "true";
+	// Low threshold so the auto-trigger gate trips on our small mock context.
+	// Tier tests opt out (keepTier/keepThreshold) so they can drive the real
+	// tier resolution instead of the forced 50-token threshold.
+	if (!opts.keepThreshold) process.env.MEGACOMPACT_THRESHOLD_TOKENS = "50";
+	if (!opts.keepTier) delete process.env.MEGACOMPACT_TIER;
+	process.env.MEGACOMPACT_FAST_GATE_PCT = "1";
+
+	const handlers: Record<string, Function[]> = {};
+	const commands: Record<
+		string,
+		{ handler: (a: string, c: any) => Promise<void> }
+	> = {};
+	const appended: any[] = [];
+	let statusKey: string | undefined;
+	let statusText: string | undefined;
+	const notifies: string[] = [];
+	const compactCalls: any[] = [];
+	const sendUserMessages: string[] = [];
+
+	// Minimal AgentMessage factory for the session we project into the extension.
+	function msg(role: string, text: string, toolName?: string): AgentMessage {
+		if (role === "assistant" && toolName) {
+			return {
+				role: "assistant",
+				content: [
+					{ type: "toolCall", name: toolName, id: "c1", arguments: {} },
+				],
+				api: "anthropic-messages",
+				provider: "anthropic",
+				model: "m",
+				usage: {
+					inputTokens: 1,
+					outputTokens: 1,
+					cacheReadTokens: 0,
+					cacheWriteTokens: 0,
+				},
+				stopReason: "tool_use",
+				timestamp: 0,
+			} as unknown as AgentMessage;
+		}
+		if (role === "toolResult" && toolName) {
+			return {
+				role: "toolResult",
+				toolCallId: "c1",
+				toolName,
+				content: [{ type: "text", text }],
+				isError: false,
+				timestamp: 0,
+			} as unknown as AgentMessage;
+		}
+		return {
+			role: "user",
+			content: text,
+			timestamp: 0,
+		} as unknown as AgentMessage;
+	}
+
+	const session: AgentMessage[] = [
+		msg("user", "read src/vec.ts and understand the index"),
+		msg("assistant", "ok", "Read"),
+		msg("user", "edit src/vec.ts to add a cosine helper"),
+		msg("assistant", "ok", "Edit"),
+		msg("user", "now fix the dedupe bug in store.ts"),
+		msg("assistant", "ok", "Edit"),
+		msg("user", "actually we should add recall sorting too"),
+		msg("assistant", "ok", "Edit"),
+	];
+
+	// Mirror the REAL SessionManager: getEntries() returns SessionEntry objects,
+	// which the extension projects to messages via the SDK's
+	// sessionEntryToContextMessages(entry). The harness must use the same shape
+	// (type:"message" with a .message) or recentUserQuery() silently queries "".
+	const toEntry = (m: AgentMessage, i: number): any => ({
+		type: "message",
+		id: `e${i}`,
+		parentId: null,
+		timestamp: String(i),
+		message: m,
+	});
+	const sessionManager = {
+		getSessionId: () => "sess_ext_001",
+		getEntries: () => session.map(toEntry),
+		// Faithful mock: getBranch() returns the current branch's entries, which
+		// piCompactWouldNoop() reads to predict whether ctx.compact() would no-op.
+		getBranch: () => session.map(toEntry),
+	};
+
+	function makeCtx(over: Partial<any> = {}) {
+		return {
+			ui: {
+				setStatus: (k: string, t: string | undefined) => {
+					statusKey = k;
+					statusText = t;
+				},
+				notify: (s: string) => notifies.push(s),
+				select: () => {},
+				confirm: async () => true,
+				input: async () => "",
+				setWidget: () => {},
+			},
+			mode: "tui" as any,
+			hasUI: true,
+			cwd: stateDir,
+			sessionManager,
+			modelRegistry: {} as any,
+			model: undefined,
+			isIdle: () => true,
+			isProjectTrusted: () => true,
+			signal: undefined,
+			abort: () => {},
+			hasPendingMessages: () => false,
+			shutdown: () => {},
+			getContextUsage: () => ({
+				tokens: 200000,
+				contextWindow: 200000,
+				percent: 100,
+			}),
+			// Faithful mock: ctx.compact() starts pi's flow, which fires the
+			// session_before_compact handler (where WE supply the durable trim).
+			compact: (opts?: any) => {
+				compactCalls.push(opts);
+				const _sbc = handlers["session_before_compact"]; if (_sbc && _sbc.length) {
+					return _sbc[0](
+						{
+							type: "session_before_compact",
+							reason: "threshold",
+							willRetry: false,
+							signal: undefined,
+							// pi computed the cut honoring anchor floor + tool-pair (PREVENT-PI-002);
+							// our handler reuses it as firstKeptEntryId.
+							preparation: {
+								firstKeptEntryId: "e2",
+								messagesToSummarize: session.slice(0, 2),
+								tokensBefore: 500,
+							},
+						} as any,
+						makeCtx(),
+					);
+				}
+				return undefined;
+			},
+			getSystemPrompt: () => "system base",
+			...over,
+		} as any;
+	}
+
+	const pi = {
+		on: (ev: string, h: Function) => {
+			if (!handlers[ev]) handlers[ev] = [];
+			handlers[ev].push(h);
+		},
+		registerCommand: (name: string, opts: any) => {
+			commands[name] = opts;
+		},
+		registerTool: () => {},
+		registerShortcut: () => {},
+		registerFlag: () => {},
+		getFlag: () => undefined,
+		registerMessageRenderer: () => {},
+		registerEntryRenderer: () => {},
+		sendMessage: (_m: any) => {},
+		sendUserMessage: (m: string) => {
+			sendUserMessages.push(m);
+		},
+		appendEntry: (t: string, d: any) => appended.push({ t, d }),
+		setSessionName: () => {},
+		getSessionName: () => undefined,
+		setLabel: () => {},
+		exec: async () => ({ stdout: "", stderr: "", code: 0 }),
+		getActiveTools: () => [],
+		getAllTools: () => [],
+		setActiveTools: () => {},
+		getCommands: () => [],
+		setModel: async () => false,
+		getThinkingLevel: () => "off" as any,
+		setThinkingLevel: () => {},
+	} as any;
+
+	// Import the compiled extension (same dist dir as this test's parent).
+	const mod = require("../mega-compact.js") as { default: (p: any) => void };
+	mod.default(pi);
+
+	return {
+		stateDir,
+		handlers,
+		commands,
+		appended,
+		get status() {
+			return { statusKey, statusText };
+		},
+		notifies,
+		compactCalls,
+		sendUserMessages,
+		fire: async (ev: string, event: any, ctx: any) => { let r: any; for (const h of handlers[ev] || []) r = await h(event, ctx); return r; },
+		ctx: makeCtx,
+		session,
+	};
+}

package/extensions/mega-compact.ts

@@ -29,7 +29,7 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { closeVectorIndex } from "../src/store/vectorIndex.js";
 import { closeMemoryIndex } from "../src/store/memoryIndex.js";
-import { loadConfig } from "./mega-config.js";
+import { loadConfig, repoStateDir } from "./mega-config.js";
 import { loadMegaEnv } from "./mega-runtime/env-loader.js";
 import { MegaRuntime } from "./mega-runtime.js";
 import { registerEventHandlers } from "./mega-events.js";
@@ -45,7 +45,17 @@ export default function (pi: ExtensionAPI) {
 	// Load .mega-compact.env (written by /mega-setup + dashboard Setup tab) BEFORE
 	// config resolution so the wizard's choices take effect. Shell env wins over
 	// the file (see env-loader.ts). Non-fatal: missing/malformed file is a no-op.
-	loadMegaEnv(loadConfig().stateDir);
+	//
+	// The dashboard Setup tab + /mega-setup write the file to the PER-REPO state
+	// dir (<repo>/.pi/mega-compact, resolved via repoStateDir), NOT the global
+	// default (~/.pi/agent/extensions/pi-mega-compact). Loading from the global
+	// dir was the root cause of the embedder-never-activates bug: the file sat in
+	// the per-repo dir, the loader read the empty global dir, and the configured
+	// embedder silently never loaded after restart. Load per-repo first (where
+	// the wizard writes), then the global dir as a legacy fallback.
+	const globalStateDir = loadConfig().stateDir;
+	loadMegaEnv(repoStateDir(process.cwd(), globalStateDir));
+	loadMegaEnv(globalStateDir);
 	const config = loadConfig();
 	// S38.9: preflight env validation — check for obviously invalid values at startup.
 	// Non-fatal: log warnings and fall back to defaults.