pi-mega-compact 0.13.0 → 0.13.2

package/extensions/dashboard-server/api-contracts/index.ts

@@ -211,6 +211,9 @@ export type { MemoryStatusResponse, MemoryStatsTopMemory } from "./memory.js";
 // Setup wizard — embedder configuration + backend detection
 export type { SetupStatusResponse, SetupDetectResponse, SetupConfigureRequest, SetupConfigureResponse, DetectResult, OllamaDetectResult } from "./setup.js";
 
+// Embedder health probe
+export type { EmbedderHealthResponse } from "./embedder-health.js";
+
 // S46/D3 — memory graph (visual memory map) + validation report
 export type {
 	MemoryMapResponse,
@@ -220,6 +223,9 @@ export type {
 	GraphValidationReport,
 } from "./memory-map.js";
 
+// Part B — RAPTOR tree
+export type { RaptorNodeDTO, RaptorTreeResponse } from "./raptor.js";
+
 // A3 — cache stripe distribution + health
 export type {
 	StripeBucket,

package/extensions/dashboard-server/api-contracts/raptor.ts

@@ -0,0 +1,49 @@
+/**
+ * api-contracts/raptor.ts — RAPTOR tree API contract.
+ *
+ * Response shape for GET /api/raptor-tree: the hierarchical RAPTOR tree
+ * (summary nodes by level) for a session. Client-safe — the raw 512-dim
+ * embedding vector is stripped before sending.
+ */
+
+/**
+ * A single RAPTOR tree node (client-safe — no `embedding` vector).
+ * Mirrors src/store/sqlite/raptor.ts StoredRaptorNode minus the embedding.
+ */
+export interface RaptorNodeDTO {
+	/** Stable node id (persisted in raptor_nodes.id). */
+	readonly id: string;
+	/** Session this node belongs to. */
+	readonly sessionId: string;
+	/** Tree depth level (0 = root summary, higher = more granular). */
+	readonly level: number;
+	/** Parent node id, or null for the root of the tree. */
+	readonly parentId: string | null;
+	/** Child node ids. */
+	readonly children: string[];
+	/** Summarized content of this cluster. */
+	readonly summary: string;
+	/** Quality marker of the summary (e.g. "high" | "med" | "low"). */
+	readonly qualityMarker: string;
+	/** Estimated token count of the summary. */
+	readonly tokenEstimate: number;
+	/** Epoch ms when the tree containing this node was built. */
+	readonly builtAt: number;
+}
+
+/**
+ * Response body for GET /api/raptor-tree.
+ * `empty: true` when no RAPTOR nodes exist for the resolved session.
+ */
+export interface RaptorTreeResponse {
+	/** Flat list of all nodes (tree shape recoverable via parentId/children). */
+	readonly nodes: RaptorNodeDTO[];
+	/** Max level across nodes (0 when empty). */
+	readonly levels: number;
+	/** The root node id (node with parentId === null), or null when empty. */
+	readonly rootId: string | null;
+	/** Max builtAt across nodes (epoch ms), or null when empty. */
+	readonly builtAt: number | null;
+	/** True when there are no RAPTOR nodes for the resolved session. */
+	readonly empty: boolean;
+}

package/extensions/dashboard-server/api-contracts.test/_helpers.ts

@@ -0,0 +1,47 @@
+/**
+ * Shared runtime-helpers for the api-contracts split test files.
+ *
+ * Extracted from api-contracts.test.ts so each per-endpoint describe file can
+ * reuse the same field/type assertion primitives without duplicating them.
+ * Only runtime helpers live here — compile-time `satisfies` checks and the
+ * ENDPOINTS registry assertions stay in endpoints-registry.test.ts.
+ */
+import assert from "node:assert/strict";
+
+/** Primitive type name from a JSON-parsed value (matches typeof for JSON types). */
+export function primitiveType(v: unknown): string {
+	if (v === null) return "null";
+	if (Array.isArray(v)) return "array";
+	return typeof v;
+}
+
+/** Assert that a field exists on an object and has one of the expected primitive types. */
+export function assertField(
+	obj: Record<string, unknown>,
+	field: string,
+	expected: string[],
+): void {
+	assert.ok(field in obj, `field "${field}" must exist`);
+	const actual = primitiveType(obj[field]);
+	assert.ok(
+		expected.includes(actual),
+		`field "${field}" expected type ${expected.join("|")}, got ${actual}`,
+	);
+}
+
+/** Assert that a nested object field exists and is an object (or null if allowed). */
+export function assertObject(
+	obj: Record<string, unknown>,
+	field: string,
+	allowNull: boolean = false,
+): Record<string, unknown> | null {
+	assert.ok(field in obj, `field "${field}" must exist`);
+	const val = obj[field];
+	if (allowNull && val === null) return null;
+	assert.equal(
+		primitiveType(val),
+		"object",
+		`field "${field}" must be an object${allowNull ? " or null" : ""}`,
+	);
+	return val as Record<string, unknown>;
+}

package/extensions/dashboard-server/routes-cache.test/_helpers.ts

@@ -0,0 +1,94 @@
+/**
+ * Shared harness + types for the routes-cache split files.
+ * Extracted from routes-cache.test.ts: spawn-and-fetch server harness + DTOs.
+ */
+import { mkdtempSync, rmSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawn } from "node:child_process";
+
+const SERVER_ENTRY = new URL("../server.js", import.meta.url).pathname;
+
+export function waitFor(
+	cond: () => boolean | Promise<boolean>,
+	timeoutMs = 6000,
+): Promise<void> {
+	const start = Date.now();
+	return new Promise((resolve, reject) => {
+		const tick = async () => {
+			if (await cond()) return resolve();
+			if (Date.now() - start > timeoutMs) return reject(new Error("timeout"));
+			setTimeout(tick, 50);
+		};
+		tick();
+	});
+}
+
+export function freshDir(prefix: string): string {
+	return mkdtempSync(join(tmpdir(), prefix));
+}
+
+export async function withServer<T>(
+	port: string,
+	dir: string,
+	fn: (port: number) => Promise<T>,
+): Promise<T> {
+	process.env.MEGACOMPACT_DASHBOARD_PORT = port;
+	const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
+	try {
+		await waitFor(async () => {
+			try {
+				const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+				const res = await fetch(`http://localhost:${raw.port}/api/version`);
+				return res.ok;
+			} catch {
+				return false;
+			}
+		});
+		const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+		return await fn(raw.port);
+	} finally {
+		child.kill("SIGTERM");
+		delete process.env.MEGACOMPACT_DASHBOARD_PORT;
+		rmSync(dir, { recursive: true, force: true });
+	}
+}
+
+export interface ProviderCacheByModel {
+	model: string;
+	hitPct: number;
+	totalCacheRead: number;
+	totalCacheWrite: number;
+	sampleCount: number;
+}
+
+export interface ProviderCacheResponse {
+	cache: {
+		avgHitPct: number;
+		turnCount: number;
+		totalCacheRead: number;
+		totalCacheWrite: number;
+		totalInput: number;
+		firstTurnAt: string | null;
+		latestTurnAt: string | null;
+		byModel: ProviderCacheByModel[];
+	};
+	savings: {
+		cacheReadSaved: number;
+		cacheWriteCost: number;
+		netSaved: number;
+		model: string;
+		inputRate: number;
+	} | null;
+	updatedAt: string;
+	windowMinutes?: number | null;
+	prefixBreaks: Array<{
+		id: number;
+		ts: number;
+		cause: string;
+		confidence: number;
+		prevHitPct: number;
+		currHitPct: number;
+		breakAt: number;
+	}>;
+}

package/extensions/dashboard-server/routes-embedder-health.ts

@@ -0,0 +1,77 @@
+/**
+ * dashboard-server/routes-embedder-health.ts — Embedder health probe route.
+ *
+ * GET /api/embedder-health — Round-trips a test embed through the active
+ * embedder and reports reachability + latency + dimensions.
+ *
+ * PREVENT-PI-004: probes the active embedder, which is either the built-in
+ * TrigramEmbedder (pure local math) or the HttpEmbedder pointed at a
+ * user-spawned localhost server — the existing audited loopback exception
+ * (see httpEmbedder.ts). No new network paths are introduced here.
+ */
+
+import { performance } from "node:perf_hooks";
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RouteContext } from "./routes-core.js";
+import { detectCurrentEmbedder } from "./routes-setup.js";
+import { defaultEmbedder } from "../../src/embedder.js";
+import type { EmbedderHealthResponse } from "./api-contracts/embedder-health.js";
+
+/** Mask an embedding URL down to scheme://hostname:port, stripping path +
+ *  any embedded credentials. Returns null when the env URL is unset/empty. */
+function maskEmbeddingUrl(): string | null {
+	const raw = process.env["MEGACOMPACT_EMBEDDING_URL"];
+	if (!raw || raw.trim().length === 0) return null;
+	try {
+		const u = new URL(raw);
+		return `${u.protocol}//${u.hostname}${u.port ? `:${u.port}` : ""}`;
+	} catch {
+		return null;
+	}
+}
+
+export function handleEmbedderHealth(
+	req: IncomingMessage,
+	res: ServerResponse,
+	_ctx: RouteContext,
+): boolean {
+	if (req.url !== "/api/embedder-health") return false;
+	if (req.method !== "GET") {
+		// guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+		res.writeHead(405, { "Content-Type": "application/json" });
+		res.end(JSON.stringify({ error: "method_not_allowed" }));
+		return true;
+	}
+
+	const activeEmbedder = detectCurrentEmbedder();
+	const url = maskEmbeddingUrl();
+
+	let status: "ok" | "unreachable" | "error" = "ok";
+	let dim = 0;
+	let error: string | undefined;
+	const start = performance.now();
+	try {
+		const embedder = defaultEmbedder();
+		embedder.embed("mega-compact health probe");
+		dim = embedder.dim;
+	} catch (e) {
+		const msg = e instanceof Error ? e.message : String(e);
+		status = /unreachable/i.test(msg) ? "unreachable" : "error";
+		error = msg;
+	}
+	const latencyMs = performance.now() - start;
+
+	const body: EmbedderHealthResponse = {
+		activeEmbedder,
+		status,
+		latencyMs,
+		dim,
+		url,
+		...(error !== undefined ? { error } : {}),
+	};
+
+	// guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+	res.writeHead(200, { "Content-Type": "application/json" });
+	res.end(JSON.stringify(body));
+	return true;
+}

package/extensions/dashboard-server/routes-raptor.ts

@@ -0,0 +1,128 @@
+/**
+ * routes-raptor.ts — GET /api/raptor-tree route handler (Part B).
+ *
+ * Returns the hierarchical RAPTOR tree (summary nodes by level) for a session.
+ * When no sessionId is given, resolves to the most recent session that has
+ * raptor nodes. Reads only from the local SQLite store (PREVENT-PI-004 OK).
+ * Uses an inline require to the pi-agnostic src/store/sqlite/raptor.ts reader.
+ */
+import { createRequire } from "node:module";
+import type { ServerResponse, IncomingMessage } from "node:http";
+import { parse as parseUrl } from "node:url";
+import type { RouteContext } from "./routes-core.js";
+import type { RaptorTreeResponse, RaptorNodeDTO } from "./api-contracts/raptor.js";
+import { openStore } from "../../src/store/sqlite.js";
+
+const _req = createRequire(import.meta.url);
+
+// guardrails-allow PREVENT-PI-004: local loopback-only dashboard endpoint — reads the local SQLite store for RAPTOR tree data, zero remote network calls.
+
+/** Full node shape returned by listRaptorNodes (includes the embedding vector). */
+type StoredRaptorNode = {
+	id: string;
+	sessionId: string;
+	level: number;
+	parentId: string | null;
+	children: string[];
+	summary: string;
+	embedding: number[];
+	qualityMarker: string;
+	tokenEstimate: number;
+	builtAt: number;
+};
+
+/** Find the most recent sessionId that has raptor nodes (else null). */
+function latestRaptorSession(stateDir: string): string | null {
+	try {
+		const db = openStore(stateDir);
+		const row = db
+			.prepare(
+				"SELECT DISTINCT session_id FROM raptor_nodes ORDER BY built_at DESC LIMIT 1",
+			)
+			.get() as { session_id: string } | undefined;
+		return row?.session_id ?? null;
+	} catch {
+		// no raptor_nodes table yet (fresh DB) — treat as empty
+		return null;
+	}
+}
+
+function toDTO(node: StoredRaptorNode): RaptorNodeDTO {
+	return {
+		id: node.id,
+		sessionId: node.sessionId,
+		level: node.level,
+		parentId: node.parentId,
+		children: node.children,
+		summary: node.summary,
+		qualityMarker: node.qualityMarker,
+		tokenEstimate: node.tokenEstimate,
+		builtAt: node.builtAt,
+	};
+}
+
+function emptyResponse(): RaptorTreeResponse {
+	return { nodes: [], levels: 0, rootId: null, builtAt: null, empty: true };
+}
+
+export function handleRaptorTree(
+	req: IncomingMessage,
+	res: ServerResponse,
+	ctx: RouteContext,
+): boolean {
+	if (req.method !== "GET") return false;
+	const parsed = parseUrl(req.url ?? "", true);
+	if (parsed.pathname !== "/api/raptor-tree") return false;
+
+	try {
+		const { listRaptorNodes } = _req(
+			"../../src/store/sqlite/raptor.js",
+		) as {
+			listRaptorNodes: (
+				sessionId: string,
+				stateDir: string,
+			) => StoredRaptorNode[];
+		};
+
+		let sessionId =
+			typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : "";
+
+		if (!sessionId) {
+			sessionId = latestRaptorSession(ctx.stateDir) ?? "";
+			if (!sessionId) {
+				const body = emptyResponse();
+				res.writeHead(200, { "Content-Type": "application/json" });
+				res.end(JSON.stringify(body));
+				return true;
+			}
+		}
+
+		const nodes = listRaptorNodes(sessionId, ctx.stateDir);
+		if (nodes.length === 0) {
+			const body = emptyResponse();
+			res.writeHead(200, { "Content-Type": "application/json" });
+			res.end(JSON.stringify(body));
+			return true;
+		}
+
+		const dtos: RaptorNodeDTO[] = nodes.map(toDTO);
+		const levels = dtos.reduce((m, n) => Math.max(m, n.level), 0);
+		const root = dtos.find((n) => n.parentId === null);
+		const builtAt = dtos.reduce((m, n) => Math.max(m, n.builtAt), 0);
+		const body: RaptorTreeResponse = {
+			nodes: dtos,
+			levels,
+			rootId: root?.id ?? null,
+			builtAt,
+			empty: false,
+		};
+
+		res.writeHead(200, { "Content-Type": "application/json" });
+		res.end(JSON.stringify(body));
+		return true;
+	} catch (e) {
+		res.writeHead(500, { "Content-Type": "application/json" });
+		res.end(JSON.stringify({ error: String(e) }));
+		return true;
+	}
+}

package/extensions/dashboard-server/routes-setup.ts

@@ -28,7 +28,7 @@ import type {
 // handleSetupStatus — "/api/setup-status"
 // ---------------------------------------------------------------------------
 
-function detectCurrentEmbedder(): SetupStatusResponse["currentEmbedder"] {
+export function detectCurrentEmbedder(): SetupStatusResponse["currentEmbedder"] {
 	const url = process.env["MEGACOMPACT_EMBEDDING_URL"];
 	const minilm = process.env["MEGACOMPACT_MINILM"];
 	if (url && url.trim().length > 0) return "http";

package/extensions/dashboard-server/routes.ts

@@ -23,5 +23,7 @@ export { handleProviderCache } from "./routes-cache.js";
 export { handleMemoryStatus } from "./routes-memory.js";
 export { handleSetupStatus, handleSetupDetect, handleSetupConfigure } from "./routes-setup.js";
 export { handleMemoryMap } from "./routes-memory-map.js";
+export { handleRaptorTree } from "./routes-raptor.js";
 export { handleCacheStripes } from "./routes-cache.js";
 export { handleContextHealth, handleCachePoison, handleHealthSettings } from "./routes-health.js";
+export { handleEmbedderHealth } from "./routes-embedder-health.js";

package/extensions/dashboard-server/server.ts

@@ -44,9 +44,11 @@ import {
 	handleSetupDetect,
 	handleSetupConfigure,
 	handleMemoryMap,
+	handleRaptorTree,
 	handleContextHealth,
 	handleCachePoison,
 	handleHealthSettings,
+	handleEmbedderHealth,
 	handleStatic,
 } from "./routes.js";
 
@@ -242,9 +244,11 @@ export async function launchDashboardServer(
 		if (handleSetupDetect(req, res, ctx)) return;
 		if (handleSetupConfigure(req, res, ctx)) return;
 		if (handleMemoryMap(req, res, ctx)) return;
+		if (handleRaptorTree(req, res, ctx)) return;
 		if (handleContextHealth(req, res, ctx)) return;
 		if (handleCachePoison(req, res, ctx)) return;
 		if (handleHealthSettings(req, res, ctx)) return;
+		if (handleEmbedderHealth(req, res, ctx)) return;
 		handleStatic(req, res, ctx);
 	});
 

package/extensions/mega-cache-replay.test/_helpers.ts

@@ -0,0 +1,188 @@
+/**
+ * Shared helpers for the mega-cache-replay.test split files.
+ * Extracted from extensions/mega-cache-replay.test.ts: baseTmp, env, harness().
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createRequire } from "node:module";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+
+export const baseTmp = mkdtempSync(join(tmpdir(), "mc-cache-"));
+
+let counter = 0;
+
+export function setupEnv(): void {
+  process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
+  process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
+}
+
+export function harness() {
+  const stateDir = join(baseTmp, `run-${counter++}`);
+  process.env.MEGACOMPACT_STATE_DIR = stateDir;
+  process.env.MEGACOMPACT_DEBUG = "true";
+  process.env.MEGACOMPACT_THRESHOLD_TOKENS = "50";
+  process.env.MEGACOMPACT_FAST_GATE_PCT = "1";
+  process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES = "1";
+  process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR = "0";
+  process.env.MEGACOMPACT_MEMORY_AUTO_REVIEW = "false";
+  process.env.MEGACOMPACT_RAPTOR_ENABLED = "false";
+  process.env.MEGACOMPACT_L1_ENABLED = "false";
+  process.env.MEGACOMPACT_L2_ENABLED = "false";
+  delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
+
+  const usage = {
+    tokens: 200000,
+    contextWindow: 200000,
+    percent: 100 as number | null,
+  };
+
+  const handlers: Record<string, Function[]> = {};
+  const compactCalls: any[] = [];
+
+  function msg(role: string, text: string, toolName?: string): AgentMessage {
+    if (role === "assistant" && toolName) {
+      return {
+        role: "assistant",
+        content: [
+          { type: "toolCall", name: toolName, id: "c1", arguments: {} },
+        ],
+        api: "anthropic-messages",
+        provider: "anthropic",
+        model: "m",
+        usage: {
+          inputTokens: 1,
+          outputTokens: 1,
+          cacheReadTokens: 0,
+          cacheWriteTokens: 0,
+        },
+        stopReason: "tool_use",
+        timestamp: 0,
+      } as unknown as AgentMessage;
+    }
+    if (role === "toolResult" && toolName) {
+      return {
+        role: "toolResult",
+        content: [{ type: "text", text }],
+        toolCallId: "c1",
+        toolName,
+        isError: false,
+        timestamp: 0,
+      } as unknown as AgentMessage;
+    }
+    return {
+      role: "user",
+      content: text,
+      timestamp: 0,
+    } as unknown as AgentMessage;
+  }
+
+  function buildSession(tag: string, n: number): AgentMessage[] {
+    const s: AgentMessage[] = [];
+    for (let i = 0; i < n; i++) {
+      s.push(
+        msg("user", `[${tag}] we decided to use approach ${i} for module ${i}`),
+      );
+      s.push(msg("assistant", `[${tag}] edited module ${i}`, "Edit"));
+      s.push(msg("toolResult", `[${tag}] edited module ${i}`, "Edit"));
+    }
+    return s;
+  }
+
+  const toEntry = (m: AgentMessage, i: number): any => ({
+    type: "message",
+    id: `e${i}`,
+    parentId: null,
+    timestamp: String(i),
+    message: m,
+  });
+  const sessionManager = {
+    getSessionId: () => "sess_cache_001",
+    getEntries: () => buildSession("A", 14).map(toEntry),
+    getBranch: () => buildSession("A", 14).map(toEntry),
+  };
+
+  function makeCtx(over: Partial<any> = {}) {
+    return {
+      ui: {
+        setStatus: () => {},
+        notify: () => {},
+        select: () => {},
+        confirm: async () => true,
+        input: async () => "",
+        setWidget: () => {},
+      },
+      mode: "tui" as any,
+      hasUI: true,
+      cwd: stateDir,
+      sessionManager,
+      modelRegistry: {} as any,
+      model: undefined,
+      isIdle: () => true,
+      isProjectTrusted: () => true,
+      signal: undefined,
+      abort: () => {},
+      hasPendingMessages: () => false,
+      shutdown: () => {},
+      getContextUsage: () => ({ ...usage }),
+      compact: (opts?: any) => {
+        compactCalls.push(opts);
+        return undefined;
+      },
+      getSystemPrompt: () => "system base",
+      ...over,
+    } as any;
+  }
+
+  const pi = {
+    on: (ev: string, h: Function) => {
+      if (!handlers[ev]) handlers[ev] = [];
+      handlers[ev].push(h);
+    },
+    registerCommand: () => {},
+    registerTool: () => {},
+    registerShortcut: () => {},
+    registerFlag: () => {},
+    getFlag: () => undefined,
+    registerMessageRenderer: () => {},
+    registerEntryRenderer: () => {},
+    sendMessage: () => {},
+    sendUserMessage: () => {},
+    appendEntry: () => {},
+    setSessionName: () => {},
+    getSessionName: () => undefined,
+    setLabel: () => {},
+    exec: async () => ({ stdout: "", stderr: "", code: 0 }),
+    getActiveTools: () => [],
+    getAllTools: () => [],
+    setActiveTools: () => {},
+    getCommands: () => [],
+    setModel: async () => false,
+    getThinkingLevel: () => "off" as any,
+    setThinkingLevel: () => {},
+  } as any;
+
+  const require = createRequire(import.meta.url);
+  const mod = require("../mega-compact.js") as { default: (p: any) => void };
+  mod.default(pi);
+  const { lastRuntime } = require("../mega-events.js") as { lastRuntime: any };
+
+  const fire = async (ev: string, event: any, ctx: any) => {
+    let r: any;
+    for (const h of handlers[ev] || []) r = await h(event, ctx);
+    return r;
+  };
+  return {
+    stateDir,
+    handlers,
+    compactCalls,
+    fire,
+    ctx: makeCtx,
+    usage,
+    buildSession,
+    runtime: lastRuntime,
+    clearDebounce: () => {
+      if (lastRuntime) lastRuntime.debounceUntil = 0;
+    },
+  };
+}