pi-mega-compact 0.20.29 → 0.20.31

package/extensions/dashboard-server/routes-setup-detect-cache.ts

@@ -0,0 +1,224 @@
+/**
+ * dashboard-server/routes-setup-detect-cache.ts — memoized embedder-barrier
+ * detection (VC9D).
+ *
+ * The three raw fresh detectors (`detectOllama` / `detectLlamaCpp` /
+ * `detectOnnx`) moved verbatim out of routes-setup.ts — they are the source of
+ * truth for "is this embedder backend present". Each also has a memoized
+ * sibling (`memoizedDetectOllama` / `memoizedDetectLlamaCpp` /
+ * `memoizedDetectOnnx`) that runs the expensive spawn ONLY when its mutable
+ * input changes, matching the `healthFactCacheKey` invalidation semantics from
+ * routes-vector-cortex-health.ts:71-122: the cache key is the resolved binary
+ * path + its size/mtime (onnx: resolved module path + package mtime), so an
+ * install/upgrade on PATH or in node_modules naturally invalidates the entry,
+ * while a steady environment reuses the result instead of re-spawning.
+ *
+ * A single module-level slot per target is shared across every
+ * /api/setup-detect request, so consecutive polls reuse the result.
+ *
+ * Guardrails: PREVENT-PI-004 (local subprocess + filesystem reads only, no
+ * network), PREVENT-011 (no `any`). Each decoration line carries a
+ * guardrails-allow annotation.
+ */
+
+import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only
+import { createRequire } from "node:module";
+import { statSync } from "node:fs";
+import type { DetectResult, OllamaDetectResult } from "./api-contracts/setup.js";
+
+// ---------------------------------------------------------------------------
+// Raw fresh detectors (moved verbatim from routes-setup.ts — single source).
+// ---------------------------------------------------------------------------
+
+export function detectOllama(): OllamaDetectResult | null {
+	try {
+		const version = spawnSync("ollama", ["--version"], {
+			timeout: 5000,
+			encoding: "utf-8",
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+		if (version.status !== 0) {
+			return { installed: false, models: [], running: false, detail: version.stderr?.trim() || "not found" };
+		}
+		// Check for running server
+		let running = false;
+		let models: string[] = [];
+		try {
+			const listResult = spawnSync("ollama", ["list"], {
+				timeout: 5000,
+				encoding: "utf-8",
+				stdio: ["ignore", "pipe", "pipe"],
+			});
+			if (listResult.status === 0) {
+				running = true;
+				// Parse lines: NAME  ID  SIZE  MODIFIED
+				const lines = listResult.stdout?.split("\n") ?? [];
+				for (const line of lines) {
+					const name = line.split(/\s+/)[0];
+					if (name && name !== "NAME") models.push(name);
+				}
+			}
+		} catch {
+			// ollama list failed — server may not be running
+			running = false;
+		}
+		return {
+			installed: true,
+			models,
+			running,
+			detail: version.stdout?.trim() || null,
+		};
+	} catch {
+		return { installed: false, models: [], running: false, detail: "detection error" };
+	}
+}
+
+export function detectLlamaCpp(): DetectResult | null {
+	try {
+		const which = spawnSync("which", ["llama-server", "llama.cpp"], {
+			timeout: 3000,
+			encoding: "utf-8",
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+		const installed = which.status === 0 && which.stdout.trim().length > 0;
+		return {
+			installed,
+			detail: installed ? which.stdout.trim() : null,
+		};
+	} catch {
+		return { installed: false, detail: "detection error" };
+	}
+}
+
+export function detectOnnx(): DetectResult | null {
+	try {
+		// Check for onnxruntime-node in a parent node_modules
+		const req = createRequire(import.meta.url);
+		try {
+			req.resolve("onnxruntime-node");
+			return { installed: true, detail: "onnxruntime-node found in node_modules" };
+		} catch {
+			return { installed: false, detail: "onnxruntime-node not found in node_modules" };
+		}
+	} catch {
+		return { installed: false, detail: "detection error" };
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Detect memo slot — generic, deterministic, unit-testable.
+// ---------------------------------------------------------------------------
+
+export interface DetectMemoSlot<V> {
+	/** Return the stored value when `key` matches the cached key, else null. */
+	get(key: string): V | null;
+	/** Store a value under a key. */
+	set(key: string, value: V): void;
+	/** Drop the cached entry. */
+	clear(): void;
+}
+
+/** A single memo slot. `value` is never null for a detect result (objects only). */
+export function createDetectMemo<V>(): DetectMemoSlot<V> {
+	let key: string | null = null;
+	let value: V | null = null;
+	return {
+		get(k: string): V | null {
+			return key !== null && key === k ? value : null;
+		},
+		set(k: string, v: V): void {
+			key = k;
+			value = v;
+		},
+		clear(): void {
+			key = null;
+			value = null;
+		},
+	};
+}
+
+/**
+ * Run `compute` only when the derived key changes; a null key (inputs not
+ * resolvable — e.g. the binary is absent) is never cached, so every call runs
+ * the (cheap, fast-failing) detect fresh. This ordering is what the
+ * SETUP-CORTEX-030/031 fixtures pin: an unchanged key returns the stored value
+ * without re-running compute; a key mutation forces a recompute.
+ */
+export function withDetectMemo<V>(
+	slot: DetectMemoSlot<V>,
+	deriveKey: () => string | null,
+	compute: () => V,
+): V {
+	const key = deriveKey();
+	const cached = key !== null ? slot.get(key) : null;
+	if (cached !== null) return cached;
+	const value = compute();
+	if (key !== null) slot.set(key, value);
+	return value;
+}
+
+// ---------------------------------------------------------------------------
+// Mutable-input cache-key derivations.
+// ---------------------------------------------------------------------------
+
+/** Resolve the first PATH entry for a binary, or null when absent. */
+function whichBin(bin: string): string | null {
+	const r = spawnSync("which", [bin], {
+		timeout: 3000,
+		encoding: "utf-8",
+		// guardrails-allow PREVENT-PI-004: local which lookup only
+		stdio: ["ignore", "pipe", "pipe"],
+	});
+	if (r.status !== 0 || !r.stdout) return null;
+	const first = r.stdout.trim().split("\n")[0]?.trim();
+	return first && first.length > 0 ? first : null;
+}
+
+/** Key = resolved path + size + mtimeMs, so an install/upgrade invalidates. */
+function fileKey(p: string | null): string | null {
+	if (p === null || p.length === 0) return null;
+	try {
+		const s = statSync(p);
+		return `${p}:${s.size}:${s.mtimeMs}`;
+	} catch {
+		return null;
+	}
+}
+
+/** Key = resolved module path + its package.json mtime (node_modules rebuild). */
+function moduleKey(packageName: string): string | null {
+	try {
+		const req = createRequire(import.meta.url);
+		const resolved = req.resolve(packageName + "/package.json");
+		return fileKey(resolved);
+	} catch {
+		return null;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Memoized wrappers (shared module-level slots across requests).
+// ---------------------------------------------------------------------------
+
+const ollamaSlot = createDetectMemo<OllamaDetectResult | null>();
+const llamaSlot = createDetectMemo<DetectResult | null>();
+const onnxSlot = createDetectMemo<DetectResult | null>();
+
+const ollamaKey = (): string | null => fileKey(whichBin("ollama"));
+const llamaKey = (): string | null => fileKey(whichBin("llama-server") ?? whichBin("llama.cpp"));
+const onnxKey = (): string | null => moduleKey("onnxruntime-node");
+
+export function memoizedDetectOllama(): OllamaDetectResult | null {
+	// guardrails-allow PREVENT-PI-004: local subprocess detection only
+	return withDetectMemo(ollamaSlot, ollamaKey, detectOllama);
+}
+
+export function memoizedDetectLlamaCpp(): DetectResult | null {
+	// guardrails-allow PREVENT-PI-004: local subprocess detection only
+	return withDetectMemo(llamaSlot, llamaKey, detectLlamaCpp);
+}
+
+export function memoizedDetectOnnx(): DetectResult | null {
+	// guardrails-allow PREVENT-PI-004: local module resolution only
+	return withDetectMemo(onnxSlot, onnxKey, detectOnnx);
+}

package/extensions/dashboard-server/routes-setup.ts

@@ -9,12 +9,19 @@
  * Each decoration line carries a guardrails-allow annotation.
  */
 
-import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only
-import { createRequire } from "node:module";
 import { writeFileSync, mkdirSync, readFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import type { RouteContext } from "./routes-core.js";
+import { VC9D_ENABLED } from "../../src/config.js";
+import {
+	detectOllama,
+	detectLlamaCpp,
+	detectOnnx,
+	memoizedDetectOllama,
+	memoizedDetectLlamaCpp,
+	memoizedDetectOnnx,
+} from "./routes-setup-detect-cache.js";
 import type {
 	SetupStatusResponse,
 	SetupDetectResponse,
@@ -91,80 +98,10 @@ export function handleSetupStatus(
 // handleSetupDetect — "/api/setup-detect"
 // ---------------------------------------------------------------------------
 
-function detectOllama(): OllamaDetectResult | null {
-	try {
-		const version = spawnSync("ollama", ["--version"], {
-			timeout: 5000,
-			encoding: "utf-8",
-			stdio: ["ignore", "pipe", "pipe"],
-		});
-		if (version.status !== 0) {
-			return { installed: false, models: [], running: false, detail: version.stderr?.trim() || "not found" };
-		}
-		// Check for running server
-		let running = false;
-		let models: string[] = [];
-		try {
-			const listResult = spawnSync("ollama", ["list"], {
-				timeout: 5000,
-				encoding: "utf-8",
-				stdio: ["ignore", "pipe", "pipe"],
-			});
-			if (listResult.status === 0) {
-				running = true;
-				// Parse lines: NAME  ID  SIZE  MODIFIED
-				const lines = listResult.stdout?.split("\n") ?? [];
-				for (const line of lines) {
-					const name = line.split(/\s+/)[0];
-					if (name && name !== "NAME") models.push(name);
-				}
-			}
-		} catch {
-			// ollama list failed — server may not be running
-			running = false;
-		}
-		return {
-			installed: true,
-			models,
-			running,
-			detail: version.stdout?.trim() || null,
-		};
-	} catch {
-		return { installed: false, models: [], running: false, detail: "detection error" };
-	}
-}
-
-function detectLlamaCpp(): DetectResult | null {
-	try {
-		const which = spawnSync("which", ["llama-server", "llama.cpp"], {
-			timeout: 3000,
-			encoding: "utf-8",
-			stdio: ["ignore", "pipe", "pipe"],
-		});
-		const installed = which.status === 0 && which.stdout.trim().length > 0;
-		return {
-			installed,
-			detail: installed ? which.stdout.trim() : null,
-		};
-	} catch {
-		return { installed: false, detail: "detection error" };
-	}
-}
-
-function detectOnnx(): DetectResult | null {
-	try {
-		// Check for onnxruntime-node in a parent node_modules
-		const req = createRequire(import.meta.url);
-		try {
-			req.resolve("onnxruntime-node");
-			return { installed: true, detail: "onnxruntime-node found in node_modules" };
-		} catch {
-			return { installed: false, detail: "onnxruntime-node not found in node_modules" };
-		}
-	} catch {
-		return { installed: false, detail: "detection error" };
-	}
-}
+// Detection bodies live in routes-setup-detect-cache.ts (single source). When
+// MEGACOMPACT_VC9D is ON the memoized wrappers reuse the result across requests
+// (keyed by the mutable input: resolved binary path + mtime); when OFF we call
+// the raw fresh detectors — byte-identical to the VC9C-era per-request spawn.
 
 export function handleSetupDetect(
 	req: IncomingMessage,
@@ -183,20 +120,23 @@ export function handleSetupDetect(
 	let llamaCpp: DetectResult | null = null;
 	let onnx: DetectResult | null = null;
 
+	// guardrails-allow PREVENT-PI-004: local subprocess detection only (memoized or fresh)
+	const cached = VC9D_ENABLED();
+
 	try {
-		ollama = detectOllama();
+		ollama = cached ? memoizedDetectOllama() : detectOllama();
 	} catch (e) {
 		error = `ollama detection failed: ${e instanceof Error ? e.message : String(e)}`;
 	}
 
 	try {
-		llamaCpp = detectLlamaCpp();
+		llamaCpp = cached ? memoizedDetectLlamaCpp() : detectLlamaCpp();
 	} catch (e) {
 		error = error ?? `llama.cpp detection failed: ${e instanceof Error ? e.message : String(e)}`;
 	}
 
 	try {
-		onnx = detectOnnx();
+		onnx = cached ? memoizedDetectOnnx() : detectOnnx();
 	} catch (e) {
 		error = error ?? `onnx detection failed: ${e instanceof Error ? e.message : String(e)}`;
 	}

package/extensions/mega-config-types.ts

@@ -0,0 +1,181 @@
+/**
+ * mega-config-types.ts — MegaConfig type-only declarations for the extension.
+ *
+ * Split out of mega-config.ts (delegate-shell) so the runtime barrel stays under
+ * the extensions/ soft line limit. Type-only file: no runtime code, no logic.
+ * `import type` keeps the only cross-reference (CompactTier) erased at build,
+ * so there is no runtime import cycle.
+ */
+import type { CompactTier } from "./mega-config.js";
+
+/**
+ * Resolved, frozen-at-load config. `tier` is the base compaction PRESET chosen
+ * by env (low/medium/high/ultra/mega) — it sets the threshold token budget and
+ * is NOT changed at runtime (the /mega-tier command was removed in S24). The
+ * *displayed* tier the user sees in the toolbar/dashboard is the LIVE pressure
+ * band (see MegaRuntime.pressureBand), which climbs low→mega as context fills.
+ */
+export interface MegaConfig {
+	tier: CompactTier | "custom";
+	/**
+	 * Compaction threshold as a fraction of the model context window (e.g. 0.70
+	 * for "high"). null for `custom` (explicit MEGACOMPACT_THRESHOLD_TOKENS, which
+	 * stays an ABSOLUTE token count, never percent-scaled).
+	 */
+	tierPct: number | null;
+	thresholdTokens: number;
+	stateDir: string;
+	fastGatePct: number;
+	anchorUserMessages: number;
+	preserveRecent: number;
+	/** High-pressure floor for preserveRecent — when context is near the limit
+	 *  we compact deeper, but never below this (keeps recent turns for coherence). */
+	preserveRecentMin: number;
+	/** D.1: minimum context-growth percentage delta before re-compacting instead of
+	 *  replaying the cached trim (Default 50). Set via MEGACOMPACT_RECOMPACT_PCT_DELTA.*/
+	recompactPctDelta: number;
+	auto: boolean;
+	autoInline: boolean;
+	autoInlineK: number;
+	/** S28: auto-continue the agent after a max-output-token length stop by
+	 *  reusing the existing S16 resume-nudge. Default true. Off = silent (the
+	 *  prior behavior). PREVENT-PI-003: restart via user-role sendUserMessage. */
+	autoContinueLengthStop: boolean;
+	/** S38: max retries for transient errors (5xx/429/network/max-output-token
+	 *  text that is NOT a length stopReason — S28 owns those). Default 5.
+	 *  `0` disables all transient retries (reverts to S28-only). */
+	autoRetryTransientMax: number;
+	/** S38: max retries for permanent errors (auth/config/malformed). Default 1.
+	 *  `0` disables permanent-error retries. */
+	autoRetryPermanentMax: number;
+	/** S38.5: strict race-guard — 30s cooldown + deferred ctx.compact() re-check
+	 *  (closes the first-race-in-burst window). Default true. `false` reverts to
+	 *  the v0.7.4 synchronous 10s-cooldown behavior. */
+	raceGuardStrict: boolean;
+	/** S38.6: max consecutive errors before circuit-breaker trips (stops retrying).
+	 *  Default 10. When `errorRetryCount` exceeds this across multiple turns,
+	 *  the extension stops retrying until a successful turn resets it. */
+	maxConsecutiveErrors: number;
+	/** S38.7: hard-stop switch — when true, ALL error retries are disabled.
+	 *  Default false. Set via env to force S28-only behavior (length-stop continues
+	 *  only). */
+	errorRetryHardStop: boolean;
+	/** R1 (retry redesign): base unit (ms) for errorRetryBackoffMs(count) pacing.
+	 *  The schedule is base, 2*base, 4*base, 6*base (cap) — so the default 5000
+	 *  yields 5s/10s/20s/30s. errorRetryUntil is now GATING (previously it was
+	 *  documented as non-gating); a nudge cannot fire before errorRetryUntil
+	 *  elapses. */
+	errorRetryBackoffMs: number;
+	/** R2: session-global cap on total S38 nudges across ALL bursts. Hitting it
+	 *  is terminal for the session — the extension stops nudging entirely,
+	 *  independent of the per-burst max and the circuit breaker. Default 3.
+	 *  `0` disables (reverts to per-burst + circuit-breaker only). */
+	errorRetrySessionMax: number;
+	/** R3: consecutive identical error-text count at which a 'transient'
+	 *  classification is upgraded to 'poisoned-context' (the stateful repeat
+	 *  signal). Default 3. Raise to make the upgrade less aggressive. */
+	poisonedContextRepeatThreshold: number;
+	/** R10: consecutive transient errors at which a calm "provider outage"
+	 *  advisory is sent to the user (distinct from the poisoned /clear advise).
+	 *  Default 3. `0` disables the advisory entirely. */
+	providerOutageAdviseThreshold: number;
+	/** R13: when true (default), poisoned-context and provider-outage advisories
+	 *  are dashboard-only (events tab + log) — no user-visible message injection.
+	 *  When false, the legacy sendUserMessage path runs (byte-identical pre-R13). */
+	advisoryChannel: boolean;
+	/** S29: override the auto-compact fire point for tiered configs, as a
+	 *  fraction of the context window (e.g. 0.85). null = inherit the tier's
+	 *  tierPct (default; preserves existing fire points). The context-handler
+	 *  gate fires on context % (reliable), not token count (under-reported),
+	 *  so it catches the overshoot that causes max-output-token truncation.
+	 *  `custom` (tierPct null) ignores this — it keeps the absolute token gate. */
+	autoPctTrigger: number | null;
+	dedupSim: number;
+	/** RAPTOR hierarchical recall enabled (Fix D). Drives both live recall and
+	 *  the durable-trim summary source (root summary). */
+	raptorEnabled: boolean;
+	/** Legacy v0.4.28 behavior: auto-trigger calls ctx.compact() (which STOPS
+	 *  the agent). Default false — the S16 redesign uses the live context-event
+	 *  trim + pi native auto-compaction instead (compact and continue). Kept for
+	 *  one release as rollback. */
+	legacyDurableTrim: boolean;
+	/** S27: durable raw-transcript DB mirror (MEGACOMPACT_DB_MIRROR). When on,
+	 *  raw message bytes + checkpoint-epoch bookkeeping are appended to the
+	 *  SQLite store so a compacted window can be rehydrated locally instead of
+	 *  from the pi runtime transcript. Default OFF — additive, no behavior
+	 *  change until flipped on. legacyDurableTrim takes precedence (the legacy
+	 *  v0.4.28 ctx.compact() path does not emit the S27 mirror hook). */
+	dbMirror: boolean;
+	/** S49: isolated per-turn store (turns.db). Default ON. OFF = legacy main-db
+	 *  turn path (S48 behavior — byte-identical). Mirrors TurnsConfig.TURNS_DB_ENABLED. */
+	turnsDbEnabled: boolean;
+	/** S51: auto-categorizing wiki (k-means + TF-IDF over real embeddings). Default ON.
+	 *  Mirrors TurnsConfig.AUTO_WIKI_ENABLED. Rebuild fires every Nth compaction. */
+	autoWikiEnabled: boolean;
+	/** Cross-repo recall enabled (S17). Resume + /mega-recall --cross-repo can
+	 *  pull checkpoints from OTHER repos via the PGlite HNSW index. Default true. */
+	crossRepoEnabled: boolean;
+	/** Stricter cosine floor for cross-repo hits (S17). Default 0.90 (trigram) /
+	 *  tighter than same-repo so only genuinely-relevant cross-repo context is
+	 *  injected. */
+	crossRepoCosine: number;
+	/** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
+	 *  conversation is auto-reviewed into durable add/replace/remove memories. */
+	memoryAutoReview: boolean;
+	/** Turn cadence for the auto-review scan (S20). Default 10. */
+	memoryReviewInterval: number;
+	/** Token ceiling for the re-injected recall block (Fix C). Recall stops
+	 *  adding checkpoints once the block would exceed this — bounds read-path
+	 *  token cost so it can never net-inflate the window. */
+	recallMaxTokens: number;
+	/** Inline-dedupe recalled checkpoints against the live window (Fix C): drop
+	 *  a hit whose summary is ≥ dedupSim similar to a live message — "dedupe on
+	 *  inline/read" so we never re-inject context already resident. */
+	windowDedupe: boolean;
+	/** S53: Recall Tail Injection — inject staged recall block as a user message at
+	 *  the tail of the view when auto is OFF AND no trim action is needed. Default ON
+	 *  (true). When false, restores the pre-sprint systemPrompt prepend behavior. */
+	recallTailInject: boolean;
+	/** A1 PLAN_V2 Phase 2: Message Separation — isolate user/assistant turns
+	 *  from volatile tool results so the prompt-cache prefix stays stable.
+	 *  PC-A: positive sprint flag, now default ON; flag-OFF (=0) is byte-identical
+	 *  to the pre-change OFF state. The single gate lives at the call site
+	 *  (tailResult.ts, config.messageSeparation), not inside buildSeparatedPrompt. */
+	messageSeparation: boolean;
+	/** P3: Cache-aware striping (PLAN_V2 Phase 3). Inserts stability-ordered
+	 *  cache stripes between summaries and thread. Default OFF. */
+	cacheStriping: boolean;
+	debug: boolean;
+	/** Master reconciliation: TUI shutdown widget (MEGACOMPACT_TUI_WIDGET=0 to disable). */
+	tuiWidget: boolean;
+	/** S57 B1: Query reformulation via embedding-neighbor keyword expansion. */
+	ragQueryReformulation: boolean;
+	/** S57 B2: Tiered recall router (L0 cache -> L1 FTS5 -> L2). */
+	ragTieredRouter: boolean;
+	/** S57 B3: Recall quality metrics (precision/recall scoring + logging). */
+	ragRecallMetrics: boolean;
+	/** S57 B4: Memory graph traversal (dashboard-oriented). */
+	ragMemoryGraph: boolean;
+	/** D1: Seed initial wiki topic model from live turns when no context_chunks exist yet. */
+	wikiSeedFromTurns: boolean;
+	/** D3 Source A: Include structural turn nodes in the memory graph (metadata only, no content). */
+	memoryGraphSeedTurns: boolean;
+	/** D3 Source B: Include raw_transcript content nodes in the memory graph (requires dbMirror). */
+	memoryGraphSeedTurnContent: boolean;
+	/** D3 Source C: Include memory review nodes in the memory graph. */
+	memoryGraphSeedMemories: boolean;
+	/** D3 edges: Stricter cosine floor for cross-type edges (e.g. turn↔checkpoint). */
+	memoryGraphCrossTypeThreshold: number;
+	/** D3 edges: Cosine floor for within-type semantic edges. */
+	memoryGraphWithinTypeThreshold: number;
+	/** v0.12: Context health monitoring (drift + output quality + cache poison). Default ON. */
+	contextHealth: boolean;
+	/** Sub-flag: drift detection (topic drift + error escalation + prefix instability). */
+	contextHealthDrift: boolean;
+	/** Sub-flag: output quality analysis (repetition, coherence, token salad). */
+	contextHealthOutputQuality: boolean;
+	/** Sub-flag: tri-layer KV cache poison validation. */
+	contextHealthCachePoison: boolean;
+	/** v0.12: KV cache poison mitigation — inject prefix break on mismatch. Default OFF. */
+	contextHealthMitigate: boolean;
+}