pi-mega-compact 0.16.3 → 0.17.1

package/dist/dedup/raptor/summarizer.js

@@ -0,0 +1,86 @@
+/**
+ * summarizer.ts — per-cluster summary for the RAPTOR tree (Sprint 13, Phase 6).
+ *
+ * Default: pure extractive (deterministic, zero network, zero model) reusing
+ * extractive.ts. Optional: a LOCAL Ollama model (llama3.2:3b by default) when
+ * MEGACOMPACT_RAPTOR_MODEL is set — localhost only, same PREVENT-PI-004
+ * exception class as HttpEmbedder/the dashboard. No remote API is ever called.
+ *
+ * The summarizer returns structured text + a token estimate. Faithfulness is
+ * enforced downstream by guardrails.ts — this module only produces candidates.
+ */
+import { extractiveSummarize } from "../../extractive.js";
+import { estimateBlockTokens } from "../../tokens.js";
+import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned Ollama server (BYO local model, never remote)
+/** The local Ollama endpoint (loopback). Read lazily so tests can avoid it. */
+function ollamaEndpoint() {
+    const model = process.env.MEGACOMPACT_RAPTOR_MODEL;
+    if (!model)
+        return null;
+    const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434"; // guardrails-allow PREVENT-PI-004: loopback-only Ollama endpoint for local RAPTOR summarization
+    // Guard: only loopback is permitted (remote Ollama would violate PREVENT-PI-004).
+    if (!/^https?:\/\/(localhost|127\.0\.0\.1)([:/]|$)/.test(base)) {
+        throw new Error(`MEGACOMPACT_RAPTOR_URL must be localhost/127.0.0.1 (got ${base}). ` +
+            `Remote Ollama is not allowed (PREVENT-PI-004).`);
+    }
+    return { url: `${base}/api/generate`, model };
+}
+/**
+ * Extractive summarization of a cluster's source messages. Deterministic and
+ * fully local — the on-by-default path.
+ */
+export function extractiveClusterSummary(messages) {
+    const s = extractiveSummarize(messages);
+    return { summary: s.topicSummary, tokenEstimate: s.tokenEstimate };
+}
+/**
+ * Build a summary for one cluster of source messages.
+ *
+ * Uses local Ollama when MEGACOMPACT_RAPTOR_MODEL is set (localhost-only);
+ * otherwise falls back to deterministic extractive. The `fetch` is a localhost
+ * call inside the PREVENT-PI-004 exception — annotated accordingly.
+ */
+export function summarizeCluster(messages) {
+    const ollama = ollamaEndpoint();
+    if (!ollama)
+        return extractiveClusterSummary(messages);
+    return ollamaSummarize(messages, ollama);
+}
+function ollamaSummarize(messages, ollama) {
+    // The fetch below is localhost-only (loopback Ollama) — the PREVENT-PI-004
+    // sanctioned local-model exceptions (same class as /dashboard, HttpEmbedder).
+    const prompt = messages.map((m) => `${m.role}: ${m.text}`).join("\n");
+    // Synchronous bridge: spawnSync an inline worker so the call blocks without
+    // deadlocking fetch (mirrors HttpEmbedder — Atomics.wait on main thread would
+    // hang). A blocked main thread cannot pump the socket.
+    const WORKER = String.raw `
+    const url = process.env.R_URL, model = process.env.R_MODEL, prompt = process.env.R_PROMPT;
+    try {
+      const r = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model, prompt, stream: false }) }); // guardrails-allow PREVENT-PI-004: localhost-only user-spawned Ollama server (BYO local model, never remote)
+      const j = await r.json();
+      process.stdout.write(JSON.stringify({ ok: r.ok, text: j.response || "" }));
+    } catch (e) {
+      process.stdout.write(JSON.stringify({ ok: false, error: String(e && e.message ? e.message : e) }));
+    }
+  `;
+    const res = spawnSync(process.execPath, ["-e", WORKER], {
+        encoding: "utf8",
+        timeout: 30_000,
+        env: { ...process.env, R_URL: ollama.url, R_MODEL: ollama.model, R_PROMPT: prompt },
+    });
+    let parsed = { ok: false, error: "no response" };
+    if (typeof res.stdout === "string" && res.stdout.length > 0) {
+        try {
+            parsed = JSON.parse(res.stdout);
+        }
+        catch {
+            parsed = { ok: false, error: "bad json" };
+        }
+    }
+    if (!parsed.ok || !parsed.text) {
+        // Ollama unavailable → deterministic extractive fallback (never fail the build).
+        return extractiveClusterSummary(messages);
+    }
+    const summary = parsed.text.trim();
+    return { summary, tokenEstimate: estimateBlockTokens(summary) };
+}

package/dist/dedup/raptor/tree.js

@@ -0,0 +1,202 @@
+/**
+ * tree.ts — RAPTOR hierarchical summary-tree builder (Sprint 13, Phase 6).
+ *
+ * Builds a multi-level tree of summary nodes over leaf chunks: leaves are the
+ * original regions; each higher level summarizes clusters of the level below
+ * until a single root remains. QA ops: a wall-clock budget guard — on exhaustion
+ * we build an extractive fallback root. <10 leaves → a single summary node.
+ *
+ * Node model (kept simple + flat): every RaptorNode stores the LIST OF LEAF IDS
+ * it ultimately covers in `children` (not a mix of node/leaf ids). So the node
+ * map holds ONLY internal summary nodes — never per-leaf wrappers — which is
+ * what makes RAPTOR consolidate (nodes.size << leaves) and makes retrieval's
+ * leaf walk trivial.
+ *
+ * PREVENT-PI-004: no network here. summarizeCluster() may call a localhost
+ * Ollama (annotated in summarizer.ts); extractive is the default.
+ */
+import { kmeanspp, meanVector } from "./kmeans.js";
+import { summarizeCluster } from "./summarizer.js";
+import { applyHallucinationGuardrails, sourceTokenSet } from "./guardrails.js";
+const DEFAULT_BUDGET_MS = 5000;
+const DEFAULT_CLUSTERS = 5;
+function defaultNextId(level, index) {
+    return `r${level}_${index}`;
+}
+function summarizeInto(item, centroid, embedder, consistencyThreshold) {
+    let summary = summarizeCluster(item.messages);
+    const guard = applyHallucinationGuardrails({
+        summary: summary.summary,
+        sources: item.sources,
+        centroid,
+        embedder,
+        sourceTokens: sourceTokenSet(item.sources),
+        consistencyThreshold,
+    });
+    if (guard.marker === "extractive_fallback") {
+        summary = summarizeCluster(item.messages); // deterministic extractive text
+    }
+    return {
+        summary: summary.summary,
+        tokenEstimate: summary.tokenEstimate,
+        qualityMarker: guard.marker === "extractive_fallback" ? "low" : guard.marker,
+    };
+}
+/**
+ * Build a RAPTOR tree from leaf chunks. Synchronous; guarded by an elapsed-time
+ * budget. Returns a tree whose `nodes` map holds ONLY internal summary nodes.
+ */
+export function buildRaptorTree(leaves, opts) {
+    const embedder = opts.embedder;
+    const budgetMs = opts.budgetMs ?? DEFAULT_BUDGET_MS;
+    const clustersPerLevel = opts.clustersPerLevel ?? DEFAULT_CLUSTERS;
+    const nextId = opts.nextId ?? defaultNextId;
+    const now = opts.now ?? (() => Date.now());
+    const start = now();
+    const within = () => now() - start <= budgetMs;
+    const nodes = new Map();
+    // <10 leaves → single summary root (no hierarchy needed).
+    if (leaves.length < 10) {
+        const item = {
+            id: "root",
+            parentId: null,
+            embedding: meanVector(leaves.map((l) => l.embedding)),
+            leafIds: leaves.map((l) => l.id),
+            messages: leaves.flatMap((l) => l.messages),
+            sources: leaves.map((l) => l.sourceText),
+        };
+        const centroid = item.embedding;
+        const { summary, tokenEstimate, qualityMarker } = summarizeInto(item, centroid, embedder, opts.consistencyThreshold);
+        const rootId = nextId(0, 0);
+        nodes.set(rootId, {
+            id: rootId,
+            level: 0,
+            parentId: null,
+            children: item.leafIds,
+            summary,
+            embedding: centroid,
+            qualityMarker,
+            tokenEstimate,
+        });
+        return { nodes, rootId, levels: 1, timedOut: false };
+    }
+    let currentLevel = leaves.map((l) => ({
+        id: l.id,
+        parentId: null,
+        embedding: l.embedding,
+        leafIds: [l.id],
+        messages: l.messages,
+        sources: [l.sourceText],
+    }));
+    let level = 0;
+    while (currentLevel.length > 1) {
+        if (!within())
+            return extractiveFallbackRoot(leaves, nodes, nextId);
+        // Once we're down to a handful of items, collapse them all into one root.
+        // (k === currentLevel.length would make every item its own singleton
+        // cluster and never shrink — an infinite loop until the budget blows.)
+        if (currentLevel.length <= clustersPerLevel) {
+            const merged = {
+                id: "merge",
+                parentId: null,
+                embedding: meanVector(currentLevel.map((c) => c.embedding)),
+                leafIds: currentLevel.flatMap((c) => c.leafIds),
+                messages: currentLevel.flatMap((c) => c.messages),
+                sources: currentLevel.flatMap((c) => c.sources),
+            };
+            const centroid = merged.embedding;
+            const { summary, tokenEstimate, qualityMarker } = summarizeInto(merged, centroid, embedder, opts.consistencyThreshold);
+            const rootId = nextId(level + 1, 0);
+            nodes.set(rootId, {
+                id: rootId,
+                level: level + 1,
+                parentId: null,
+                children: merged.leafIds,
+                summary,
+                embedding: centroid,
+                qualityMarker,
+                tokenEstimate,
+            });
+            // Populate parentId for the internal nodes being merged into this root.
+            // currentLevel items with ids in `nodes` are internal summary nodes (level
+            // >= 1); raw leaf ids are not in `nodes` and are correctly skipped.
+            for (const c of currentLevel) {
+                const child = nodes.get(c.id);
+                if (child && child.id !== rootId && child.parentId === null) {
+                    child.parentId = rootId;
+                }
+            }
+            return { nodes, rootId, levels: level + 2, timedOut: false };
+        }
+        const k = Math.max(1, Math.min(clustersPerLevel, currentLevel.length));
+        const clustered = kmeanspp(currentLevel.map((c) => c.embedding), k, { seed: 0x1234 + level });
+        const groups = Array.from({ length: clustered.k }, () => []);
+        clustered.assignments.forEach((c, i) => groups[c].push(currentLevel[i]));
+        const nextLevel = [];
+        for (let g = 0; g < groups.length; g++) {
+            const group = groups[g];
+            if (group.length === 0)
+                continue;
+            const merged = {
+                id: nextId(level + 1, g),
+                parentId: null,
+                embedding: clustered.centroids[g],
+                leafIds: group.flatMap((c) => c.leafIds),
+                messages: group.flatMap((c) => c.messages),
+                sources: group.flatMap((c) => c.sources),
+            };
+            const { summary, tokenEstimate, qualityMarker } = summarizeInto(merged, merged.embedding, embedder, opts.consistencyThreshold);
+            nodes.set(merged.id, {
+                id: merged.id,
+                level: level + 1,
+                parentId: null,
+                children: merged.leafIds,
+                summary,
+                embedding: merged.embedding,
+                qualityMarker,
+                tokenEstimate,
+            });
+            // Populate parentId for the internal nodes being absorbed into this
+            // parent summary. Group members with ids in `nodes` are internal summary
+            // nodes (level >= 1); raw leaf ids are not in `nodes` (per-leaf wrappers
+            // are intentionally absent) and are correctly skipped — leaf→summary
+            // walks go through the parent's `children` list instead.
+            for (const c of group) {
+                const child = nodes.get(c.id);
+                if (child && child.id !== merged.id) {
+                    child.parentId = merged.id;
+                }
+            }
+            nextLevel.push(merged);
+        }
+        currentLevel = nextLevel;
+        level++;
+    }
+    const root = currentLevel[0];
+    const rootId = root ? root.id : null;
+    return {
+        nodes,
+        rootId,
+        levels: level + 1,
+        timedOut: false,
+    };
+}
+/**
+ * Budget-exceeded fallback: build a single deterministic extractive root over
+ * all leaves and mark it low quality. Keeps a valid (if shallow) tree.
+ */
+function extractiveFallbackRoot(leaves, nodes, nextId) {
+    const summary = summarizeCluster(leaves.flatMap((l) => l.messages));
+    const rootId = nextId(99, 0);
+    nodes.set(rootId, {
+        id: rootId,
+        level: 99,
+        parentId: null,
+        children: leaves.map((l) => l.id),
+        summary: summary.summary,
+        embedding: meanVector(leaves.map((l) => l.embedding)),
+        qualityMarker: "low",
+        tokenEstimate: summary.tokenEstimate,
+    });
+    return { nodes, rootId, levels: 100, timedOut: true };
+}

package/dist/dedup/topk.js

@@ -0,0 +1,60 @@
+/**
+ * topk.ts — min-heap based top-K selection (Sprint 12, QA #4).
+ *
+ * Replaces the O(N log N) full `.sort()` in search() with an O(N log k) heap,
+ * which matters once a session holds thousands of checkpoints. Generic over any
+ * scored item with a numeric `score`.
+ *
+ * Pure, no deps, no network (PREVENT-PI-004).
+ */
+/**
+ * Return the `k` highest-scoring items (stable insertion order on ties).
+ * O(N log k) — a bounded min-heap of size k.
+ */
+export function topK(items, k) {
+    if (k <= 0)
+        return [];
+    if (items.length <= k)
+        return [...items].sort((a, b) => b.score - a.score);
+    // Min-heap of the current top-k, stored as a flat array of Scored<T>.
+    const heap = [];
+    const push = (e) => {
+        heap.push(e);
+        let i = heap.length - 1;
+        while (i > 0) {
+            const parent = (i - 1) >> 1;
+            if (heap[parent].score <= heap[i].score)
+                break;
+            [heap[parent], heap[i]] = [heap[i], heap[parent]];
+            i = parent;
+        }
+    };
+    const siftDown = (start) => {
+        let i = start;
+        for (;;) {
+            const l = 2 * i + 1;
+            const r = 2 * i + 2;
+            let smallest = i;
+            if (l < heap.length && heap[l].score < heap[smallest].score)
+                smallest = l;
+            if (r < heap.length && heap[r].score < heap[smallest].score)
+                smallest = r;
+            if (smallest === i)
+                break;
+            [heap[smallest], heap[i]] = [heap[i], heap[smallest]];
+            i = smallest;
+        }
+    };
+    for (const it of items) {
+        if (heap.length < k) {
+            push(it);
+        }
+        else if (it.score > heap[0].score) {
+            // Replace the current minimum with this better-scoring item, then sift
+            // it down to restore the min-heap invariant.
+            heap[0] = it;
+            siftDown(0);
+        }
+    }
+    return heap.sort((a, b) => b.score - a.score);
+}

package/dist/extensions/dashboard-server/api-contracts/endpoints/registry-ext.js

@@ -0,0 +1,30 @@
+/** Additive endpoint entries spread into the ENDPOINTS registry. */
+export const EXTRA_ENDPOINTS = {
+    // ─── Per-model compaction thresholds (S52 / v0.16.1) ───────────────
+    /** GET /api/model-thresholds — List known models + their thresholds. */
+    modelThresholds: {
+        method: "GET",
+        path: "/api/model-thresholds",
+        description: "Every known model (from model_snapshots) with its per-model threshold override (or defaults).",
+    },
+    // ─── RAG Metrics (Sprint H2) ──────────────────────────────────────
+    /** GET /api/rag-metrics — HyDE + recall-quality telemetry aggregates. */
+    ragMetrics: {
+        method: "GET",
+        path: "/api/rag-metrics",
+        description: "HyDE invocation + recall-quality telemetry: flags, totals, recent turns, and daily series.",
+    },
+    // ─── Vector Cortex (VC0A) ────────────────────────────────────────
+    /** GET /api/vector-cortex/evaluation — reader-only eval aggregate (VC0A). */
+    vectorCortexEvaluation: {
+        method: "GET",
+        path: "/api/vector-cortex/evaluation",
+        description: "Reader-only vector-cortex evaluation summary: latency histogram + per-mode sample counts (never payloads).",
+    },
+    /** GET /api/vector-cortex/ledger — reader-only occurrence ledger view (VC1B). */
+    vectorCortexLedger: {
+        method: "GET",
+        path: "/api/vector-cortex/ledger",
+        description: "Reader-only occurrence-ledger identity view: seq/eventId/kind/digest + high-water (never source payloads).",
+    },
+};

package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js

@@ -1,3 +1,4 @@
+import { EXTRA_ENDPOINTS } from "./registry-ext.js";
 // ─── ENDPOINTS Registry ─────────────────────────────────────────────────────
 /**
  * Central registry of all 14 dashboard API endpoints. Each entry is an
@@ -290,18 +291,7 @@ export const ENDPOINTS = {
         path: "/api/rag-settings",
         description: "Update RAG feature flag states by writing _DISABLED env vars to .mega-compact.env.",
     },
-    // ─── RAG Metrics (Sprint H2) ──────────────────────────────────────
-    /** GET /api/rag-metrics — HyDE + recall-quality telemetry aggregates. */
-    ragMetrics: {
-        method: "GET",
-        path: "/api/rag-metrics",
-        description: "HyDE invocation + recall-quality telemetry: flags, totals, recent turns, and daily series.",
-    },
-    // ─── Per-model compaction thresholds (S52 / v0.16.1) ───────────────
-    /** GET /api/model-thresholds — List known models + their thresholds. */
-    modelThresholds: {
-        method: "GET",
-        path: "/api/model-thresholds",
-        description: "Every known model (from model_snapshots) with its per-model threshold override (or defaults).",
-    },
+    // ─── Additive groups (rag-metrics, thresholds, vector-cortex) live in
+    // ─── registry-ext.ts so this registry stays under the 500-line limit.
+    ...EXTRA_ENDPOINTS,
 };

package/dist/extensions/dashboard-server/api-contracts/vector-cortex.js

@@ -0,0 +1,12 @@
+/**
+ * api-contracts/vector-cortex.ts — vector-cortex dashboard API contracts.
+ *
+ * Owned by the vector-cortex sprints. VC0A ships only the reader-only aggregate
+ * `GET /api/vector-cortex/evaluation`. VC0C adds the breaker health endpoint
+ * and reset mutation into this SAME file — keep entries small and additive so
+ * the file stays within extension limits (400 soft / 500 hard).
+ *
+ * PREVENT-PI-004: type definitions only, no network code.
+ * PREVENT-011: no `any` type.
+ */
+export {};

package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js

@@ -129,6 +129,17 @@ export const SETTINGS = [
             num("MEGACOMPACT_EMBED_CACHE", "Embed Cache Size", "Embedding cache entries (0 = disabled)", 256, 0, 10000),
         ],
     },
+    {
+        name: "Vector Cortex",
+        settings: [
+            boolDirect("MEGACOMPACT_VC0A", "VC0A Baseline Observability", "Structured evaluation observer (MetricEventV1 + latency histogram). OFF = mode C, byte-identical to predecessor.", true),
+            boolDirect("MEGACOMPACT_VC0B", "VC0B Replay Correctness", "ReplayCutV2 effective-cut (min of boundary-safe/commit/capture high-water + pair retreat + anchor floor) and M3 effective-cut-v2 migration. OFF = legacy capped replay, byte-identical.", true),
+            boolDirect("MEGACOMPACT_VC1A", "VC1A Canonical Byte Events", "EventV2 byte-authority ledger codec (original bytes + SHA-256, strict UTF-8, derived NFC) and canonical validator (EVT_DIGEST_MISMATCH / EVT_UTF8_TAG_INVALID / EVT_DUPLICATE_ID). OFF = mode C, transcript codec unchanged, byte-identical.", true),
+            boolDirect("MEGACOMPACT_VC1B", "VC1B Occurrence Ledger + Tool Identity", "Neutral occurrence ledger (LedgerReader/Writer/Admin + CompatJournalV1): per-session monotonic seq, tool result references one earlier call, uniqueness by (eventId,digest) only, and the M2 copy-validate-switch downgrade journal. OFF = mode C, ledger unwritten, byte-identical.", true),
+            boolDirect("MEGACOMPACT_VC0C", "VC0C Live Safety Envelope", "TriadResult/Breaker live circuit breaker (60s window, 20 attempts, 30s cooldown, 3 probes, 5min healthy residence) + durable spool before provider invocation; manual reset clears cooldown but never evidence. OFF = mode C, unchanged transcript, byte-identical.", true),
+            boolDirect("MEGACOMPACT_VC1C", "VC1C Cross-Language Conformance v2", "FixtureManifestV2 canonical manifest validator + DowngradeReport deterministic downgrade export + MinHashV2 exact big-integer signatures and the M4 copy/validate/switch minhash-v2 migration (seed table frozen, cross-language byte-exact). OFF = mode C, v1 sync dedup scan unchanged, byte-identical.", true),
+        ],
+    },
     {
         name: "Cost API",
         settings: [