pi-mega-compact 0.13.5 → 0.13.7

package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js

@@ -151,6 +151,49 @@ export const ENDPOINTS = {
         path: "/api/topics/:topicId/memories",
         description: "Member memories assigned to a wiki topic (drill-down).",
     },
+    // ─── Wiki Revival (W2) ───────────────────────────────────────────
+    /** GET /api/wiki/index — wiki landing page with resolved labels. */
+    wikiIndex: {
+        method: "GET",
+        path: "/api/wiki/index",
+        description: "Wiki index: auto-categorized topics with resolved (override-aware) labels.",
+    },
+    /** GET /api/wiki/topic/:topicId — single topic page + provenance. */
+    wikiTopic: {
+        method: "GET",
+        path: "/api/wiki/topic/:topicId",
+        description: "Single wiki topic page: summary, key memories, and memory provenance.",
+    },
+    /** PUT /api/wiki/topic/:topicId/label — rename a topic (user curation). */
+    renameTopic: {
+        method: "PUT",
+        path: "/api/wiki/topic/:topicId/label",
+        description: "Rename a wiki topic (persists a label override; emits wiki_topic_renamed).",
+    },
+    /** POST /api/wiki/merge — merge one topic into another. */
+    mergeTopics: {
+        method: "POST",
+        path: "/api/wiki/merge",
+        description: "Merge source topic into target (reassigns memories; emits wiki_topics_merged).",
+    },
+    /** POST /api/wiki/topic/:topicId/split — split listed memories into a new topic. */
+    splitTopic: {
+        method: "POST",
+        path: "/api/wiki/topic/:topicId/split",
+        description: "Split listed member memories into a new topic (emits wiki_topic_split).",
+    },
+    /** GET /api/wiki/topic/:topicId/timeline — per-topic time buckets. */
+    topicTimeline: {
+        method: "GET",
+        path: "/api/wiki/topic/:topicId/timeline",
+        description: "Per-topic memory-assignment timeline buckets for charts.",
+    },
+    /** GET /api/wiki/evolution — node/edge D3 evolution graph. */
+    topicEvolution: {
+        method: "GET",
+        path: "/api/wiki/evolution",
+        description: "Topic evolution nodes + edges + time buckets from topic_evolution.",
+    },
     // ─── S49B Maintenance Tab ─────────────────────────────────────────
     /** GET /api/maintenance — DB stats (table row counts + storage). */
     maintenanceStats: {
@@ -235,4 +278,11 @@ export const ENDPOINTS = {
         path: "/api/rag-settings",
         description: "Update RAG feature flag states by writing _DISABLED env vars to .mega-compact.env.",
     },
+    // ─── RAG Metrics (Sprint H2) ──────────────────────────────────────
+    /** GET /api/rag-metrics — HyDE + recall-quality telemetry aggregates. */
+    ragMetrics: {
+        method: "GET",
+        path: "/api/rag-metrics",
+        description: "HyDE invocation + recall-quality telemetry: flags, totals, recent turns, and daily series.",
+    },
 };

package/dist/extensions/dashboard-server/api-contracts/rag-metrics.js

@@ -0,0 +1 @@
+export {};

package/dist/extensions/dashboard-server/api-contracts/rag-settings.js

@@ -1,8 +1,8 @@
 /**
- * api-contracts/rag-settings.ts — RAG feature flag API response types.
+ * api-contracts/rag-settings.ts — comprehensive settings API response types.
  *
- * Types for the /api/rag-settings endpoint used by the dashboard RAG Settings
- * panel to read and toggle the five RAG feature flags (B1–B5).
+ * Types for the /api/rag-settings endpoint used by the dashboard Setup panel to
+ * read and update every adjustable MEGACOMPACT_* setting, grouped by category.
  *
  * PREVENT-PI-004: type definitions only, no network code.
  * PREVENT-011: no `any` type — all types are explicit.

package/dist/extensions/dashboard-server/api-contracts/wiki.js

@@ -0,0 +1 @@
+export {};

package/dist/extensions/dashboard-server/index-reader.js

@@ -13,6 +13,7 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import { DatabaseSync } from "node:sqlite";
 import { readProviderCacheForRepo, } from "../../src/store/sqlite/perf-samples.js";
+import { lookupModelInputRate } from "../../src/pricing.js";
 export function getIndexDir() {
     const override = process.env.MEGACOMPACT_INDEX_DIR;
     if (override && override.trim() !== "")
@@ -48,7 +49,8 @@ export function readIndex() {
             provider: r.provider ?? null,
             providerName: r.provider_name ?? null,
             modelName: r.model_name ?? null,
-            inputRate: r.input_rate ?? null,
+            inputRate: (r.input_rate ||
+                lookupModelInputRate(String(r.model_name ?? ""))) ?? null,
             outputRate: r.output_rate ?? null,
             lastSeen: Number(r.last_seen ?? 0),
             // Defaults — enriched below from each repo's own store.

package/dist/extensions/dashboard-server/resolve-snapshot.js

@@ -0,0 +1,81 @@
+/**
+ * dashboard-server/resolve-snapshot.ts — dynamic per-repo snapshot resolver.
+ *
+ * The dashboard server is launched with a single static stateDir, but the
+ * active pi session may be running in a different repo. This module queries
+ * the repo registry to find the most recently active repo's dashboard.json,
+ * so the Overview always shows live data from the repo pi is actually running
+ * in — not the stale launcher dir.
+ *
+ * Resolution order:
+ * 1. Most recently seen repo (by last_seen in repo_registry) whose
+ *    dashboard.json exists and has a non-null updatedAt
+ * 2. The static launch stateDir's dashboard.json (original behavior)
+ */
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { createRequire } from "node:module";
+import { readSnapshot } from "./snapshot.js";
+import { getIndexDir } from "./index-reader.js";
+const require = createRequire(import.meta.url);
+/**
+ * Find the most recently active repo's stateDir by scanning the repo registry
+ * for repos whose dashboard.json exists and was written recently. Returns null
+ * if no suitable repo is found.
+ */
+function mostRecentRepoStateDir(staticStateDir) {
+    const indexPath = join(getIndexDir(), "index.sqlite");
+    if (!existsSync(indexPath))
+        return null;
+    let db;
+    try {
+        const { DatabaseSync } = require("node:sqlite");
+        db = new DatabaseSync(indexPath, { readOnly: true });
+        const rows = db
+            .prepare(`SELECT state_dir, last_seen FROM repo_registry
+				 WHERE state_dir IS NOT NULL AND state_dir != ?
+				 ORDER BY last_seen DESC`)
+            .all(staticStateDir);
+        for (const r of rows) {
+            const dashPath = join(r.state_dir, "dashboard.json");
+            if (!existsSync(dashPath))
+                continue;
+            const snap = readSnapshot(dashPath);
+            if (snap.updatedAt) {
+                return r.state_dir;
+            }
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+    finally {
+        try {
+            db?.close();
+        }
+        catch {
+            /* ignore */
+        }
+    }
+}
+/**
+ * Resolve the best snapshot to serve. Tries the most recently active repo's
+ * dashboard.json first, then falls back to the static launch path.
+ */
+export function resolveSnapshot(staticSnapshotPath, staticStateDir) {
+    // Try the most recently active repo first.
+    const activeDir = mostRecentRepoStateDir(staticStateDir);
+    if (activeDir) {
+        const activePath = join(activeDir, "dashboard.json");
+        const snap = readSnapshot(activePath);
+        if (snap.updatedAt) {
+            return { snapshot: snap, stateDir: activeDir };
+        }
+    }
+    // Fall back to the static launch path.
+    return {
+        snapshot: readSnapshot(staticSnapshotPath),
+        stateDir: staticStateDir,
+    };
+}

package/dist/extensions/dashboard-server/routes-core.js

@@ -5,7 +5,7 @@
  * builds it. Route handler files import from here; the barrel (routes.ts)
  * re-exports everything for server.ts.
  */
-import { readSnapshot } from "./snapshot.js";
+import { resolveSnapshot } from "./resolve-snapshot.js";
 // ---------------------------------------------------------------------------
 // Helper: overlayCurrentRepo
 // ---------------------------------------------------------------------------
@@ -13,16 +13,21 @@ function makeOverlayCurrentRepo(snapshotPath, stateDir) {
     return (idx) => {
         if (!idx || !idx.repos.length)
             return;
+        // Dynamically resolve the most recently active repo's snapshot so the
+        // overlay matches what /api/snapshot serves (not the stale launch dir).
         let snap = null;
+        let activeStateDir = stateDir;
         try {
-            snap = readSnapshot(snapshotPath);
+            const resolved = resolveSnapshot(snapshotPath, stateDir);
+            snap = resolved.snapshot;
+            activeStateDir = resolved.stateDir;
         }
         catch {
             return;
         }
         if (!snap || !snap.repo)
             return;
-        const cur = idx.repos.find((r) => r.stateDir === stateDir);
+        const cur = idx.repos.find((r) => r.stateDir === activeStateDir);
         if (!cur)
             return;
         const prevSaved = cur.tokensSaved;

package/dist/extensions/dashboard-server/routes-rag-metrics.js

@@ -0,0 +1,60 @@
+/**
+ * dashboard-server/routes-rag-metrics.ts — GET /api/rag-metrics route (H2).
+ *
+ * Aggregates the H1 HyDE + recall-quality telemetry columns from turns.db into
+ * a compact response (flags, totals, recent rows, daily series).
+ *
+ * Guardrails: PREVENT-PI-004 (loopback-only), PREVENT-002 (parameterized),
+ * PREVENT-011 (no `any`).
+ */
+import { openTurnStore } from "../../src/store/turns/connection.js";
+import { listTelemetryTurns, aggregateDailyTelemetry, } from "../../src/store/turns/hydeStore.js";
+import { RAG_HYDE_ENABLED, RAG_RECALL_METRICS } from "../../src/config.js";
+function sendJson(res, status, body) {
+    // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+    res.writeHead(status, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(body));
+}
+export function handleRagMetrics(req, res, ctx) {
+    const url = req.url ?? "";
+    if (url !== "/api/rag-metrics" || req.method !== "GET")
+        return false;
+    let db;
+    try {
+        db = openTurnStore(ctx.stateDir);
+    }
+    catch {
+        db = null;
+    }
+    const recent = db ? listTelemetryTurns(db, { limit: 200 }) : [];
+    const daily = db ? aggregateDailyTelemetry(db, 14) : [];
+    const hydeRanTurns = recent.filter((t) => t.hydeRan === 1).length;
+    const liftSum = recent.reduce((s, t) => s + t.hydeLift, 0);
+    const genSum = recent.reduce((s, t) => s + t.hydeGenerationMs, 0);
+    const scored = recent.filter((t) => t.recallScore > 0 || t.recallPass === 1);
+    const avgScore = scored.length > 0
+        ? Math.round((scored.reduce((s, t) => s + t.recallScore, 0) / scored.length) * 100) / 100
+        : null;
+    const passCount = scored.filter((t) => t.recallPass === 1).length;
+    const recentPassRate = scored.length > 0 ? Math.round((passCount / scored.length) * 100) / 100 : 0;
+    const body = {
+        flags: {
+            hydeEnabled: RAG_HYDE_ENABLED(),
+            recallMetricsEnabled: RAG_RECALL_METRICS(),
+        },
+        totals: {
+            telemetryTurns: recent.length,
+            hydeRanTurns,
+            avgLift: recent.length > 0
+                ? Math.round((liftSum / recent.length) * 100) / 100
+                : 0,
+            avgScore,
+            avgGenerationMs: recent.length > 0 ? Math.round(genSum / recent.length) : 0,
+            recentPassRate,
+        },
+        recent: recent.slice(0, 50),
+        daily,
+    };
+    sendJson(res, 200, body);
+    return true;
+}

package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js

@@ -0,0 +1,167 @@
+/**
+ * dashboard-server/routes-rag-settings-helpers.ts — SETTINGS inventory.
+ *
+ * The full catalog of user-adjustable MEGACOMPACT_* settings surfaced by the
+ * dashboard Setup panel, grouped by category. Kept separate from the route
+ * handler so the inventory can grow without bloating the handler file.
+ *
+ * PREVENT-011: no `any` type.
+ */
+// Shorthand builders to keep the inventory terse and unambiguous.
+const boolFlag = (key, label, description, def, requiresLlm = false) => ({
+    key,
+    label,
+    description,
+    type: "boolean",
+    default: def,
+    disabledConvention: true,
+    requiresLlm,
+});
+const boolDirect = (key, label, description, def) => ({
+    key,
+    label,
+    description,
+    type: "boolean",
+    default: def,
+    disabledConvention: false,
+    requiresLlm: false,
+});
+const num = (key, label, description, def, min, max, unit) => ({
+    key,
+    label,
+    description,
+    type: "number",
+    default: def,
+    disabledConvention: false,
+    requiresLlm: false,
+    min,
+    max,
+    ...(unit ? { unit } : {}),
+});
+const str = (key, label, description, def) => ({
+    key,
+    label,
+    description,
+    type: "string",
+    default: def,
+    disabledConvention: false,
+    requiresLlm: false,
+});
+/** Every adjustable setting, grouped by category. Read-only after module load. */
+export const SETTINGS = [
+    {
+        name: "RAG Pipeline",
+        settings: [
+            boolFlag("MEGACOMPACT_QUERY_REFORMULATION", "Query Reformulation", "TF-IDF keyword expansion for vague recall queries", true),
+            boolFlag("MEGACOMPACT_TIERED_ROUTER", "Tiered Recall Router", "L0 cache → L1 FTS5 → L2 HNSW routing", true),
+            boolFlag("MEGACOMPACT_RECALL_METRICS", "Recall Quality Metrics", "Precision/recall scoring and logging", true),
+            boolFlag("MEGACOMPACT_MEMORY_GRAPH", "Memory Graph", "Dashboard-oriented memory graph traversal", true),
+            boolFlag("MEGACOMPACT_HYDE", "HyDE", "Generate hypothetical answer via LLM, embed it, RRF-fuse", true, true),
+            boolFlag("MEGACOMPACT_NEW_UI", "New Dashboard UI", "Tailwind + shadcn visual design (sidebar, glass panels)", true),
+        ],
+    },
+    {
+        name: "Wiki / Turns",
+        settings: [
+            boolFlag("MEGACOMPACT_WIKI_ENHANCED", "Wiki Enhanced", "User curation (rename/merge/split) + topic evolution", true),
+            boolFlag("MEGACOMPACT_WIKI_INCREMENTAL", "Wiki Incremental", "Assign new memories to nearest centroid", true),
+            boolDirect("MEGACOMPACT_AUTO_WIKI", "Auto Wiki", "Auto-categorizing wiki (k-means + TF-IDF)", true),
+            boolDirect("MEGACOMPACT_WIKI_SEED_FROM_TURNS", "Wiki Seed from Turns", "Seed topic model from raw_transcript when chunks thin", true),
+            boolDirect("MEGACOMPACT_TURNS_DB", "Turns DB", "Isolated turns.db store", true),
+            num("MEGACOMPACT_WIKI_SILHOUETTE_MIN", "Silhouette Minimum", "Full rebuild trigger threshold", 0.2, 0, 1),
+            num("MEGACOMPACT_TURNS_RETENTION_DAYS", "Turn Retention", "Per-turn provenance retention window", 30, 1, 365, "days"),
+            num("MEGACOMPACT_TURNS_KEEP_MIN", "Turns Keep Minimum", "Min turns kept per conversation during prune", 5, 1, 100),
+            num("MEGACOMPACT_WIKI_LABEL_TOP_TERMS", "Wiki Label Terms", "How many TF-IDF terms form a topic label", 5, 1, 20),
+            num("MEGACOMPACT_WIKI_REBUILD_EVERY", "Wiki Rebuild Every", "Rebuild topic model every Nth compaction", 3, 1, 50),
+        ],
+    },
+    {
+        name: "Dedup Tiers",
+        settings: [
+            boolDirect("MEGACOMPACT_L0_ENABLED", "L0 Exact Match", "Exact-hash dedup tier", true),
+            boolDirect("MEGACOMPACT_L1_ENABLED", "L1 MinHash/LSH", "Near-dup tier", true),
+            boolDirect("MEGACOMPACT_L2_ENABLED", "L2 Semantic", "Cosine similarity dedup tier", true),
+            boolDirect("MEGACOMPACT_RAPTOR_ENABLED", "RAPTOR", "Hierarchical clustering tier", true),
+            boolDirect("MEGACOMPACT_RAPTOR_MULTILEVEL", "RAPTOR Multi-level", "Multi-level retrieval (score all tree levels)", true),
+            boolDirect("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", "RAPTOR Leaf Expansion", "Leaf node expansion in retrieval", true),
+            boolDirect("MEGACOMPACT_RAPTOR_INCREMENTAL", "RAPTOR Incremental", "Incremental rebuild mode", true),
+            boolDirect("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", "RAPTOR Inject Summaries", "Inject top-level summary nodes into recall", true),
+            boolDirect("MEGACOMPACT_MARK_ONLY_L0", "Mark Only L0", "L0 runs but does not collapse", false),
+            boolDirect("MEGACOMPACT_MARK_ONLY_L1", "Mark Only L1", "L1 runs but does not collapse", false),
+            boolDirect("MEGACOMPACT_MARK_ONLY_L2", "Mark Only L2", "L2 runs but does not collapse", false),
+            boolDirect("MEGACOMPACT_MINILM", "MiniLM Embedder", "Use MiniLM instead of trigram", false),
+        ],
+    },
+    {
+        name: "Dedup Thresholds",
+        settings: [
+            num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
+            num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
+            num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
+            num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
+            num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
+            num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),
+            num("MEGACOMPACT_SIMILARITY_BUDGET_MS", "Similarity Budget", "Time budget for similarity computation", 50, 1, 10000, "ms"),
+            num("MEGACOMPACT_L1_VERIFY_BUDGET_MS", "L1 Verify Budget", "L1 verification time budget", 20, 1, 10000, "ms"),
+            num("MEGACOMPACT_L1_CANDIDATE_CAP", "L1 Candidate Cap", "L1 candidate cap", 100, 1, 10000),
+        ],
+    },
+    {
+        name: "RAPTOR Tuning",
+        settings: [
+            num("MEGACOMPACT_RAPTOR_BUDGET_MS", "RAPTOR Budget", "RAPTOR time budget", 5000, 100, 60000, "ms"),
+            num("MEGACOMPACT_RAPTOR_CLUSTERS", "RAPTOR Clusters", "Clusters per level", 5, 1, 50),
+            num("MEGACOMPACT_RAPTOR_CONSISTENCY", "RAPTOR Consistency", "Consistency threshold", 0.6, 0, 1),
+            num("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", "RAPTOR Max Leaf Expansion", "Max leaf expansion nodes", 10, 1, 100),
+            num("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", "RAPTOR Freshness", "Skip rebuild when tree is fresh", 4, 1, 168, "hours"),
+            num("MEGACOMPACT_FP_RATE_L0", "FP Rate L0", "FP alert threshold for exact tier", 0.01, 0, 1),
+            num("MEGACOMPACT_FP_RATE_L1L2", "FP Rate L1/L2", "FP alert threshold for fuzzy tiers", 0.05, 0, 1),
+            num("MEGACOMPACT_ALERT_WINDOW_MS", "Alert Window", "Alert evaluation window", 600000, 1000, 3600000, "ms"),
+            num("MEGACOMPACT_P95_BUDGET_MS", "P95 Budget", "Canary p95 budget per tier", 100, 1, 10000, "ms"),
+        ],
+    },
+    {
+        name: "Models",
+        settings: [
+            str("MEGACOMPACT_HYDE_MODEL", "HyDE Model", "LLM model name for HyDE generation", "llama3.2"),
+            str("MEGACOMPACT_RAPTOR_MODEL", "RAPTOR Summary Model", "Ollama model for cluster summarization (empty = extractive)", ""),
+            str("MEGACOMPACT_RAPTOR_URL", "RAPTOR Ollama URL", "Ollama endpoint for RAPTOR summarization", "http://127.0.0.1:11434"),
+            num("MEGACOMPACT_EMBED_CACHE", "Embed Cache Size", "Embedding cache entries (0 = disabled)", 256, 0, 10000),
+        ],
+    },
+];
+/** Flat index of every setting by key for O(1) lookup + validation. */
+export const SETTING_BY_KEY = new Map(SETTINGS.flatMap((c) => c.settings).map((s) => [s.key, s]));
+/**
+ * Adjustable MEGACOMPACT_* env vars intentionally NOT surfaced in the settings
+ * UI. Handled by other panels, install-time paths, dev-only, internal tuning,
+ * or special string formats the generic form cannot represent.
+ * Kept as string literals so regression_check.py's coverage scan sees them.
+ */
+export const EXCLUDED_SETTINGS = [
+    // Embedder telemetry — handled by EmbedderSetup.
+    "MEGACOMPACT_EMBEDDING_URL",
+    "MEGACOMPACT_EMBEDDING_KEY",
+    "MEGACOMPACT_EMBEDDING_HEADERS",
+    "MEGACOMPACT_EMBEDDING_DIM",
+    "MEGACOMPACT_OLLAMA_MODEL",
+    "MEGACOMPACT_ALLOW_REMOTE_EMBEDDER",
+    // Install-time paths.
+    "MEGACOMPACT_STATE_DIR",
+    "MEGACOMPACT_INDEX_DIR",
+    "MEGACOMPACT_VECTOR_INDEX_DIR",
+    "MEGACOMPACT_TURNS_DB_PATH",
+    // Dev-only.
+    "MEGACOMPACT_EXT_SCAN_DIR",
+    "MEGACOMPACT_EXT_USER_DIR",
+    // PGlite internal tuning.
+    "MEGACOMPACT_PGLITE_DISABLED",
+    "MEGACOMPACT_PGLITE_OPEN_TIMEOUT_MS",
+    "MEGACOMPACT_PGLITE_CLOSE_TIMEOUT_MS",
+    "MEGACOMPACT_PGLITE_QUERY_TIMEOUT_MS",
+    // Special string formats the generic form cannot represent.
+    "MEGACOMPACT_WIKI_K_RANGE",
+    "MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS",
+    // Internal calibration, not user-facing.
+    "MEGACOMPACT_FTS5_MAX_BM25",
+];