pi-mega-compact 0.13.4 → 0.13.6

package/extensions/dashboard-server/api-contracts/rag-settings.ts

@@ -1,42 +1,52 @@
 /**
- * api-contracts/rag-settings.ts — RAG feature flag API response types.
+ * api-contracts/rag-settings.ts — comprehensive settings API response types.
  *
- * Types for the /api/rag-settings endpoint used by the dashboard RAG Settings
- * panel to read and toggle the five RAG feature flags (B1–B5).
+ * Types for the /api/rag-settings endpoint used by the dashboard Setup panel to
+ * read and update every adjustable MEGACOMPACT_* setting, grouped by category.
  *
  * PREVENT-PI-004: type definitions only, no network code.
  * PREVENT-011: no `any` type — all types are explicit.
  */
 
-/**单个 RAG feature flag state for the dashboard. */
-export interface RagFlagState {
-	/** Env var key without the _DISABLED suffix, e.g. "MEGACOMPACT_HYDE". */
+/** A single configurable setting's metadata + current value. */
+export interface SettingState {
 	readonly key: string;
-	/** Short human-readable label, e.g. "HyDE (Hypothetical Document Embeddings)". */
 	readonly label: string;
-	/** One-line description of what the flag does. */
 	readonly description: string;
-	/** Whether the flag is currently enabled (true = ON, false = disabled via _DISABLED). */
-	readonly enabled: boolean;
-	/** Whether this flag requires an LLM embedder (HttpEmbedder) to function. */
+	readonly category: string;
+	readonly type: "boolean" | "number" | "string";
+	readonly value: string | number | boolean;
+	readonly default: string | number | boolean;
+	readonly disabledConvention: boolean;
 	readonly requiresLlm: boolean;
+	readonly unit?: string;
+	readonly min?: number;
+	readonly max?: number;
 }
 
-/** Response for GET /api/rag-settings. */
-export interface RagSettingsResponse {
-	readonly flags: RagFlagState[];
-	/** Whether an HttpEmbedder (LLM) is currently active — gates HyDE toggle. */
+/** Response for GET /api/rag-settings — all settings grouped by category. */
+export interface SettingsResponse {
+	readonly categories: ReadonlyArray<{ name: string; settings: SettingState[] }>;
 	readonly llmActive: boolean;
 }
 
-/** Request body for POST /api/rag-settings. */
-export interface RagSettingsRequest {
-	/** Map of flag key (e.g. "MEGACOMPACT_HYDE") to desired enabled state. */
-	readonly flags: Record<string, boolean>;
+/** Request body for POST /api/rag-settings — update a single setting. */
+export interface SettingsUpdateRequest {
+	readonly key: string;
+	readonly value: string;
 }
 
 /** Response for POST /api/rag-settings. */
-export interface RagSettingsResponsePost {
+export interface SettingsResponsePost {
 	readonly envPath: string;
 	readonly restartRequired: boolean;
 }
+
+/** @deprecated Use SettingState instead */
+export type RagFlagState = SettingState;
+/** @deprecated Use SettingsResponse instead */
+export type RagSettingsResponse = SettingsResponse;
+/** @deprecated Use SettingsUpdateRequest instead */
+export type RagSettingsRequest = SettingsUpdateRequest;
+/** @deprecated Use SettingsResponsePost instead */
+export type RagSettingsResponsePost = SettingsResponsePost;

package/extensions/dashboard-server/api-contracts/wiki.ts

@@ -0,0 +1,138 @@
+/**
+ * api-contracts/wiki.ts — W2 wiki revival contract types.
+ *
+ * Shapes for the 7 wiki endpoints plus the SSE events the mutations emit.
+ * `CurationResult` is the canonical type from src/wiki/curation.ts (single
+ * source of truth), re-exported here for route + client parity. Type-only —
+ * no network (PREVENT-PI-004), no `any` (PREVENT-011).
+ */
+import type { CurationResult } from "../../../src/wiki/curation.js";
+
+/** One topic row in the wiki index (with resolved custom label). */
+export interface WikiIndexEntry {
+	id: string;
+	label: string;
+	memoryCount: number;
+	lastUpdated: number;
+	/** True when the label is a user override, not the auto-generated one. */
+	edited: boolean;
+	/** Cumulative override kinds for badge provenance. */
+	overrideKinds: Array<"label" | "merge" | "split">;
+}
+
+/** GET /api/wiki/index — wiki landing page payload. */
+export interface WikiIndexResponse {
+	topics: WikiIndexEntry[];
+	totalTopics: number;
+	totalMemories: number;
+	lastRebuildAt: number | null;
+}
+
+/** Provenance for one memory within a wiki topic. */
+export interface MemoryProvenance {
+	memoryId: string;
+	sessionId: string;
+	assignedAt: number;
+	method: string;
+}
+
+/** GET /api/wiki/topic/:topicId — single topic page. */
+export interface WikiPageResponse {
+	topic: WikiIndexEntry;
+	summary: string;
+	keyMemories: Array<{
+		memoryId: string;
+		content: string;
+		timestamp: number;
+		importance: number;
+	}>;
+	provenance: MemoryProvenance[];
+	relatedTopicIds: string[];
+	generatedAt: number;
+}
+
+/** PUT /api/wiki/topic/:topicId/label — rename request body. */
+export interface RenameTopicRequest {
+	label: string;
+}
+
+/** POST /api/wiki/merge — merge source into target. */
+export interface MergeTopicsRequest {
+	sourceTopicId: string;
+	targetTopicId: string;
+}
+
+/** POST /api/wiki/topic/:topicId/split — split request body. */
+export interface SplitTopicRequest {
+	memoryIds: string[];
+}
+
+/** One flat node row in the D3 evolution graph. */
+export interface TopicEvolutionNode {
+	id: string;
+	label: string;
+	memoryCount: number;
+}
+
+/** One edge between evolved topics. */
+export interface TopicEvolutionEdge {
+	source: string;
+	target: string;
+	kind: "merge" | "split";
+	at: number;
+}
+
+/** GET /api/wiki/evolution — nodes + edges + time buckets for D3. */
+export interface TopicEvolutionResponse {
+	nodes: TopicEvolutionNode[];
+	edges: TopicEvolutionEdge[];
+	buckets: Array<{ bucket: number; count: number }>;
+}
+
+/** GET /api/wiki/topic/:topicId/timeline — per-topic time buckets. */
+export interface TopicTimelineResponse {
+	topicId: string;
+	buckets: Array<{ bucket: number; count: number }>;
+	total: number;
+}
+
+/** SSE: wiki model rebuilt. */
+export interface SseWikiRebuilt {
+	type: "wiki_rebuilt";
+	ts: string;
+	topicCount: number;
+}
+
+/** SSE: a topic was renamed. */
+export interface SseWikiTopicRenamed {
+	type: "wiki_topic_renamed";
+	ts: string;
+	topicId: string;
+	label: string;
+	edited: boolean;
+}
+
+/** SSE: two topics were merged. */
+export interface SseWikiTopicsMerged {
+	type: "wiki_topics_merged";
+	ts: string;
+	sourceTopicId: string;
+	targetTopicId: string;
+}
+
+/** SSE: a topic was split. */
+export interface SseWikiTopicSplit {
+	type: "wiki_topic_split";
+	ts: string;
+	fromTopicId: string;
+	toTopicId: string;
+}
+
+/** Union of all wiki SSE events. */
+export type WikiSseEvent =
+	| SseWikiRebuilt
+	| SseWikiTopicRenamed
+	| SseWikiTopicsMerged
+	| SseWikiTopicSplit;
+
+export type { CurationResult };

package/extensions/dashboard-server/routes-rag-metrics.ts

@@ -0,0 +1,82 @@
+/**
+ * dashboard-server/routes-rag-metrics.ts — GET /api/rag-metrics route (H2).
+ *
+ * Aggregates the H1 HyDE + recall-quality telemetry columns from turns.db into
+ * a compact response (flags, totals, recent rows, daily series).
+ *
+ * Guardrails: PREVENT-PI-004 (loopback-only), PREVENT-002 (parameterized),
+ * PREVENT-011 (no `any`).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { DatabaseSync } from "node:sqlite";
+import type { RouteContext } from "./routes-core.js";
+import { openTurnStore } from "../../src/store/turns/connection.js";
+import {
+	listTelemetryTurns,
+	aggregateDailyTelemetry,
+} from "../../src/store/turns/hydeStore.js";
+import { RAG_HYDE_ENABLED, RAG_RECALL_METRICS } from "../../src/config.js";
+import type { RagMetricsResponse } from "./api-contracts/rag-metrics.js";
+
+function sendJson(res: ServerResponse, status: number, body: unknown): void {
+	// guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+	res.writeHead(status, { "Content-Type": "application/json" });
+	res.end(JSON.stringify(body));
+}
+
+export function handleRagMetrics(
+	req: IncomingMessage,
+	res: ServerResponse,
+	ctx: RouteContext,
+): boolean {
+	const url = req.url ?? "";
+	if (url !== "/api/rag-metrics" || req.method !== "GET") return false;
+
+	let db: DatabaseSync | null;
+	try {
+		db = openTurnStore(ctx.stateDir);
+	} catch {
+		db = null;
+	}
+	const recent = db ? listTelemetryTurns(db, { limit: 200 }) : [];
+	const daily = db ? aggregateDailyTelemetry(db, 14) : [];
+
+	const hydeRanTurns = recent.filter((t) => t.hydeRan === 1).length;
+	const liftSum = recent.reduce((s, t) => s + t.hydeLift, 0);
+	const genSum = recent.reduce((s, t) => s + t.hydeGenerationMs, 0);
+	const scored = recent.filter(
+		(t) => t.recallScore > 0 || t.recallPass === 1,
+	);
+	const avgScore =
+		scored.length > 0
+			? Math.round(
+					(scored.reduce((s, t) => s + t.recallScore, 0) / scored.length) * 100,
+				) / 100
+			: null;
+	const passCount = scored.filter((t) => t.recallPass === 1).length;
+	const recentPassRate =
+		scored.length > 0 ? Math.round((passCount / scored.length) * 100) / 100 : 0;
+
+	const body: RagMetricsResponse = {
+		flags: {
+			hydeEnabled: RAG_HYDE_ENABLED(),
+			recallMetricsEnabled: RAG_RECALL_METRICS(),
+		},
+		totals: {
+			telemetryTurns: recent.length,
+			hydeRanTurns,
+			avgLift:
+				recent.length > 0
+					? Math.round((liftSum / recent.length) * 100) / 100
+					: 0,
+			avgScore,
+			avgGenerationMs: recent.length > 0 ? Math.round(genSum / recent.length) : 0,
+			recentPassRate,
+		},
+		recent: recent.slice(0, 50),
+		daily,
+	};
+	sendJson(res, 200, body);
+	return true;
+}

package/extensions/dashboard-server/routes-rag-settings-helpers.ts

@@ -0,0 +1,314 @@
+/**
+ * dashboard-server/routes-rag-settings-helpers.ts — SETTINGS inventory.
+ *
+ * The full catalog of user-adjustable MEGACOMPACT_* settings surfaced by the
+ * dashboard Setup panel, grouped by category. Kept separate from the route
+ * handler so the inventory can grow without bloating the handler file.
+ *
+ * PREVENT-011: no `any` type.
+ */
+
+
+/**
+ * Base metadata for a single setting entry before its live `value` is resolved.
+ * `category` and `value` are filled in at read time by the handler.
+ */
+export interface SettingSpec {
+	key: string;
+	label: string;
+	description: string;
+	type: "boolean" | "number" | "string";
+	default: string | number | boolean;
+	/** True when this is a `_DISABLED`-convention opt-out flag. */
+	disabledConvention: boolean;
+	requiresLlm: boolean;
+	unit?: string;
+	min?: number;
+	max?: number;
+}
+
+// Shorthand builders to keep the inventory terse and unambiguous.
+const boolFlag = (
+	key: string,
+	label: string,
+	description: string,
+	def: boolean,
+	requiresLlm = false,
+): SettingSpec => ({
+	key,
+	label,
+	description,
+	type: "boolean",
+	default: def,
+	disabledConvention: true,
+	requiresLlm,
+});
+
+const boolDirect = (
+	key: string,
+	label: string,
+	description: string,
+	def: boolean,
+): SettingSpec => ({
+	key,
+	label,
+	description,
+	type: "boolean",
+	default: def,
+	disabledConvention: false,
+	requiresLlm: false,
+});
+
+const num = (
+	key: string,
+	label: string,
+	description: string,
+	def: number,
+	min: number,
+	max: number,
+	unit?: string,
+): SettingSpec => ({
+	key,
+	label,
+	description,
+	type: "number",
+	default: def,
+	disabledConvention: false,
+	requiresLlm: false,
+	min,
+	max,
+	...(unit ? { unit } : {}),
+});
+
+const str = (
+	key: string,
+	label: string,
+	description: string,
+	def: string,
+): SettingSpec => ({
+	key,
+	label,
+	description,
+	type: "string",
+	default: def,
+	disabledConvention: false,
+	requiresLlm: false,
+});
+
+/** Every adjustable setting, grouped by category. Read-only after module load. */
+export const SETTINGS: ReadonlyArray<{
+	name: string;
+	settings: SettingSpec[];
+}> = [
+	{
+		name: "RAG Pipeline",
+		settings: [
+			boolFlag(
+				"MEGACOMPACT_QUERY_REFORMULATION",
+				"Query Reformulation",
+				"TF-IDF keyword expansion for vague recall queries",
+				true,
+			),
+			boolFlag(
+				"MEGACOMPACT_TIERED_ROUTER",
+				"Tiered Recall Router",
+				"L0 cache → L1 FTS5 → L2 HNSW routing",
+				true,
+			),
+			boolFlag(
+				"MEGACOMPACT_RECALL_METRICS",
+				"Recall Quality Metrics",
+				"Precision/recall scoring and logging",
+				true,
+			),
+			boolFlag(
+				"MEGACOMPACT_MEMORY_GRAPH",
+				"Memory Graph",
+				"Dashboard-oriented memory graph traversal",
+				true,
+			),
+			boolFlag(
+				"MEGACOMPACT_HYDE",
+				"HyDE",
+				"Generate hypothetical answer via LLM, embed it, RRF-fuse",
+				true,
+				true,
+			),
+			boolFlag(
+				"MEGACOMPACT_NEW_UI",
+				"New Dashboard UI",
+				"Tailwind + shadcn visual design (sidebar, glass panels)",
+				true,
+			),
+		],
+	},
+	{
+		name: "Wiki / Turns",
+		settings: [
+			boolFlag(
+				"MEGACOMPACT_WIKI_ENHANCED",
+				"Wiki Enhanced",
+				"User curation (rename/merge/split) + topic evolution",
+				true,
+			),
+			boolFlag(
+				"MEGACOMPACT_WIKI_INCREMENTAL",
+				"Wiki Incremental",
+				"Assign new memories to nearest centroid",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_AUTO_WIKI",
+				"Auto Wiki",
+				"Auto-categorizing wiki (k-means + TF-IDF)",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_WIKI_SEED_FROM_TURNS",
+				"Wiki Seed from Turns",
+				"Seed topic model from raw_transcript when chunks thin",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_TURNS_DB",
+				"Turns DB",
+				"Isolated turns.db store",
+				true,
+			),
+			num(
+				"MEGACOMPACT_WIKI_SILHOUETTE_MIN",
+				"Silhouette Minimum",
+				"Full rebuild trigger threshold",
+				0.2,
+				0,
+				1,
+			),
+			num(
+				"MEGACOMPACT_TURNS_RETENTION_DAYS",
+				"Turn Retention",
+				"Per-turn provenance retention window",
+				30,
+				1,
+				365,
+				"days",
+			),
+			num(
+				"MEGACOMPACT_TURNS_KEEP_MIN",
+				"Turns Keep Minimum",
+				"Min turns kept per conversation during prune",
+				5,
+				1,
+				100,
+			),
+			num(
+				"MEGACOMPACT_WIKI_LABEL_TOP_TERMS",
+				"Wiki Label Terms",
+				"How many TF-IDF terms form a topic label",
+				5,
+				1,
+				20,
+			),
+			num(
+				"MEGACOMPACT_WIKI_REBUILD_EVERY",
+				"Wiki Rebuild Every",
+				"Rebuild topic model every Nth compaction",
+				3,
+				1,
+				50,
+			),
+		],
+	},
+	{
+		name: "Dedup Tiers",
+		settings: [
+			boolDirect("MEGACOMPACT_L0_ENABLED", "L0 Exact Match", "Exact-hash dedup tier", true),
+			boolDirect("MEGACOMPACT_L1_ENABLED", "L1 MinHash/LSH", "Near-dup tier", true),
+			boolDirect("MEGACOMPACT_L2_ENABLED", "L2 Semantic", "Cosine similarity dedup tier", true),
+			boolDirect("MEGACOMPACT_RAPTOR_ENABLED", "RAPTOR", "Hierarchical clustering tier", true),
+			boolDirect("MEGACOMPACT_RAPTOR_MULTILEVEL", "RAPTOR Multi-level", "Multi-level retrieval (score all tree levels)", true),
+			boolDirect("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", "RAPTOR Leaf Expansion", "Leaf node expansion in retrieval", true),
+			boolDirect("MEGACOMPACT_RAPTOR_INCREMENTAL", "RAPTOR Incremental", "Incremental rebuild mode", true),
+			boolDirect("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", "RAPTOR Inject Summaries", "Inject top-level summary nodes into recall", true),
+			boolDirect("MEGACOMPACT_MARK_ONLY_L0", "Mark Only L0", "L0 runs but does not collapse", false),
+			boolDirect("MEGACOMPACT_MARK_ONLY_L1", "Mark Only L1", "L1 runs but does not collapse", false),
+			boolDirect("MEGACOMPACT_MARK_ONLY_L2", "Mark Only L2", "L2 runs but does not collapse", false),
+			boolDirect("MEGACOMPACT_MINILM", "MiniLM Embedder", "Use MiniLM instead of trigram", false),
+		],
+	},
+	{
+		name: "Dedup Thresholds",
+		settings: [
+			num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
+			num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
+			num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
+			num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
+			num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
+			num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),
+			num("MEGACOMPACT_SIMILARITY_BUDGET_MS", "Similarity Budget", "Time budget for similarity computation", 50, 1, 10000, "ms"),
+			num("MEGACOMPACT_L1_VERIFY_BUDGET_MS", "L1 Verify Budget", "L1 verification time budget", 20, 1, 10000, "ms"),
+			num("MEGACOMPACT_L1_CANDIDATE_CAP", "L1 Candidate Cap", "L1 candidate cap", 100, 1, 10000),
+		],
+	},
+	{
+		name: "RAPTOR Tuning",
+		settings: [
+			num("MEGACOMPACT_RAPTOR_BUDGET_MS", "RAPTOR Budget", "RAPTOR time budget", 5000, 100, 60000, "ms"),
+			num("MEGACOMPACT_RAPTOR_CLUSTERS", "RAPTOR Clusters", "Clusters per level", 5, 1, 50),
+			num("MEGACOMPACT_RAPTOR_CONSISTENCY", "RAPTOR Consistency", "Consistency threshold", 0.6, 0, 1),
+			num("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", "RAPTOR Max Leaf Expansion", "Max leaf expansion nodes", 10, 1, 100),
+			num("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", "RAPTOR Freshness", "Skip rebuild when tree is fresh", 4, 1, 168, "hours"),
+			num("MEGACOMPACT_FP_RATE_L0", "FP Rate L0", "FP alert threshold for exact tier", 0.01, 0, 1),
+			num("MEGACOMPACT_FP_RATE_L1L2", "FP Rate L1/L2", "FP alert threshold for fuzzy tiers", 0.05, 0, 1),
+			num("MEGACOMPACT_ALERT_WINDOW_MS", "Alert Window", "Alert evaluation window", 600000, 1000, 3600000, "ms"),
+			num("MEGACOMPACT_P95_BUDGET_MS", "P95 Budget", "Canary p95 budget per tier", 100, 1, 10000, "ms"),
+		],
+	},
+	{
+		name: "Models",
+		settings: [
+			str("MEGACOMPACT_HYDE_MODEL", "HyDE Model", "LLM model name for HyDE generation", "llama3.2"),
+			str("MEGACOMPACT_RAPTOR_MODEL", "RAPTOR Summary Model", "Ollama model for cluster summarization (empty = extractive)", ""),
+			str("MEGACOMPACT_RAPTOR_URL", "RAPTOR Ollama URL", "Ollama endpoint for RAPTOR summarization", "http://127.0.0.1:11434"),
+			num("MEGACOMPACT_EMBED_CACHE", "Embed Cache Size", "Embedding cache entries (0 = disabled)", 256, 0, 10000),
+		],
+	},
+];
+
+/** Flat index of every setting by key for O(1) lookup + validation. */
+export const SETTING_BY_KEY: ReadonlyMap<string, SettingSpec> = new Map(
+	SETTINGS.flatMap((c) => c.settings).map((s) => [s.key, s]),
+);
+
+/**
+ * Adjustable MEGACOMPACT_* env vars intentionally NOT surfaced in the settings
+ * UI. Handled by other panels, install-time paths, dev-only, internal tuning,
+ * or special string formats the generic form cannot represent.
+ * Kept as string literals so regression_check.py's coverage scan sees them.
+ */
+export const EXCLUDED_SETTINGS: readonly string[] = [
+	// Embedder telemetry — handled by EmbedderSetup.
+	"MEGACOMPACT_EMBEDDING_URL",
+	"MEGACOMPACT_EMBEDDING_KEY",
+	"MEGACOMPACT_EMBEDDING_HEADERS",
+	"MEGACOMPACT_EMBEDDING_DIM",
+	"MEGACOMPACT_OLLAMA_MODEL",
+	"MEGACOMPACT_ALLOW_REMOTE_EMBEDDER",
+	// Install-time paths.
+	"MEGACOMPACT_STATE_DIR",
+	"MEGACOMPACT_INDEX_DIR",
+	"MEGACOMPACT_VECTOR_INDEX_DIR",
+	"MEGACOMPACT_TURNS_DB_PATH",
+	// Dev-only.
+	"MEGACOMPACT_EXT_SCAN_DIR",
+	"MEGACOMPACT_EXT_USER_DIR",
+	// PGlite internal tuning.
+	"MEGACOMPACT_PGLITE_DISABLED",
+	"MEGACOMPACT_PGLITE_OPEN_TIMEOUT_MS",
+	"MEGACOMPACT_PGLITE_CLOSE_TIMEOUT_MS",
+	"MEGACOMPACT_PGLITE_QUERY_TIMEOUT_MS",
+	// Special string formats the generic form cannot represent.
+	"MEGACOMPACT_WIKI_K_RANGE",
+	"MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS",
+	// Internal calibration, not user-facing.
+	"MEGACOMPACT_FTS5_MAX_BM25",
+];