pi-mega-compact 0.8.21 → 0.8.23

package/extensions/mega-commands.ts

@@ -14,6 +14,7 @@ import { decompressSmart } from "../src/store/compression.js";
 import { loadMetrics, fpRate, p95, defaultMetricsPath } from "../src/monitoring.js";
 import { type MegaRuntime, C, recentUserQuery } from "./mega-runtime.js";
 import { runCompact, doRecall, doRecallAsync } from "./mega-pipeline.js";
+import { vectorStats, vectorRepoStats, vectorDataInvariant } from "../src/vectorStore.js";
 import type { MegaConfig } from "./mega-config.js";
 
 /** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
@@ -95,9 +96,9 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
       const pct = usage?.percent != null ? `${usage.percent}%` : "n/a";
       const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
       const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
-      const st = runtime.store.stats(sid);
-      const repo = runtime.store.repoStats();
-      const di = runtime.store.dataInvariant();
+      const st = vectorStats(runtime.store, sid);
+      const repo = vectorRepoStats(runtime.store);
+      const di = vectorDataInvariant(runtime.store);
       const fmtB = (b: number) =>
         b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
           b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;

package/extensions/mega-events/agent-handlers.ts

@@ -20,6 +20,7 @@ import { isMegaCache } from "../../src/game/scoring.js";
 import { resolveRepoRoot } from "../mega-config.js";
 import { classifyError } from "./error-classifier.js";
 import { safeSendUserMessage } from "./send-safe.js";
+import { vectorStats } from "../../src/vectorStore.js";
 
 /** Register agent/turn tracking event handlers. */
 export function registerAgentHandlers(
@@ -244,7 +245,7 @@ export function registerAgentHandlers(
 		try {
 			if (runtime.getCachedGameState().game_mode_on) {
 				const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
-				const st = runtime.store.stats(runtime.rt.sessionId);
+				const st = vectorStats(runtime.store, runtime.rt.sessionId);
 				const cachePct = st.dedupHitRate * 100;
 				const modelId = runtime.currentModel?.modelId ?? "unknown";
 				recordScore(runtime.currentStateDir, {

package/extensions/mega-events/compact-handlers.ts

@@ -110,6 +110,34 @@ export function registerCompactHandlers(
 				activeAgents: runtime.activeAgents,
 			});
 			if (!config.auto) return {}; // let pi run its own native compaction
+
+			// S38.5: COMPACT-DEDUP RACE GUARD — pi's _runAutoCompaction fires
+			// session_before_compact from within the agent loop (after agent_end,
+			// _handlePostAgentRun → _checkCompaction → _runAutoCompaction). If a
+			// compaction just completed (within the cooldown window), pi's
+			// prepareCompaction may still see the last entry as a compaction but
+			// the extension's session_compact handler hasn't been called yet, OR
+			// the setTimeout deferred ctx.compact() from the agent_end handler is
+			// about to race. Skip the durable trim and let pi run its own compact
+			// (or no-op if the branch is already compacted). Matches the cooldown
+			// in agent-handlers.ts so both call sites are consistent.
+			const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
+			const sinceCompact = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
+			if (sinceCompact < cooldownMs) {
+				runtime.logger.info("before-compact-skip-recent", {
+					sessionId: runtime.rt.sessionId,
+					reason: event.reason,
+					sinceCompactMs: sinceCompact,
+					cooldownMs,
+				});
+				runtime.diagBeforeCompactSupplied++;
+				const fb = fallbackCompaction(event);
+				if (fb) {
+					return { compaction: fb.compaction };
+				}
+				return {};
+			}
+
 			try {
 				const result = driveNativeCompaction(event, runtime, config);
 				if (result && result.compaction.summary?.trim()) {

package/extensions/mega-events/session-handlers.ts

@@ -17,6 +17,7 @@ import {
 	doRecallAsync,
 } from "../mega-pipeline.js";
 import { recallMemoriesAndInline } from "../../src/recall.js";
+import { vectorStats } from "../../src/vectorStore.js";
 import type { MegaConfig } from "../mega-config.js";
 
 /** Register session lifecycle event handlers. */
@@ -50,7 +51,7 @@ export function registerSessionHandlers(
 		if (config.autoInline) {
 			const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
 			const query = recentUserQuery(ctx);
-			if (query && runtime.store.stats(sid).checkpointCount > 0) {
+			if (query && vectorStats(runtime.store, sid).checkpointCount > 0) {
 				// S17: use the async variant on resume so cross-repo HNSW recall can
 				// augment when this repo's store is thin. session_start is an async-safe
 				// point (unlike the mid-turn context handler, which stays sync).

package/extensions/mega-pipeline/compact.ts

@@ -27,6 +27,7 @@ import { runRaptor } from "../../src/dedup/raptor/index.js";
 import { loadDedupConfig } from "../../src/config/dedup.js";
 import { upsertEmbedding as indexUpsertEmbedding } from "../../src/store/vectorIndex.js";
 import { runMemoryReview } from "./memory-review.js";
+import { vectorList } from "../../src/vectorStore.js";
 
 export type RunCompactResult =
   | { skipped: true }
@@ -212,7 +213,7 @@ function doCompact(
   if (config.raptorEnabled && !result.deduped) {
     try {
       const dd = loadDedupConfig();
-      const all = runtime.store.list(sid);
+      const all = vectorList(runtime.store, sid);
       const leaves = all.map((cp) => ({
         id: cp.checkpointId,
         messages: [],
@@ -248,7 +249,7 @@ function doCompact(
   // Non-fatal: a WASM init failure degrades to the sync scan silently.
   if (!result.deduped) {
     try {
-      const all = runtime.store.list(sid);
+      const all = vectorList(runtime.store, sid);
       const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
       if (latest?.embedding) {
         void indexUpsertEmbedding(

package/extensions/mega-runtime/state.ts

@@ -14,7 +14,7 @@ import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
 import { join } from "node:path";
 import { appendFileSync, mkdirSync } from "node:fs";
-import { VectorStore } from "../../src/vectorStore.js";
+import { VectorStore, vectorStats, vectorRepoStats, vectorDataInvariant } from "../../src/vectorStore.js";
 import { toEngineMessages } from "../../src/adapt.js";
 import { normalizeSessionId } from "../../src/store.js";
 import { Logger } from "../../src/log.js";
@@ -365,8 +365,8 @@ export class MegaRuntime {
 		// never break the per-repo compaction path. Runs only on repo-switch
 		// (this branch), so it's infrequent — not per-context-event.
 		try {
-			const repo = this.store.repoStats();
-			const di = this.store.dataInvariant();
+			const repo = vectorRepoStats(this.store);
+			const di = vectorDataInvariant(this.store);
 			const root = key !== dir ? key : (resolveRepoRoot(cwd ?? dir) ?? dir);
 			upsertRepoRegistry({
 				repoRoot: root,
@@ -404,9 +404,9 @@ export class MegaRuntime {
 			return;
 		}
 		const perfT0 = performance.now();
-		const st = this.store.stats(this.rt.sessionId);
-		const repo = this.store.repoStats();
-		const di = this.store.dataInvariant();
+		const st = vectorStats(this.store, this.rt.sessionId);
+		const repo = vectorRepoStats(this.store);
+		const di = vectorDataInvariant(this.store);
 		// Live + store-wide cache-hit / compaction counters for the dashboard.
 		const ds = getDedupStats(this.currentStateDir);
 		const cacheHitsTotal = ds.deduped + getRecallInjected(this.currentStateDir);
@@ -555,7 +555,7 @@ export class MegaRuntime {
 				total: getCompactCount(this.currentStateDir),
 			},
 			timeSaved: {
-				compact: { sessionSec: sec(this.rt.tokensSaved), totalSec: sec(this.store.repoStats().tokensSaved) },
+				compact: { sessionSec: sec(this.rt.tokensSaved), totalSec: sec(vectorRepoStats(this.store).tokensSaved) },
 				cacheHit: { sessionSec: sec(this.rt.cacheHitTokens), totalSec: sec(cacheHitsTotalTokens) },
 			},
 			model,

package/extensions/openclaw-mega-compact.ts

@@ -22,7 +22,7 @@ import {
   type CompactResult,
 } from "../src/engine.js";
 import { recallAndInline, recallMemoriesAndInline, type RecallInjectResult } from "../src/recall.js";
-import { VectorStore } from "../src/vectorStore.js";
+import { VectorStore, vectorStats } from "../src/vectorStore.js";
 import type { EngineMessage } from "../src/types.js";
 
 // ---------------------------------------------------------------------------
@@ -260,7 +260,7 @@ export default definePluginEntry({
         const sessionId = (args as Record<string, string>)?.sessionId ?? "global";
 
         try {
-          const stats = store.stats(sessionId);
+          const stats = vectorStats(store, sessionId);
           const parts: string[] = [
             `**Mega Compact Status**`,
             `Session: ${sessionId}`,

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.8.21",
+  "version": "0.8.23",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
-  "license": "BSD-2-Clause",
+  "license": "BSD-3-Clause",
   "author": "TheArchitectit",
   "homepage": "https://github.com/TheArchitectit/pi-mega-compact",
   "repository": {

package/src/dedup/raptor/multilevel.test.ts

@@ -0,0 +1,278 @@
+/**
+ * multilevel.test.ts — hermetic unit tests for S42A multi-level RAPTOR retrieval.
+ *
+ * Tests the scoreTreeLevels → expandLeafDescendants → deduplicateMultilevelHits
+ * → multilevelRetrieval pipeline. No network, no live store — uses TrigramEmbedder
+ * and synthetic RaptorTrees built from makeLeaves().
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { TrigramEmbedder } from "../../embedder.js";
+import { buildRaptorTree, type Leaf } from "./tree.js";
+import {
+  scoreTreeLevels,
+  expandLeafDescendants,
+  deduplicateMultilevelHits,
+  multilevelRetrieval,
+  type MultilevelHit,
+} from "./multilevel.js";
+import type { EngineMessage } from "../../types.js";
+
+function msg(text: string): EngineMessage {
+  return { role: "user", text };
+}
+
+/** Build N distinct leaves with deterministic content. */
+function makeLeaves(n: number, embedder = new TrigramEmbedder()): Leaf[] {
+  const leaves: Leaf[] = [];
+  for (let i = 0; i < n; i++) {
+    const text = `topic ${i % 7}: the module ${i} validated the session token and refreshed the cache for region ${i}`;
+    leaves.push({
+      id: `leaf_${i}`,
+      messages: [msg(text)],
+      sourceText: text,
+      embedding: embedder.embed(text),
+    });
+  }
+  return leaves;
+}
+
+// ── test: scoreTreeLevels returns nodes at all levels ────────────────────────
+
+test("scoreTreeLevels returns results at multiple tree levels", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const hits = scoreTreeLevels("the auth module validates the session token", tree, {
+    embedder,
+  });
+
+  assert.ok(hits.length > 0, "should return hits");
+
+  // Should include both leaf (level 0) and cluster (level ≥ 1) hits.
+  const levels = new Set(hits.map((h) => h.level));
+  assert.ok(levels.has(0), "should have leaf-level hits");
+  if (tree.levels > 1) {
+    const hasCluster = [...levels].some((l) => l >= 1);
+    assert.ok(hasCluster, "should have cluster-level hits for multi-level tree");
+  }
+
+  // All hits should have valid scores.
+  for (const h of hits) {
+    assert.ok(h.score >= 0 && h.score <= 1, `score ${h.score} out of range`);
+    assert.ok(h.rawScore >= 0 && h.rawScore <= 1, `rawScore ${h.rawScore} out of range`);
+    assert.ok(h.score <= h.rawScore, "weighted score <= raw score (level weights ≤ 1)");
+  }
+
+  // Hits should be sorted by score descending.
+  for (let i = 1; i < hits.length; i++) {
+    assert.ok(
+      hits[i - 1].score >= hits[i].score,
+      `hits not sorted: hit[${i - 1}].score=${hits[i - 1].score} < hit[${i}].score=${hits[i].score}`,
+    );
+  }
+});
+
+// ── test: level weights affect scoring ───────────────────────────────────────
+
+test("level weights shift scores: higher weight for level → higher weighted score", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const query = "the auth module validates the session token";
+
+  // Uniform weights: all levels scored equally.
+  const uniform = scoreTreeLevels(query, tree, {
+    embedder,
+    levelWeights: [1.0, 1.0, 1.0, 1.0, 1.0],
+  });
+
+  // Penalized weights: higher levels penalized.
+  const penalized = scoreTreeLevels(query, tree, {
+    embedder,
+    levelWeights: [1.0, 0.1, 0.1, 0.1, 0.1],
+  });
+
+  // With penalized weights, cluster-level hits should score lower.
+  const clusterUniform = uniform.filter((h) => h.level >= 1);
+  const clusterPenalized = penalized.filter((h) => h.level >= 1);
+
+  if (clusterUniform.length > 0 && clusterPenalized.length > 0) {
+    const avgUniform =
+      clusterUniform.reduce((s, h) => s + h.score, 0) / clusterUniform.length;
+    const avgPenalized =
+      clusterPenalized.reduce((s, h) => s + h.score, 0) / clusterPenalized.length;
+    assert.ok(
+      avgPenalized < avgUniform,
+      `penalized cluster avg (${avgPenalized.toFixed(3)}) should be < uniform (${avgUniform.toFixed(3)})`,
+    );
+  }
+});
+
+// ── test: expandLeafDescendants adds leaf hits ──────────────────────────────
+
+test("expandLeafDescendants adds leaf descendants for cluster hits", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const query = "the auth module validates the session token";
+  const scored = scoreTreeLevels(query, tree, { embedder });
+  const qv = embedder.embed(query);
+
+  // Take only cluster hits (level ≥ 1).
+  const clusterHits = scored.filter((h) => !h.isLeaf).slice(0, 3);
+  assert.ok(clusterHits.length > 0, "should have cluster hits");
+
+  const expanded = expandLeafDescendants(
+    clusterHits,
+    tree,
+    5, // maxPerCluster
+    embedder,
+    qv,
+  );
+
+  // Expanded set should include leaf hits.
+  const leafHits = expanded.filter((h) => h.isLeaf);
+  assert.ok(leafHits.length > 0, "should have expanded leaf hits");
+  assert.ok(
+    expanded.length > clusterHits.length,
+    `expanded (${expanded.length}) should be > cluster hits (${clusterHits.length})`,
+  );
+
+  // No duplicate ids.
+  const ids = new Set(expanded.map((h) => h.nodeId));
+  assert.equal(ids.size, expanded.length, "no duplicate node ids");
+});
+
+// ── test: deduplicateMultilevelHits removes cluster when leaves present ─────
+
+test("deduplicateMultilevelHits removes cluster hits when leaf children are present", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const query = "the auth module validates the session token";
+  const scored = scoreTreeLevels(query, tree, { embedder });
+
+  // Take a cluster hit and its leaf children.
+  const cluster = scored.find((h) => !h.isLeaf);
+  assert.ok(cluster, "should have a cluster hit");
+
+  const leafChildren: MultilevelHit[] = cluster!.leafIds.slice(0, 2).map((lid) => ({
+    nodeId: lid,
+    level: 0,
+    score: 0.5,
+    rawScore: 0.5,
+    isLeaf: true,
+    leafIds: [lid],
+    summary: "",
+    embedding: cluster!.embedding,
+  }));
+
+  const mixed = [cluster!, ...leafChildren];
+  const deduped = deduplicateMultilevelHits(mixed);
+
+  // Cluster should be removed because its leaf children are present.
+  assert.ok(
+    !deduped.find((h) => h.nodeId === cluster!.nodeId),
+    "cluster hit should be removed when leaf children are present",
+  );
+  assert.equal(deduped.length, leafChildren.length, "only leaf hits remain");
+});
+
+test("deduplicateMultilevelHits keeps cluster hits when no leaf children present", () => {
+  const clusterHit: MultilevelHit = {
+    nodeId: "cluster_1",
+    level: 1,
+    score: 0.8,
+    rawScore: 0.8,
+    isLeaf: false,
+    leafIds: ["leaf_1", "leaf_2"],
+    summary: "summarized content",
+    embedding: [1, 0, 0],
+  };
+
+  const deduped = deduplicateMultilevelHits([clusterHit]);
+  assert.equal(deduped.length, 1, "cluster should be kept when no leaf children present");
+  assert.equal(deduped[0].nodeId, "cluster_1");
+});
+
+// ── test: multilevelRetrieval returns cluster + leaf mix ────────────────────
+
+test("multilevelRetrieval returns a mix of cluster and leaf hits", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+    embedder,
+    k: 5,
+    leafExpansion: true,
+    maxLeafExpansion: 3,
+  });
+
+  assert.ok(hits.length > 0, "should return hits");
+  assert.ok(hits.length <= 5, "should respect k=5");
+
+  // Should include leaf hits.
+  const leafHits = hits.filter((h) => h.isLeaf);
+  assert.ok(leafHits.length > 0, "should include leaf hits");
+
+  // All hits should have valid properties.
+  for (const h of hits) {
+    assert.ok(h.nodeId, "hit should have nodeId");
+    assert.ok(typeof h.level === "number", "hit should have numeric level");
+    assert.ok(h.score >= 0, "hit score should be non-negative");
+    assert.ok(Array.isArray(h.leafIds), "hit should have leafIds array");
+  }
+});
+
+test("multilevelRetrieval respects k parameter", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(100);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 8 });
+
+  for (const k of [1, 3, 5, 10]) {
+    const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+      embedder,
+      k,
+    });
+    assert.ok(
+      hits.length <= k,
+      `k=${k}: got ${hits.length} hits, should be ≤ ${k}`,
+    );
+  }
+});
+
+test("multilevelRetrieval returns empty for empty tree", () => {
+  const embedder = new TrigramEmbedder();
+  const emptyTree = { nodes: new Map(), rootId: null, levels: 0, timedOut: false };
+  const hits = multilevelRetrieval("any query", emptyTree, { embedder });
+  assert.deepEqual(hits, []);
+});
+
+test("multilevelRetrieval with leafExpansion=false skips leaf expansion", () => {
+  const embedder = new TrigramEmbedder();
+  const leaves = makeLeaves(50);
+  const tree = buildRaptorTree(leaves, { embedder, clustersPerLevel: 4 });
+
+  const hits = multilevelRetrieval("the auth module validates the session token", tree, {
+    embedder,
+    k: 5,
+    leafExpansion: false,
+  });
+
+  assert.ok(hits.length > 0, "should still return hits");
+
+  // With leaf expansion off and tree having multiple levels, we may get
+  // cluster hits that are NOT expanded. The mix depends on tree structure.
+  // Just verify we got valid results.
+  for (const h of hits) {
+    assert.ok(h.nodeId, "hit should have nodeId");
+    assert.ok(h.score >= 0, "hit score should be non-negative");
+  }
+});