pi-mega-compact 0.8.23 → 0.8.25

package/src/boundary.test.ts

@@ -7,6 +7,7 @@ function user(t: string): EngineMessage { return { role: "user", text: t }; }
 function assistant(t: string): EngineMessage { return { role: "assistant", text: t }; }
 function toolUse(n: string, i = "{}"): EngineMessage { return { role: "assistant", text: "", toolName: n, input: i }; }
 function toolResult(n: string, o = "ok"): EngineMessage { return { role: "tool", text: "", toolName: n, output: o }; }
+function custom(t: string): EngineMessage { return { role: "custom", text: t }; }
 
 test("walks back so first preserved message is not an orphaned tool result", () => {
   const messages = [
@@ -25,9 +26,14 @@ test("walks back so first preserved message is not an orphaned tool result", ()
   assert.equal(kept[0].toolName, "search"); // assistant tool-call preserved
 });
 
-test("isBoundarySafe: tool result at boundary with preceding tool use is safe", () => {
+test("isBoundarySafe: cut that drops a toolCall but keeps its toolResult is unsafe (PREVENT-PI-002)", () => {
   const messages = [user("a"), toolUse("search"), toolResult("search")];
-  assert.equal(isBoundarySafe(messages, 2), true);
+  // keepFrom=2 drops the toolUse at index 1 but keeps the toolResult at index 2 →
+  // the preserved run starts on an orphaned tool result. This is the shape the old
+  // check mis-validated (it only compared messages[keepFrom] to messages[keepFrom-1]).
+  assert.equal(isBoundarySafe(messages, 2), false);
+  // keepFrom=1 keeps the toolCall together with its toolResult → safe.
+  assert.equal(isBoundarySafe(messages, 1), true);
 });
 
 test("isBoundarySafe: orphaned tool result without preceding tool use is unsafe", () => {
@@ -59,3 +65,123 @@ test("dropBefore returns original when range is empty", () => {
   const messages = [user("a"), assistant("b")];
   assert.equal(dropBefore(messages, 0, 1), messages);
 });
+
+// --- PREVENT-PI-002 regression cases for arbitrary interleavings ---
+
+test("interleaved custom message between toolCall and toolResult: walk-back keeps the call", () => {
+  // [user, assistant(tc=read), custom/bashExecution, tool(read-result)] with
+  // desired keepFrom=3. The old walk-back saw messages[2] was not a toolUse and
+  // broke at k=2, dropping the assistant toolCall at index 1 while KEEPING its
+  // tool result at index 3 → orphaned tool result → provider 400.
+  const messages = [
+    user("Search for files"),
+    toolUse("read"),
+    custom("bash: ls -la"),
+    toolResult("read", "file contents"),
+    assistant("Done."),
+  ];
+  const [start, end] = computeDropRange(messages, 3, 0);
+  assert.equal(start, 0);
+  assert.equal(end, 1); // keep the assistant toolCall at index 1 with its result
+  const kept = messages.slice(end);
+  assert.equal(kept[0].toolName, "read"); // assistant tool-call preserved
+  assert.ok(kept.some((m) => m.role === "tool" && m.toolName === "read"));
+  // The toolCall and its toolResult are both in the kept run.
+  const callIdx = kept.findIndex((m) => m.role === "assistant" && m.toolName === "read");
+  const resultIdx = kept.findIndex((m) => m.role === "tool" && m.toolName === "read");
+  assert.ok(callIdx !== -1 && resultIdx !== -1 && callIdx < resultIdx);
+});
+
+test("consecutive tool results sharing one call: no-op when the call cannot be kept", () => {
+  // [assistant(tc), T1, T2] with keepFrom=1: dropping the call orphans BOTH T1
+  // and T2. No pair-safe positive cut exists below keepFrom → no-op compaction
+  // (the [start,end) contract preserves a non-zero result only when one exists).
+  const messages = [toolUse("multi"), toolResult("multi", "r1"), toolResult("multi", "r2")];
+  const [start, end] = computeDropRange(messages, 1, 0);
+  assert.equal(start, 0);
+  assert.equal(end, 0); // no-op — pair rule outranks dropping
+  assert.equal(dropBefore(messages, 1, 0), messages);
+});
+
+test("consecutive tool results sharing one call: safe cut keeps the call with both results", () => {
+  // [user, assistant(tc), T1, T2, user2] — keepFrom=3 would orphan T2; the guard
+  // walks back to keep the call (dropEnd=1, only the first user is dropped).
+  const messages = [
+    user("u1"),
+    toolUse("multi"),
+    toolResult("multi", "r1"),
+    toolResult("multi", "r2"),
+    user("u2"),
+  ];
+  const [start, end] = computeDropRange(messages, 3, 0);
+  assert.equal(start, 0);
+  assert.equal(end, 1); // keep [assistant(tc), T1, T2, user2]
+  const kept = messages.slice(end);
+  const callIdx = kept.findIndex((m) => m.role === "assistant" && m.toolName === "multi");
+  const r1Idx = kept.findIndex((m) => m.role === "tool" && m.output === "r1");
+  const r2Idx = kept.findIndex((m) => m.role === "tool" && m.output === "r2");
+  assert.ok(callIdx < r1Idx && r1Idx < r2Idx);
+});
+
+test("consecutive shared-call results: dropping call + all results together is safe", () => {
+  // keepFrom=4 preserves only the trailing user — the call and BOTH results are
+  // dropped together, so nothing is orphaned.
+  const messages = [
+    user("u1"),
+    toolUse("multi"),
+    toolResult("multi", "r1"),
+    toolResult("multi", "r2"),
+    user("u2"),
+  ];
+  const [start, end] = computeDropRange(messages, 4, 0);
+  assert.equal(start, 0);
+  assert.equal(end, 4);
+  assert.deepEqual(messages.slice(end), [user("u2")]);
+});
+
+test("keepFrom landing on a call whose results follow is safe", () => {
+  // [user, assistant(tc1), T1, assistant(tc2), T2, user2] keepFrom=3 → preserved
+  // run starts on assistant(tc2) at index 3, whose result T2 follows. Safe.
+  const messages = [
+    user("u1"),
+    toolUse("read"),
+    toolResult("read", "r1"),
+    toolUse("write"),
+    toolResult("write", "r2"),
+    user("u2"),
+  ];
+  const [start, end] = computeDropRange(messages, 3, 0);
+  assert.equal(start, 0);
+  assert.equal(end, 3);
+  const kept = messages.slice(end);
+  assert.equal(kept[0].role, "assistant");
+  assert.equal(kept[0].toolName, "write");
+  assert.ok(kept.some((m) => m.role === "tool" && m.toolName === "write"));
+});
+
+test("anchor floor + pair-rule conflict: pair rule wins, drop less", () => {
+  // [assistant(tc), user, T] with anchor=1: the anchor floor wants dropEnd<=1
+  // (keep the user at index 1), but keeping from index 1 orphans T (its owner at
+  // index 0 would be dropped). The pair rule outranks the floor — we drop LESS,
+  // keeping everything (no-op) rather than cross a pair.
+  const messages = [toolUse("read"), user("keep me"), toolResult("read", "r")];
+  const out = dropBefore(messages, 2, 1);
+  assert.equal(out, messages, "anchor floor would orphan the tool result → no-op");
+});
+
+test("isBoundarySafe: interleaved custom between call and result is detected unsafe", () => {
+  const messages = [user("a"), toolUse("read"), custom("bash"), toolResult("read", "r")];
+  // keepFrom=3 drops the call at index 1, keeps the result at index 3 → unsafe.
+  assert.equal(isBoundarySafe(messages, 3), false);
+  // keepFrom=2 ALSO drops the call at index 1 (drop [0,2) = [user, toolUse]) and
+  // keeps the result at index 3 → still orphaned → unsafe.
+  assert.equal(isBoundarySafe(messages, 2), false);
+  // keepFrom=1 keeps the call (index 1) together with its result at index 3 → safe.
+  assert.equal(isBoundarySafe(messages, 1), true);
+});
+
+test("isBoundarySafe: cut before any tool result is safe", () => {
+  const messages = [user("a"), toolUse("read"), toolResult("read", "r"), assistant("done")];
+  assert.equal(isBoundarySafe(messages, 0), true); // out of range → safe
+  assert.equal(isBoundarySafe(messages, messages.length), true); // out of range → safe
+});

package/src/boundary.ts

@@ -5,10 +5,19 @@
  *  1. ANCHOR FLOOR: never drop the most recent N user messages.
  *  2. TOOL-PAIR: never split an assistant(toolCall) from its following
  *     tool-result message — an orphaned `tool` role with no preceding
- *     assistant tool call causes a 400 on the OpenAI-compat path.
+ *     assistant tool call causes a 400 on the OpenAI-compat path. The pair
+ *     invariant outranks the anchor floor: on conflict we drop LESS (lower the
+ *     drop end), never cross a pair.
  *
  * The engine reasons over EngineMessage; the pi adapter maps role "tool" +
- * toolName to the tool-result shape.
+ * toolName to the tool-result shape. EngineMessage carries no tool-call id, so
+ * ownership is positional: a tool result's owner is its nearest preceding
+ * assistant tool-call (the last assistant message with a `toolName` before it).
+ * A preserved tool result is orphaned by a cut when its owner is dropped; the
+ * guard rejects any cut that drops an owner while preserving its result, for
+ * ARBITRARY interleavings (custom/non-tool messages between call and result,
+ * consecutive results sharing one call, a cut landing directly on a call whose
+ * results follow).
  */
 
 import type { EngineMessage } from "./types.js";
@@ -23,16 +32,53 @@ function hasToolUse(m: EngineMessage): boolean {
   return Boolean(m.toolName) && m.role !== "tool";
 }
 
+/**
+ * Is the drop boundary at `dropEnd` pair-safe? The preserved run is
+ * [dropEnd, messages.length). The cut is pair-safe iff NO preserved tool result
+ * is orphaned: for every tool result at index >= dropEnd, its nearest preceding
+ * assistant tool-call must EXIST and be PRESERVED (index >= dropEnd). A tool
+ * result with no preceding assistant tool-call is already orphaned in the
+ * input — we treat that as unsafe too, so the guard never endorses shipping an
+ * orphaned result to the provider.
+ *
+ * O(messages.length) single forward pass; early-exits on the first orphan. The
+ * owner of each result is the most recent `hasToolUse` message seen so far
+ * (tracked across the whole stream, including dropped messages, because a
+ * dropped assistant tool-call is exactly the owner we must reject).
+ */
+export function isPairSafe(messages: EngineMessage[], dropEnd: number): boolean {
+  if (dropEnd <= 0 || dropEnd >= messages.length) return true;
+  let lastToolCall = -1;
+  for (let i = 0; i < messages.length; i++) {
+    if (hasToolUse(messages[i])) lastToolCall = i;
+    if (i >= dropEnd && isToolResult(messages[i])) {
+      if (lastToolCall === -1) return false; // no preceding call → orphaned
+      if (lastToolCall < dropEnd) return false; // owner dropped → orphaned
+    }
+  }
+  return true;
+}
+
 /**
  * Compute the safe drop range [dropStart, dropEnd) within `messages`.
- * `keepFrom` is the caller's desired first-preserved index. We then:
- *  1. Walk it back (lower dropEnd = keep more) so the first preserved message
- *     is never an orphaned tool result (tool-pair invariant).
- *  2. Raise it (lower dropEnd) to the anchor floor so the last N user messages
- *     are never dropped, when enough user messages exist.
  *
- * dropEnd is the first index KEPT. Returns [dropStart, dropEnd]; empty range
- * if nothing should be dropped.
+ * Contract:
+ *  - `keepFrom` is the caller's desired first-preserved index (drop [0, keepFrom)).
+ *  - `dropEnd` is the first index KEPT; we may LOWER it (keep more) to satisfy the
+ *    guards, never raise it above keepFrom.
+ *  - The anchor floor (PREVENT-PI-001) caps dropEnd at the index of the
+ *    Nth-from-last user message so the last N user messages are never dropped.
+ *  - The tool-pair invariant (PREVENT-PI-002) rejects any dropEnd that orphans a
+ *    preserved tool result; on conflict with the anchor floor the pair rule wins
+ *    (we drop less, never cross a pair).
+ *  - We return the LARGEST pair-safe dropEnd <= min(keepFrom, anchorStart) so the
+ *    caller drops as much as is safe. When no pair-safe positive cut exists at
+ *    or below keepFrom, we return [0, 0] (no-op) — the pair rule outranks
+ *    dropping. dropStart is always 0 today (we drop a prefix); reserved for
+ *    future two-sided trimming.
+ *
+ * Returns [0, 0] (empty range, drop nothing) when keepFrom is out of range or no
+ * pair-safe positive cut exists.
  */
 export function computeDropRange(
   messages: EngineMessage[],
@@ -44,48 +90,38 @@ export function computeDropRange(
   const userIndexes: number[] = [];
   messages.forEach((m, i) => { if (m.role === "user") userIndexes.push(i); });
   const anchorActive = anchorUserMessages > 0 && userIndexes.length >= anchorUserMessages;
-  const anchorStart = anchorActive ? userIndexes[userIndexes.length - anchorUserMessages] : 0;
-  const floor = anchorActive ? anchorStart : 0;
+  const anchorStart = anchorActive ? userIndexes[userIndexes.length - anchorUserMessages] : keepFrom;
+  // Upper bound on dropEnd: never keep less than the caller asked (dropEnd <= keepFrom)
+  // and never drop a must-keep user message (dropEnd <= anchorStart).
+  const upperBound = Math.min(keepFrom, anchorActive ? anchorStart : keepFrom);
 
-  // Walk back for the tool-pair invariant (keep more when needed).
-  let k = keepFrom;
-  while (k > floor) {
-    const firstPreserved = messages[k];
-    if (!firstPreserved || !isToolResult(firstPreserved)) break;
-    const preceding = messages[k - 1];
-    if (preceding && hasToolUse(preceding)) {
-      k -= 1; // pair intact across boundary — include the assistant turn
-      break;
-    }
-    k -= 1;
+  // Walk down from the upper bound to find the largest pair-safe cut. dropEnd=0
+  // (drop nothing) is always pair-safe; the loop finds the largest positive cut,
+  // and falls back to [0, 0] when none exists — the pair rule outranks dropping.
+  for (let dropEnd = upperBound; dropEnd > 0; dropEnd--) {
+    if (isPairSafe(messages, dropEnd)) return [0, dropEnd];
   }
-  if (k < floor) k = floor;
-
-  // Anchor floor: never drop a must-keep user message. Raise dropEnd so we keep
-  // from anchorStart onward when the walk didn't already.
-  if (anchorActive && k > anchorStart) k = anchorStart;
-
-  if (k <= 0) return [0, 0];
-  return [0, k];
+  return [0, 0];
 }
 
 /**
  * Validate that the intended split at `keepFrom` (drop [0, keepFrom), keep the
- * rest) does not start the preserved run on an orphaned tool result. Checks
- * messages[keepFrom] against messages[keepFrom-1] directly — independent of the
- * walk-back that computeDropRange may apply.
+ * rest) does not orphan any preserved tool result. Checks the FULL preserved
+ * run, not just the first message, so it holds for arbitrary interleavings
+ * (custom messages between call and result, consecutive shared-call results, a
+ * cut landing on a call whose results follow). Used on the every-LLM-call
+ * live-trim hot path (extensions/mega-trim.ts) and by dropCompactedRange
+ * (src/adapt.ts).
  */
 export function isBoundarySafe(messages: EngineMessage[], keepFrom: number): boolean {
-  if (keepFrom <= 0 || keepFrom >= messages.length) return true;
-  const firstPreserved = messages[keepFrom];
-  if (!isToolResult(firstPreserved)) return true;
-  const preceding = messages[keepFrom - 1];
-  return Boolean(preceding && hasToolUse(preceding));
+  return isPairSafe(messages, keepFrom);
 }
 
 /**
  * Drop everything before the safe keep-index, honoring both guards, returning
- * the filtered message list.
+ * the filtered message list. Returns the original array reference (unchanged)
+ * when the safe range is empty so callers can short-circuit on reference
+ * equality.
  */
 export function dropBefore(messages: EngineMessage[], keepFrom: number, anchorUserMessages: number): EngineMessage[] {
   const [dropStart, dropEnd] = computeDropRange(messages, keepFrom, anchorUserMessages);

package/src/canary.ts

@@ -8,6 +8,16 @@
  * The controller owns a MUTABLE working copy of the dedup config; callers read
  * `controller.config` after each step. Tiers disabled via MARK_ONLY degrade
  * gracefully rather than fully off.
+ *
+ * Persistence design: disabled state is IN-MEMORY ONLY by design. The dedup
+ * config loads from MEGACOMPACT_* env vars (config/dedup.ts) with no durable
+ * save mechanism. `setEnabled` DOES mutate `this.config.L1_ENABLED` etc. so
+ * callers reading `controller.config` see the disabled state for the current
+ * session. On restart, env defaults re-apply and the canary sequences again
+ * from L0 — this is intentional: a tier disabled due to a cold cache or
+ * transient load gets a fresh evaluation each run rather than being locked
+ * out forever. To permanently disable a tier, set the corresponding
+ * MEGACOMPACT_*_ENABLED env var to false.
  */
 
 import type { DedupConfigShape, DedupTier } from "./config/dedup.js";

package/src/config/dedup.ts

@@ -28,6 +28,14 @@ function envNum(name: string, def: number): number {
   return Number.isFinite(n) ? n : def;
 }
 
+/** S42B: parse a comma-separated numeric env var (e.g. "1.0,0.9,0.8"). */
+function envNumArray(name: string, def: number[]): number[] {
+  const v = process.env[name];
+  if (v === undefined) return def;
+  const parts = v.split(",").map((x) => Number(x.trim()));
+  return parts.length > 0 && parts.every((n) => Number.isFinite(n)) ? parts : def;
+}
+
 export interface DedupConfigShape {
   // Tier enable flags.
   L0_ENABLED: boolean;
@@ -54,6 +62,17 @@ export interface DedupConfigShape {
   RAPTOR_BUDGET_MS: number;
   RAPTOR_CLUSTERS_PER_LEVEL: number;
   RAPTOR_CONSISTENCY: number;
+  // S42B: multi-level retrieval (score all tree levels + leaf expansion).
+  RAPTOR_MULTILEVEL_ENABLED: boolean;
+  RAPTOR_LEVEL_WEIGHTS: number[]; // per-level weights, index 0 = leaves (uncalibrated)
+  RAPTOR_LEAF_EXPANSION: boolean;
+  RAPTOR_MAX_LEAF_EXPANSION: number; // uncalibrated
+  RAPTOR_FRESHNESS_HOURS: number; // S42D: skip rebuild when tree is fresh (uncalibrated)
+  /** S25 Phase-2: inject top-level RAPTOR summary nodes (root + level-1
+   *  clusters) into the recall block as a hierarchical overview header.
+   *  Default true — the high-level map helps the model see the session's
+   *  topical structure before detailed checkpoint hits. */
+  RAPTOR_INJECT_SUMMARIES: boolean;
   // Monitoring / alerting.
   FP_RATE_L0: number; // FP alert threshold for exact tier
   FP_RATE_L1L2: number; // FP alert threshold for fuzzy tiers
@@ -87,6 +106,12 @@ export function loadDedupConfig(): DedupConfigShape {
     RAPTOR_BUDGET_MS: envNum("MEGACOMPACT_RAPTOR_BUDGET_MS", 5000),
     RAPTOR_CLUSTERS_PER_LEVEL: envNum("MEGACOMPACT_RAPTOR_CLUSTERS", 5),
     RAPTOR_CONSISTENCY: envNum("MEGACOMPACT_RAPTOR_CONSISTENCY", 0.6),
+    RAPTOR_MULTILEVEL_ENABLED: envBool("MEGACOMPACT_RAPTOR_MULTILEVEL", true),
+    RAPTOR_LEVEL_WEIGHTS: envNumArray("MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS", [1.0, 0.9, 0.8, 0.7, 0.5]),
+    RAPTOR_LEAF_EXPANSION: envBool("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", true),
+    RAPTOR_MAX_LEAF_EXPANSION: envNum("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", 10),
+    RAPTOR_FRESHNESS_HOURS: envNum("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", 4),
+    RAPTOR_INJECT_SUMMARIES: envBool("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", true),
     FP_RATE_L0: envNum("MEGACOMPACT_FP_RATE_L0", 0.01),
     FP_RATE_L1L2: envNum("MEGACOMPACT_FP_RATE_L1L2", 0.05),
     ALERT_WINDOW_MS: envNum("MEGACOMPACT_ALERT_WINDOW_MS", 600_000),

package/src/config.ts

@@ -37,7 +37,9 @@ export function preserveRecentForPressure(
 ): number {
   const p = pressure < 0 ? 0 : pressure > 1 ? 1 : pressure;
   const v = Math.round(preserveRecent - (preserveRecent - preserveRecentMin) * p);
-  return Math.max(preserveRecentMin, Math.min(preserveRecent, v));
+  // Floor of 1: even with preserveRecentMin=0 at full pressure, never compact
+  // ALL messages — the boundary guard (computeDropRange) needs ≥1 to anchor on.
+  return Math.max(1, preserveRecentMin, Math.min(preserveRecent, v));
 }
 
 /**