pi-mega-compact 0.4.0

package/src/boundary.ts

@@ -0,0 +1,94 @@
+/**
+ * boundary.ts — drop-boundary safety guards for the `context` hook.
+ *
+ * Two invariants the drop range must never violate (PREVENT-PI-001 / 002):
+ *  1. ANCHOR FLOOR: never drop the most recent N user messages.
+ *  2. TOOL-PAIR: never split an assistant(toolCall) from its following
+ *     tool-result message — an orphaned `tool` role with no preceding
+ *     assistant tool call causes a 400 on the OpenAI-compat path.
+ *
+ * The engine reasons over EngineMessage; the pi adapter maps role "tool" +
+ * toolName to the tool-result shape.
+ */
+
+import type { EngineMessage } from "./types.js";
+
+/** Is this message a tool result (pi `tool` role with a tool name)? */
+function isToolResult(m: EngineMessage): boolean {
+  return m.role === "tool" && Boolean(m.toolName);
+}
+
+/** Does this assistant/tool message contain a tool call (toolName set)? */
+function hasToolUse(m: EngineMessage): boolean {
+  return Boolean(m.toolName) && m.role !== "tool";
+}
+
+/**
+ * Compute the safe drop range [dropStart, dropEnd) within `messages`.
+ * `keepFrom` is the caller's desired first-preserved index. We then:
+ *  1. Walk it back (lower dropEnd = keep more) so the first preserved message
+ *     is never an orphaned tool result (tool-pair invariant).
+ *  2. Raise it (lower dropEnd) to the anchor floor so the last N user messages
+ *     are never dropped, when enough user messages exist.
+ *
+ * dropEnd is the first index KEPT. Returns [dropStart, dropEnd]; empty range
+ * if nothing should be dropped.
+ */
+export function computeDropRange(
+  messages: EngineMessage[],
+  keepFrom: number,
+  anchorUserMessages: number,
+): [number, number] {
+  if (keepFrom <= 0 || keepFrom >= messages.length) return [0, 0];
+
+  const userIndexes: number[] = [];
+  messages.forEach((m, i) => { if (m.role === "user") userIndexes.push(i); });
+  const anchorActive = anchorUserMessages > 0 && userIndexes.length >= anchorUserMessages;
+  const anchorStart = anchorActive ? userIndexes[userIndexes.length - anchorUserMessages] : 0;
+  const floor = anchorActive ? anchorStart : 0;
+
+  // Walk back for the tool-pair invariant (keep more when needed).
+  let k = keepFrom;
+  while (k > floor) {
+    const firstPreserved = messages[k];
+    if (!firstPreserved || !isToolResult(firstPreserved)) break;
+    const preceding = messages[k - 1];
+    if (preceding && hasToolUse(preceding)) {
+      k -= 1; // pair intact across boundary — include the assistant turn
+      break;
+    }
+    k -= 1;
+  }
+  if (k < floor) k = floor;
+
+  // Anchor floor: never drop a must-keep user message. Raise dropEnd so we keep
+  // from anchorStart onward when the walk didn't already.
+  if (anchorActive && k > anchorStart) k = anchorStart;
+
+  if (k <= 0) return [0, 0];
+  return [0, k];
+}
+
+/**
+ * Validate that the intended split at `keepFrom` (drop [0, keepFrom), keep the
+ * rest) does not start the preserved run on an orphaned tool result. Checks
+ * messages[keepFrom] against messages[keepFrom-1] directly — independent of the
+ * walk-back that computeDropRange may apply.
+ */
+export function isBoundarySafe(messages: EngineMessage[], keepFrom: number): boolean {
+  if (keepFrom <= 0 || keepFrom >= messages.length) return true;
+  const firstPreserved = messages[keepFrom];
+  if (!isToolResult(firstPreserved)) return true;
+  const preceding = messages[keepFrom - 1];
+  return Boolean(preceding && hasToolUse(preceding));
+}
+
+/**
+ * Drop everything before the safe keep-index, honoring both guards, returning
+ * the filtered message list.
+ */
+export function dropBefore(messages: EngineMessage[], keepFrom: number, anchorUserMessages: number): EngineMessage[] {
+  const [dropStart, dropEnd] = computeDropRange(messages, keepFrom, anchorUserMessages);
+  if (dropStart === dropEnd) return messages;
+  return messages.slice(dropEnd);
+}

package/src/canary.ts

@@ -0,0 +1,126 @@
+/**
+ * canary.ts — safe sequential tier rollout (Sprint 14, Phase 7).
+ *
+ * Enables tiers one at a time (L0 → L1 → L2 → RAPTOR), watching each tier's p95
+ * latency (from monitoring metrics) and AUTO-DISABLING a tier whose p95 breaches
+ * the budget. No human-in-the-loop: degradation is automatic and local (QA #19).
+ *
+ * The controller owns a MUTABLE working copy of the dedup config; callers read
+ * `controller.config` after each step. Tiers disabled via MARK_ONLY degrade
+ * gracefully rather than fully off.
+ */
+
+import type { DedupConfigShape, DedupTier } from "./config/dedup.js";
+import { loadDedupConfig } from "./config/dedup.js";
+import type { DedupMetrics } from "./monitoring.js";
+import { p95 } from "./monitoring.js";
+
+/** Canonical enablement order. */
+export const CANARY_ORDER: DedupTier[] = ["L0", "L1", "L2", "RAPTOR"];
+
+export interface CanaryState {
+  /** Tiers currently enabled (and not auto-disabled). */
+  enabled: Set<DedupTier>;
+  /** Tiers auto-disabled by a p95 breach. */
+  disabled: Set<DedupTier>;
+  /** Steps taken so far. */
+  step: number;
+}
+
+export class CanaryController {
+  readonly config: DedupConfigShape;
+  private readonly state: CanaryState;
+
+  constructor(base: DedupConfigShape = loadDedupConfig()) {
+    // Start from the base config but begin with L0 only (sequential rollout).
+    this.config = { ...base };
+    this.config.L1_ENABLED = false;
+    this.config.L2_ENABLED = false;
+    this.config.RAPTOR_ENABLED = false;
+    this.state = {
+      enabled: new Set<DedupTier>(["L0"]),
+      disabled: new Set<DedupTier>(),
+      step: 1,
+    };
+  }
+
+  /** Enable the next tier in CANARY_ORDER (no-op if all enabled). */
+  stepForward(): DedupTier | null {
+    for (const tier of CANARY_ORDER) {
+      if (!this.state.enabled.has(tier) && !this.state.disabled.has(tier)) {
+        this.setEnabled(tier, true);
+        this.state.step++;
+        return tier;
+      }
+    }
+    return null;
+  }
+
+  private setEnabled(tier: DedupTier, on: boolean): void {
+    switch (tier) {
+      case "L0": this.config.L0_ENABLED = on; break;
+      case "L1": this.config.L1_ENABLED = on; break;
+      case "L2": this.config.L2_ENABLED = on; break;
+      case "RAPTOR": this.config.RAPTOR_ENABLED = on; break;
+    }
+    if (on) { this.state.enabled.add(tier); this.state.disabled.delete(tier); }
+    else { this.state.enabled.delete(tier); }
+  }
+
+  private disableReason(tier: DedupTier): void {
+    this.setEnabled(tier, false);
+    this.state.disabled.add(tier);
+  }
+
+  /**
+   * Evaluate p95 latency for every enabled tier against the budget. Any tier
+   * whose p95 exceeds `config.P95_BUDGET_MS` is auto-disabled. Returns the tiers
+   * it disabled this pass (so the caller can log/alert).
+   */
+  evaluate(metrics: DedupMetrics): DedupTier[] {
+    const disabledNow: DedupTier[] = [];
+    for (const tier of CANARY_ORDER) {
+      if (!this.state.enabled.has(tier)) continue;
+      const lat = p95(metrics.latency[tier] ?? []);
+      if (lat > this.config.P95_BUDGET_MS) {
+        this.disableReason(tier);
+        disabledNow.push(tier);
+      }
+    }
+    return disabledNow;
+  }
+
+  getState(): CanaryState {
+    return {
+      enabled: new Set(this.state.enabled),
+      disabled: new Set(this.state.disabled),
+      step: this.state.step,
+    };
+  }
+}
+
+/**
+ * Run the full canary rollout against a metrics feed. `feed` is sampled at each
+ * step (after enabling a tier) to drive auto-disable. Deterministic + offline.
+ */
+export function runCanary(
+  metricsFeed: (step: number, config: DedupConfigShape) => DedupMetrics,
+  base: DedupConfigShape = loadDedupConfig(),
+): { controller: CanaryController; disabled: DedupTier[] } {
+  const controller = new CanaryController(base);
+  const allDisabled: DedupTier[] = [];
+  let step = 0;
+  // Up to one step per tier + a final eval.
+  while (true) {
+    const tier = controller.stepForward();
+    if (tier === null) break;
+    const metrics = metricsFeed(controller.getState().step, controller.config);
+    const disabled = controller.evaluate(metrics);
+    allDisabled.push(...disabled);
+    if (++step > CANARY_ORDER.length + 1) break;
+  }
+  // Final eval pass on the last enabled set.
+  const finalMetrics = metricsFeed(controller.getState().step, controller.config);
+  allDisabled.push(...controller.evaluate(finalMetrics));
+  return { controller, disabled: [...new Set(allDisabled)] };
+}

package/src/compact.test.ts

@@ -0,0 +1,99 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type { EngineMessage } from "./types.js";
+import {
+  formatCompactSummary,
+  summarizeMessages,
+  mergeCompactSummaries,
+  shouldCompact,
+  autoCompactCheck,
+  collectKeyFiles,
+  inferPendingWork,
+  extractFileCandidates,
+} from "./compact.js";
+import { estimateSessionTokens } from "./tokens.js";
+
+function user(text: string): EngineMessage { return { role: "user", text }; }
+function assistant(text: string): EngineMessage { return { role: "assistant", text }; }
+function toolUse(name: string, input: string): EngineMessage { return { role: "assistant", text: "", toolName: name, input }; }
+function toolResult(name: string, output: string): EngineMessage { return { role: "tool", text: "", toolName: name, output }; }
+
+test("formatCompactSummary strips analysis and formats summary block (claw-code parity)", () => {
+  const summary = "<analysis>scratch</analysis>\n<summary>Kept work</summary>";
+  assert.equal(formatCompactSummary(summary), "Summary:\nKept work");
+});
+
+test("leaves small sessions unchanged (shouldCompact false)", () => {
+  const messages = [user("hello")];
+  assert.equal(shouldCompact(messages, 1, 4), false);
+});
+
+test("compacts older messages into a summary with Scope + timeline", () => {
+  const messages = [
+    user("one ".repeat(200)),
+    assistant("two ".repeat(200)),
+    toolResult("bash", "ok ".repeat(200)),
+    assistant("recent"),
+  ];
+  assert.equal(shouldCompact(messages, 1, 2), true);
+  const summary = summarizeMessages(messages.slice(0, 2));
+  const formatted = formatCompactSummary(summary);
+  assert.ok(formatted.includes("Scope:"));
+  assert.ok(formatted.includes("Key timeline:"));
+});
+
+test("merge keeps previous compacted context when compacting again", () => {
+  const first = summarizeMessages([
+    user("Investigate src/compact.ts"),
+    assistant("I will inspect the compact flow."),
+  ]);
+  const second = summarizeMessages([
+    user("Also update src/boundary.ts"),
+    assistant("Next: preserve prior summary context."),
+  ]);
+  const merged = mergeCompactSummaries(first, second);
+  assert.ok(merged.includes("Previously compacted context:"));
+  assert.ok(merged.includes("Newly compacted context:"));
+  assert.ok(merged.includes("src/boundary.ts"));
+});
+
+test("infers pending work from recent messages", () => {
+  const pending = inferPendingWork([
+    user("done"),
+    assistant("Next: update tests and follow up on remaining CLI polish."),
+  ]);
+  assert.equal(pending.length, 1);
+  assert.ok(pending[0].includes("Next: update tests"));
+});
+
+test("extracts key files from message content", () => {
+  const files = collectKeyFiles([
+    user("Update src/compact.ts and extensions/mega-compact.ts next."),
+  ]);
+  assert.ok(files.includes("src/compact.ts"));
+  assert.ok(files.includes("extensions/mega-compact.ts"));
+});
+
+test("extractFileCandidates ignores plain words and non-interesting extensions", () => {
+  const files = extractFileCandidates("look at foo/bar.png and src/x.ts and justaword");
+  assert.deepEqual(files, ["src/x.ts"]);
+});
+
+test("summarizeMessages lists tool names sorted + deduped", () => {
+  const summary = summarizeMessages([toolUse("search", "{}"), toolUse("bash", "{}"), toolResult("search", "ok")]);
+  assert.ok(summary.includes("Tools mentioned: bash, search."));
+});
+
+test("autoCompactCheck reports utilization and threshold gate", () => {
+  const under = autoCompactCheck(10000, 50000);
+  assert.equal(under.shouldCompact, false);
+  assert.equal(under.utilizationPct, 20);
+  const over = autoCompactCheck(60000, 50000);
+  assert.equal(over.shouldCompact, true);
+});
+
+test("token estimator counts text + tool payloads", () => {
+  const total = estimateSessionTokens([user("abcd"), toolResult("bash", "abcd")]);
+  // "abcd"/4+1 = 2 for user; tool: name "bash"/4+1=2 plus output "abcd"/4+1=2 => 4
+  assert.equal(total, 2 + 4);
+});

package/src/compact.ts

@@ -0,0 +1,262 @@
+/**
+ * compact.ts — the COLLAPSE/summarize engine (Layer 2) + the compaction gate.
+ *
+ * Ported (conceptually) from claw-code rusty-claude-cli compact.rs
+ * (summarize_messages / merge_compact_summaries / format_compact_summary) and
+ * from memory-mcp session_context.py auto_compact_check / should_compact.
+ * Pure, pi-agnostic, deterministic, no LLM required.
+ */
+
+import type { EngineMessage } from "./types.js";
+import { estimateSessionTokens } from "./tokens.js";
+
+const INTERESTING_EXT = new Set(["rs", "ts", "tsx", "js", "json", "md"]);
+const PENDING_WORDS = ["todo", "next", "pending", "follow up", "remaining"];
+
+const COMPACT_PREAMBLE =
+  "This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation.\n\n";
+const RECENT_NOTE = "Recent messages are preserved verbatim.";
+const DIRECT_RESUME = "Continue the conversation from where it left off without asking the user any further questions. Resume directly — do not acknowledge the summary, do not recap what was happening, and do not preface with continuation text.";
+
+function truncate(s: string, max: number): string {
+  return s.length <= max ? s : `${s.slice(0, max)}…`;
+}
+
+function firstText(m: EngineMessage): string | undefined {
+  const t = m.text.trim();
+  return t.length > 0 ? t : undefined;
+}
+
+/** Heuristic: does this text look like chatty filler we can collapse? */
+export function isChatty(text: string): boolean {
+  const low = text.toLowerCase();
+  if (low.includes("hello") || low.includes("thanks") || low.includes("great") || low.includes("ok")) {
+    return true;
+  }
+  return text.length < 40 && !/(\/|\.|\{|import |def |function )/.test(text);
+}
+
+/** Extract plausible file paths (contain '/' + an interesting extension). */
+export function extractFileCandidates(content: string): string[] {
+  const out: string[] = [];
+  for (const raw of content.split(/\s+/)) {
+    // Trim surrounding punctuation only — do NOT strip internal dots, or we
+    // would erase the extension separator (src/server.ts -> src/server/ts).
+    const token = raw.replace(/^[^A-Za-z0-9/]+|[^A-Za-z0-9/]+$/g, "");
+    if (!token.includes("/") || !token.includes(".")) continue;
+    const ext = token.split(".").pop()?.toLowerCase() ?? "";
+    if (INTERESTING_EXT.has(ext)) out.push(token);
+  }
+  return out;
+}
+
+/** Collect unique key files referenced across a set of messages. */
+export function collectKeyFiles(messages: EngineMessage[]): string[] {
+  const files = new Set<string>();
+  for (const m of messages) {
+    for (const c of [m.text, m.input, m.output]) {
+      if (!c) continue;
+      for (const f of extractFileCandidates(c)) files.add(f);
+    }
+  }
+  return [...files].slice(0, 8);
+}
+
+/** Infer pending work from recent messages via keyword scan. */
+export function inferPendingWork(messages: EngineMessage[]): string[] {
+  const out: string[] = [];
+  for (const m of [...messages].reverse()) {
+    const t = firstText(m);
+    if (!t) continue;
+    const low = t.toLowerCase();
+    if (PENDING_WORDS.some((w) => low.includes(w))) {
+      out.push(truncate(t, 160));
+      if (out.length >= 3) break;
+    }
+  }
+  return out.reverse();
+}
+
+/** Latest user request (for "current work" line). */
+export function inferCurrentWork(messages: EngineMessage[]): string | undefined {
+  for (const m of [...messages].reverse()) {
+    const t = firstText(m);
+    if (t && m.role === "user") return truncate(t, 200);
+  }
+  return undefined;
+}
+
+/** Last N user requests, in original order. */
+export function collectRecentUserRequests(messages: EngineMessage[], limit: number): string[] {
+  const reqs = messages
+    .filter((m) => m.role === "user")
+    .map((m) => firstText(m))
+    .filter((t): t is string => Boolean(t))
+    .map((t) => truncate(t, 160));
+  return reqs.slice(-limit);
+}
+
+/** Summarize a block to a one-line description. */
+function summarizeBlock(m: EngineMessage): string {
+  if (m.role === "tool") return `tool_result ${m.toolName ?? "?"}: ${truncate(m.output ?? m.text, 160)}`;
+  if (m.toolName) return `tool_use ${m.toolName}(${truncate(m.input ?? "", 160)})`;
+  return truncate(m.text, 160);
+}
+
+function stripTag(block: string, tag: string): string {
+  const start = `<${tag}>`;
+  const end = `</${tag}>`;
+  const s = block.indexOf(start);
+  const e = block.indexOf(end);
+  if (s === -1 || e === -1) return block;
+  return block.slice(0, s) + block.slice(e + end.length);
+}
+
+function extractTag(block: string, tag: string): string | undefined {
+  const s = block.indexOf(`<${tag}>`);
+  const e = block.indexOf(`</${tag}>`);
+  if (s === -1 || e === -1) return undefined;
+  return block.slice(s + `<${tag}>`.length, e);
+}
+
+/** Normalize a raw summary into user-facing "Summary: ..." text. */
+export function formatCompactSummary(summary: string): string {
+  const withoutAnalysis = stripTag(summary, "analysis");
+  let formatted = withoutAnalysis;
+  const content = extractTag(withoutAnalysis, "summary");
+  if (content !== undefined) {
+    formatted = withoutAnalysis.replace(
+      `<summary>${content}</summary>`,
+      `Summary:\n${content.trim()}`,
+    );
+  }
+  return formatted.replace(/\n{3,}/g, "\n\n").trim();
+}
+
+/**
+ * Build a <summary> block from a slice of messages (the COLLAPSE output).
+ * Mirrors claw-code summarize_messages.
+ */
+export function summarizeMessages(messages: EngineMessage[]): string {
+  const users = messages.filter((m) => m.role === "user").length;
+  const assistants = messages.filter((m) => m.role === "assistant").length;
+  const tools = messages.filter((m) => m.role === "tool").length;
+
+  const toolNames = [...new Set(messages.flatMap((m) => (m.toolName ? [m.toolName] : [])))].sort();
+
+  const lines: string[] = [
+    "<summary>",
+    "Conversation summary:",
+    `- Scope: ${messages.length} earlier messages compacted (user=${users}, assistant=${assistants}, tool=${tools}).`,
+  ];
+  if (toolNames.length) lines.push(`- Tools mentioned: ${toolNames.join(", ")}.`);
+
+  const recent = collectRecentUserRequests(messages, 3);
+  if (recent.length) {
+    lines.push("- Recent user requests:");
+    recent.forEach((r) => lines.push(`  - ${r}`));
+  }
+
+  const pending = inferPendingWork(messages);
+  if (pending.length) {
+    lines.push("- Pending work:");
+    pending.forEach((p) => lines.push(`  - ${p}`));
+  }
+
+  const files = collectKeyFiles(messages);
+  if (files.length) lines.push(`- Key files referenced: ${files.join(", ")}.`);
+
+  const current = inferCurrentWork(messages);
+  if (current) lines.push(`- Current work: ${current}`);
+
+  lines.push("- Key timeline:");
+  for (const m of messages) {
+    const role = m.role;
+    lines.push(`  - ${role}: ${summarizeBlock(m)}`);
+  }
+  lines.push("</summary>");
+  return lines.join("\n");
+}
+
+/** Extract the prior "highlights" + "timeline" sections from an existing summary. */
+function extractSummaryHighlights(summary: string): string[] {
+  const lines = formatCompactSummary(summary).split("\n");
+  const out: string[] = [];
+  let inTimeline = false;
+  for (const line of lines) {
+    const t = line.trimEnd();
+    if (!t || t === "Summary:" || t === "Conversation summary:") continue;
+    if (t === "- Key timeline:") { inTimeline = true; continue; }
+    if (inTimeline) continue;
+    out.push(t);
+  }
+  return out;
+}
+
+function extractSummaryTimeline(summary: string): string[] {
+  const lines = formatCompactSummary(summary).split("\n");
+  const out: string[] = [];
+  let inTimeline = false;
+  for (const line of lines) {
+    const t = line.trimEnd();
+    if (t === "- Key timeline:") { inTimeline = true; continue; }
+    if (!inTimeline) continue;
+    if (!t) break;
+    out.push(t);
+  }
+  return out;
+}
+
+/** Merge an existing compact summary with a new one (accumulate, don't overwrite). */
+export function mergeCompactSummaries(existing: string | undefined, newSummary: string): string {
+  if (!existing) return newSummary;
+  const prevHighlights = extractSummaryHighlights(existing);
+  const newHighlights = extractSummaryHighlights(formatCompactSummary(newSummary));
+  const newTimeline = extractSummaryTimeline(formatCompactSummary(newSummary));
+
+  const lines = ["<summary>", "Conversation summary:"];
+  if (prevHighlights.length) {
+    lines.push("- Previously compacted context:");
+    prevHighlights.forEach((l) => lines.push(`  ${l}`));
+  }
+  if (newHighlights.length) {
+    lines.push("- Newly compacted context:");
+    newHighlights.forEach((l) => lines.push(`  ${l}`));
+  }
+  if (newTimeline.length) {
+    lines.push("- Key timeline:");
+    newTimeline.forEach((l) => lines.push(`  ${l}`));
+  }
+  lines.push("</summary>");
+  return lines.join("\n");
+}
+
+/** True when the compactable portion exceeds the budget. */
+export function shouldCompact(messages: EngineMessage[], maxEstimatedTokens: number, preserveRecent: number): boolean {
+  if (messages.length <= preserveRecent) return false;
+  const compactable = messages.slice(0, messages.length - preserveRecent);
+  return estimateSessionTokens(compactable) >= maxEstimatedTokens;
+}
+
+/** Local reimplementation of memory-mcp auto_compact_check. */
+export function autoCompactCheck(currentTokens: number, threshold = 50000): {
+  shouldCompact: boolean;
+  currentTokens: number;
+  threshold: number;
+  utilizationPct: number;
+} {
+  return {
+    shouldCompact: currentTokens >= threshold,
+    currentTokens,
+    threshold,
+    utilizationPct: Math.round((currentTokens / threshold) * 1000) / 10,
+  };
+}
+
+/** Build the synthetic continuation message (system-prompt prepend form). */
+export function getContinuationMessage(summary: string, suppressFollowUp: boolean, recentPreserved: boolean): string {
+  let base = COMPACT_PREAMBLE + formatCompactSummary(summary);
+  if (recentPreserved) base += `\n\n${RECENT_NOTE}`;
+  if (suppressFollowUp) base += `\n${DIRECT_RESUME}`;
+  return base;
+}