supipowers 1.2.6 → 1.5.0

package/src/context-mode/hooks.ts

@@ -7,10 +7,18 @@ import { EventStore } from "./event-store.js";
 import { extractEvents, extractPromptEvents } from "./event-extractor.js";
 import { buildResumeSnapshot } from "./snapshot-builder.js";
 import { routeToolCall } from "./routing.js";
+import { KnowledgeStore } from "./knowledge/store.js";
+import { registerContextModeTools } from "./tools.js";
+import { createHash } from "node:crypto";
 import { readFileSync, mkdirSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 
+type SessionContextLike = {
+  cwd?: string;
+  sessionManager?: { getSessionFile?: () => string } | null;
+};
+
 // Cached detection result
 let cachedStatus: ContextModeStatus | null = null;
 
@@ -24,29 +32,135 @@ function loadRoutingSkill(): string | null {
   }
 }
 
-/** Register context-mode hooks on the platform */
+function resolveSessionCwd(ctx?: SessionContextLike): string {
+  return typeof ctx?.cwd === "string" && ctx.cwd.length > 0 ? ctx.cwd : process.cwd();
+}
+
+function deriveSessionId(ctx?: SessionContextLike): string {
+  try {
+    const sessionFile = ctx?.sessionManager?.getSessionFile?.();
+    if (typeof sessionFile === "string" && sessionFile.length > 0) {
+      return createHash("sha256").update(sessionFile).digest("hex").slice(0, 16);
+    }
+  } catch {
+    // Best effort only — session lifecycle must never fail on ID derivation.
+  }
+  return `session-${Date.now()}`;
+}
+
+function getSessionDbPath(platform: Platform, cwd: string): string {
+  return join(platform.paths.project(cwd, "sessions"), "events.db");
+}
+
+/** Register supi-context-mode hooks on the platform */
 export function registerContextModeHooks(platform: Platform, config: SupipowersConfig): void {
   if (!config.contextMode.enabled) return;
 
-  // Phase 2: Event store initialization
   let eventStore: EventStore | null = null;
-  let sessionId = `session-${Date.now()}`;
+  let eventStorePath: string | null = null;
+  let sessionCwd = process.cwd();
+  let sessionId = deriveSessionId();
+
+  const ensureEventStore = (cwd: string): EventStore | null => {
+    if (!config.contextMode.eventTracking) return null;
+
+    const dbPath = getSessionDbPath(platform, cwd);
+    if (eventStore && eventStorePath === dbPath) return eventStore;
+
+    if (eventStore) {
+      try {
+        eventStore.close();
+      } catch {
+        // Best effort — we are about to reopen against the active session path.
+      }
+    }
 
-  if (config.contextMode.eventTracking) {
     try {
-      const dbDir = platform.paths.project(process.cwd(), "sessions");
-      mkdirSync(dbDir, { recursive: true });
-      eventStore = new EventStore(join(dbDir, "events.db"));
+      mkdirSync(platform.paths.project(cwd, "sessions"), { recursive: true });
+      eventStore = new EventStore(dbPath);
       eventStore.init();
+      eventStore.pruneOldSessions(7);
+      eventStorePath = dbPath;
+      _eventStoreRef = eventStore;
+      return eventStore;
     } catch (e) {
-      (platform as any).logger?.error?.("context-mode: failed to initialize event store", e);
+      eventStore = null;
+      eventStorePath = null;
+      _eventStoreRef = null;
+      (platform as any).logger?.error?.("supi-context-mode: failed to initialize event store", e);
+      return null;
     }
-  }
+  };
+
+  ensureEventStore(sessionCwd);
 
-  // Update module-level refs for compaction hooks
-  _eventStoreRef = eventStore;
   _sessionIdRef = sessionId;
 
+  // Initialize knowledge store for native ctx_* tools (independent of event tracking)
+  let knowledgeStore: KnowledgeStore | null = null;
+  try {
+    const sessionsDir = platform.paths.project(sessionCwd, "sessions");
+    mkdirSync(sessionsDir, { recursive: true });
+    const kdbPath = join(sessionsDir, "knowledge.db");
+    knowledgeStore = new KnowledgeStore(kdbPath);
+    knowledgeStore.init();
+    _knowledgeStoreRef = knowledgeStore;
+  } catch (e) {
+    (platform as any).logger?.error?.("supi-context-mode: failed to initialize knowledge store", e);
+  }
+
+  // Register native context-mode tools
+  if (knowledgeStore) {
+    registerContextModeTools(platform, knowledgeStore);
+  }
+
+  platform.on("session_start", (_event, ctx) => {
+    sessionCwd = resolveSessionCwd(ctx as SessionContextLike | undefined);
+    sessionId = deriveSessionId(ctx as SessionContextLike | undefined);
+    _sessionIdRef = sessionId;
+
+    const store = ensureEventStore(sessionCwd);
+    if (!store) return;
+
+    try {
+      store.upsertMeta(sessionId, sessionCwd);
+    } catch (e) {
+      (platform as any).logger?.warn?.("supi-context-mode: failed to initialize session metadata", e);
+    }
+  });
+
+  platform.on("session_shutdown", () => {
+    // Close knowledge store
+    if (knowledgeStore) {
+      try {
+        knowledgeStore.close();
+      } catch {
+        // Best effort
+      } finally {
+        knowledgeStore = null;
+        _knowledgeStoreRef = null;
+      }
+    }
+
+    if (!eventStore) {
+      _eventStoreRef = null;
+      _sessionIdRef = "";
+      return;
+    }
+
+    try {
+      eventStore.pruneOldSessions(7);
+      eventStore.close();
+    } catch (e) {
+      (platform as any).logger?.warn?.("supi-context-mode: failed to close event store", e);
+    } finally {
+      eventStore = null;
+      eventStorePath = null;
+      _eventStoreRef = null;
+      _sessionIdRef = "";
+    }
+  });
+
   // Phase 1: Result compression + Phase 2: Event extraction
   platform.on("tool_result", (event) => {
     // Phase 1: compression
@@ -58,7 +172,7 @@ export function registerContextModeHooks(platform: Platform, config: SupipowersC
         const events = extractEvents(event, sessionId);
         if (events.length > 0) eventStore.writeEvents(events);
       } catch (e) {
-        (platform as any).logger?.warn?.("context-mode: event extraction failed", e);
+        (platform as any).logger?.warn?.("supi-context-mode: event extraction failed", e);
       }
     }
 
@@ -88,7 +202,7 @@ export function registerContextModeHooks(platform: Platform, config: SupipowersC
           if (events.length > 0) eventStore.writeEvents(events);
         }
       } catch (e) {
-        (platform as any).logger?.warn?.("context-mode: prompt event extraction failed", e);
+        (platform as any).logger?.warn?.("supi-context-mode: prompt event extraction failed", e);
       }
     }
 
@@ -105,23 +219,80 @@ export function registerContextModeHooks(platform: Platform, config: SupipowersC
   });
 
   // Phase 3: Compaction integration
-  if (config.contextMode.compaction && eventStore) {
-    let pendingSnapshot: string | null = null;
+  const compactionStore = config.contextMode.compaction ? ensureEventStore(sessionCwd) : null;
+  if (compactionStore) {
+
+    // Initialize fallback session metadata for sessions that never emit session_start in tests.
+    try {
+      compactionStore.upsertMeta(sessionId, sessionCwd);
+    } catch {
+      // Non-fatal: metadata is supplementary
+    }
 
     platform.on("session_before_compact", () => {
+      // Re-detect MCP tools: they may have loaded since init
+      const status = cachedStatus ?? detectContextMode(platform.getActiveTools());
+      const searchAvailable = status.tools.ctxSearch;
+
+      // Determine the search tool name for reference-based snapshots
+      let searchTool: string | undefined;
+      if (searchAvailable) {
+        const tools = platform.getActiveTools();
+        searchTool = tools.find((t) => t.includes("ctx_search"));
+      }
+
+      // Read compact count from metadata
+      let compactCount = 0;
       try {
-        pendingSnapshot = buildResumeSnapshot(eventStore!, sessionId);
+        const meta = eventStore!.getMeta(sessionId);
+        compactCount = meta?.compactCount ?? 0;
+      } catch {
+        // Non-fatal
+      }
+
+      try {
+        const snapshot = buildResumeSnapshot(eventStore!, sessionId, {
+          compactCount,
+          searchTool,
+          searchAvailable,
+        });
+
+        // Persist to DB so it survives crashes
+        if (snapshot) {
+          const eventCount = Object.values(eventStore!.getEventCounts(sessionId))
+            .reduce((a, b) => a + b, 0);
+          eventStore!.upsertResume(sessionId, snapshot, eventCount);
+        }
+
+        return undefined; // don't cancel or replace compaction
       } catch (e) {
         (platform as any).logger?.warn?.("context-mode: snapshot build failed", e);
-        pendingSnapshot = null;
+        return undefined;
       }
-      return undefined; // don't cancel or replace compaction
     });
 
     platform.on("session_compact", () => {
-      if (!pendingSnapshot) return undefined;
-      const snapshot = pendingSnapshot;
-      pendingSnapshot = null;
+      // Try resume from DB first, fall back to in-memory
+      let snapshot: string | null = null;
+      try {
+        const resume = eventStore!.getResume(sessionId);
+        if (resume) {
+          snapshot = resume.snapshot;
+          eventStore!.consumeResume(sessionId);
+        }
+      } catch {
+        // Non-fatal: fall through
+      }
+
+      if (!snapshot) return undefined;
+
+      // Track compaction count in session metadata
+      try {
+        eventStore!.incrementCompactCount(sessionId);
+      } catch {
+        // Non-fatal
+      }
+
       return {
         context: snapshot.split("\n"),
         preserveData: {
@@ -145,11 +316,13 @@ export function getSessionId(): string {
 
 // Module-level refs updated by registerContextModeHooks
 let _eventStoreRef: EventStore | null = null;
+let _knowledgeStoreRef: KnowledgeStore | null = null;
 let _sessionIdRef = "";
 
 /** Reset cached state (for testing) */
 export function _resetCache(): void {
   cachedStatus = null;
   _eventStoreRef = null;
+  _knowledgeStoreRef = null;
   _sessionIdRef = "";
-}
+}

package/src/context-mode/knowledge/chunker.ts

@@ -0,0 +1,235 @@
+export interface Chunk {
+  title: string;
+  body: string;
+  contentType: "code" | "prose";
+  source: string;
+}
+
+const HEADING_RE = /^(#{1,6})\s+(.*)/;
+const FENCE_RE = /^(`{3,})/;
+const MAX_CHUNK_SIZE = 4096;
+
+/**
+ * Split markdown into searchable chunks by heading boundaries.
+ * Never splits inside fenced code blocks. Oversized chunks split at paragraph boundaries.
+ */
+export function chunkMarkdown(text: string, source: string): Chunk[] {
+  if (!text) return [];
+
+  const sections = splitByHeadings(text, source);
+  const chunks: Chunk[] = [];
+
+  for (const section of sections) {
+    const body = section.body.trim();
+    if (!body) continue;
+
+    if (body.length <= MAX_CHUNK_SIZE) {
+      chunks.push({
+        title: section.title,
+        body,
+        contentType: classifyContent(body),
+        source,
+      });
+    } else {
+      const parts = splitOversized(body);
+      for (let i = 0; i < parts.length; i++) {
+        const partBody = parts[i].trim();
+        if (!partBody) continue;
+        chunks.push({
+          title: `${section.title} (part ${i + 1})`,
+          body: partBody,
+          contentType: classifyContent(partBody),
+          source,
+        });
+      }
+    }
+  }
+
+  return chunks;
+}
+
+interface RawSection {
+  title: string;
+  body: string;
+}
+
+/** Split text into sections at heading boundaries, respecting fenced code blocks. */
+function splitByHeadings(text: string, source: string): RawSection[] {
+  const lines = text.split("\n");
+  const sections: RawSection[] = [];
+  let currentTitle = source;
+  let currentLines: string[] = [];
+  let inFence = false;
+  let fenceMarker = "";
+
+  for (const line of lines) {
+    if (inFence) {
+      currentLines.push(line);
+      // Check if this line closes the fence
+      const closeMatch = line.match(FENCE_RE);
+      if (closeMatch && line.trimStart().startsWith(fenceMarker) && !line.trimStart().slice(fenceMarker.length).match(/\S/)) {
+        inFence = false;
+        fenceMarker = "";
+      }
+      continue;
+    }
+
+    // Check for opening fence
+    const fenceMatch = line.match(FENCE_RE);
+    if (fenceMatch) {
+      // Opening fence: backticks followed optionally by a language hint
+      inFence = true;
+      fenceMarker = fenceMatch[1];
+      currentLines.push(line);
+      continue;
+    }
+
+    const headingMatch = line.match(HEADING_RE);
+    if (headingMatch) {
+      // Flush previous section
+      sections.push({ title: currentTitle, body: currentLines.join("\n") });
+      currentTitle = headingMatch[2];
+      currentLines = [];
+    } else {
+      currentLines.push(line);
+    }
+  }
+
+  // Flush final section
+  sections.push({ title: currentTitle, body: currentLines.join("\n") });
+
+  return sections;
+}
+
+/** Classify body as "code" or "prose" based on fenced code block character ratio. */
+function classifyContent(body: string): "code" | "prose" {
+  let codeChars = 0;
+  let inFence = false;
+  let fenceMarker = "";
+  let blockLines: string[] = [];
+
+  for (const line of body.split("\n")) {
+    if (inFence) {
+      const closeMatch = line.match(FENCE_RE);
+      if (closeMatch && line.trimStart().startsWith(fenceMarker) && !line.trimStart().slice(fenceMarker.length).match(/\S/)) {
+        inFence = false;
+        codeChars += blockLines.join("\n").length;
+        blockLines = [];
+        fenceMarker = "";
+      } else {
+        blockLines.push(line);
+      }
+      continue;
+    }
+
+    const fenceMatch = line.match(FENCE_RE);
+    if (fenceMatch) {
+      inFence = true;
+      fenceMarker = fenceMatch[1];
+      blockLines = [];
+    }
+  }
+
+  // If still in fence (unclosed), count accumulated lines
+  if (inFence) {
+    codeChars += blockLines.join("\n").length;
+  }
+
+  return codeChars > body.length * 0.5 ? "code" : "prose";
+}
+
+/**
+ * Split oversized body at paragraph boundaries (\n\n), never breaking inside code fences.
+ * Returns parts each ≤ MAX_CHUNK_SIZE (best effort — a single paragraph exceeding the limit is kept whole).
+ */
+function splitOversized(body: string): string[] {
+  // Identify paragraph boundaries that are outside code fences
+  const splitPoints = findSafeSplitPoints(body);
+
+  if (splitPoints.length === 0) {
+    // No safe split points — return as single part
+    return [body];
+  }
+
+  const parts: string[] = [];
+  let start = 0;
+
+  for (const point of splitPoints) {
+    const candidate = body.slice(start, point).trim();
+    if (!candidate) {
+      start = point;
+      continue;
+    }
+
+    // Check if adding this segment to the current accumulation would exceed the limit
+    if (parts.length > 0) {
+      const last = parts[parts.length - 1];
+      if (last.length + candidate.length + 2 <= MAX_CHUNK_SIZE) {
+        parts[parts.length - 1] = last + "\n\n" + candidate;
+        start = point;
+        continue;
+      }
+    }
+
+    // Start a new part, greedily accumulating paragraphs up to the limit
+    if (parts.length === 0 || parts[parts.length - 1].length > 0) {
+      parts.push(candidate);
+    }
+    start = point;
+  }
+
+  // Handle remaining text after last split point
+  const remaining = body.slice(start).trim();
+  if (remaining) {
+    if (parts.length > 0) {
+      const last = parts[parts.length - 1];
+      if (last.length + remaining.length + 2 <= MAX_CHUNK_SIZE) {
+        parts[parts.length - 1] = last + "\n\n" + remaining;
+      } else {
+        parts.push(remaining);
+      }
+    } else {
+      parts.push(remaining);
+    }
+  }
+
+  return parts;
+}
+
+/** Find byte offsets of `\n\n` boundaries that are NOT inside fenced code blocks. */
+function findSafeSplitPoints(body: string): number[] {
+  const points: number[] = [];
+  let inFence = false;
+  let fenceMarker = "";
+  let i = 0;
+
+  const lines = body.split("\n");
+  let offset = 0;
+
+  for (let li = 0; li < lines.length; li++) {
+    const line = lines[li];
+
+    if (inFence) {
+      const closeMatch = line.match(FENCE_RE);
+      if (closeMatch && line.trimStart().startsWith(fenceMarker) && !line.trimStart().slice(fenceMarker.length).match(/\S/)) {
+        inFence = false;
+        fenceMarker = "";
+      }
+    } else {
+      const fenceMatch = line.match(FENCE_RE);
+      if (fenceMatch) {
+        inFence = true;
+        fenceMarker = fenceMatch[1];
+      }
+    }
+
+    // A blank line outside a fence is a paragraph boundary (\n\n in the original text)
+    if (!inFence && line === "") {
+      points.push(offset + 1); // offset after the blank line's \n
+    }
+
+    offset += line.length + 1; // +1 for the \n
+  }
+
+  return points;
+}

package/src/context-mode/knowledge/store.ts

@@ -0,0 +1,187 @@
+import { Database } from "bun:sqlite";
+import fs from "node:fs";
+import type { Chunk } from "./chunker.js";
+
+export interface SearchOptions {
+  source?: string;
+  contentType?: "code" | "prose";
+  limit?: number;
+}
+
+export interface SearchResult {
+  title: string;
+  body: string;
+  source: string;
+  contentType: string;
+  score: number;
+}
+
+export interface QueryGroupedResults {
+  query: string;
+  results: SearchResult[];
+}
+
+export interface StoreStats {
+  totalChunks: number;
+  sources: string[];
+  dbSizeBytes: number;
+}
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS content_chunks (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  source TEXT NOT NULL,
+  title TEXT NOT NULL DEFAULT '',
+  body TEXT NOT NULL,
+  content_type TEXT NOT NULL DEFAULT 'prose'
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS content_chunks_fts USING fts5(
+  title,
+  body,
+  content='content_chunks',
+  content_rowid='id',
+  tokenize='porter'
+);
+
+CREATE TRIGGER IF NOT EXISTS content_chunks_ai AFTER INSERT ON content_chunks BEGIN
+  INSERT INTO content_chunks_fts(rowid, title, body) VALUES (new.id, new.title, new.body);
+END;
+
+CREATE TRIGGER IF NOT EXISTS content_chunks_ad AFTER DELETE ON content_chunks BEGIN
+  INSERT INTO content_chunks_fts(content_chunks_fts, rowid, title, body) VALUES ('delete', old.id, old.title, old.body);
+END;
+
+CREATE TABLE IF NOT EXISTS url_cache (
+  url TEXT NOT NULL,
+  source TEXT NOT NULL,
+  fetched_at INTEGER NOT NULL,
+  PRIMARY KEY (url, source)
+);
+`;
+
+export class KnowledgeStore {
+  private _db: Database;
+  private dbPath: string;
+
+  /** Public accessor for direct SQL on extension tables (e.g. url_cache). */
+  get db(): Database {
+    return this._db;
+  }
+
+  constructor(dbPath: string) {
+    this.dbPath = dbPath;
+    this._db = new Database(dbPath);
+    this._db.exec("PRAGMA journal_mode=WAL");
+  }
+
+  init(): void {
+    this._db.exec(SCHEMA);
+  }
+  index(chunks: Chunk[], source: string): void {
+    const del = this._db.prepare("DELETE FROM content_chunks WHERE source = ?");
+    const ins = this._db.prepare(
+      "INSERT INTO content_chunks (source, title, body, content_type) VALUES (?, ?, ?, ?)",
+    );
+
+    this._db.transaction(() => {
+      del.run(source);
+      for (const chunk of chunks) {
+        ins.run(source, chunk.title, chunk.body, chunk.contentType);
+      }
+    })();
+  }
+
+  search(queries: string[], options?: SearchOptions): QueryGroupedResults[] {
+    if (!queries.length) return [];
+
+    const limit = options?.limit ?? 3;
+    const results: QueryGroupedResults[] = [];
+
+    for (const query of queries) {
+      const sanitized = sanitizeFtsQuery(query);
+      if (!sanitized) {
+        results.push({ query, results: [] });
+        continue;
+      }
+
+      let sql = `
+        SELECT c.title, c.body, c.source, c.content_type AS contentType,
+               bm25(content_chunks_fts, 5.0, 1.0) AS score
+        FROM content_chunks_fts f
+        JOIN content_chunks c ON c.id = f.rowid
+        WHERE content_chunks_fts MATCH ?
+      `;
+      const params: (string | number)[] = [sanitized];
+
+      if (options?.source) {
+        sql += " AND c.source LIKE '%' || ? || '%'";
+        params.push(options.source);
+      }
+      if (options?.contentType) {
+        sql += " AND c.content_type = ?";
+        params.push(options.contentType);
+      }
+
+      sql += " ORDER BY score LIMIT ?";
+      params.push(limit);
+
+      try {
+        const rows = this._db.prepare(sql).all(...params) as SearchResult[];
+        results.push({ query, results: rows });
+      } catch {
+        // FTS5 query syntax error — return empty for this query
+        results.push({ query, results: [] });
+      }
+    }
+
+    return results;
+  }
+
+  purge(): number {
+    const row = this._db.prepare("SELECT COUNT(*) AS cnt FROM content_chunks").get() as {
+      cnt: number;
+    };
+    const count = row.cnt;
+    this._db.exec("DELETE FROM content_chunks");
+    this._db.exec("INSERT INTO content_chunks_fts(content_chunks_fts) VALUES('rebuild')");
+    this._db.exec("DELETE FROM url_cache");
+    return count;
+  }
+
+  getStats(): StoreStats {
+    const countRow = this._db.prepare("SELECT COUNT(*) AS cnt FROM content_chunks").get() as {
+      cnt: number;
+    };
+    const sourceRows = this._db
+      .prepare("SELECT DISTINCT source FROM content_chunks ORDER BY source")
+      .all() as { source: string }[];
+    const dbSizeBytes = fs.statSync(this.dbPath).size;
+
+    return {
+      totalChunks: countRow.cnt,
+      sources: sourceRows.map((r) => r.source),
+      dbSizeBytes,
+    };
+  }
+
+  pruneExpiredUrls(ttlHours = 24): number {
+    const cutoff = Math.floor(Date.now() / 1000) - ttlHours * 3600;
+    const result = this._db.prepare("DELETE FROM url_cache WHERE fetched_at < ?").run(cutoff);
+    return result.changes;
+  }
+
+  close(): void {
+    this._db.close();
+  }
+}
+
+/** Strip FTS5 special operators to prevent syntax errors. Keep alphanumeric + spaces. */
+function sanitizeFtsQuery(query: string): string {
+  // Remove characters that have special meaning in FTS5: ^, *, ", (, ), {, }, +, -
+  // Keep words separated by spaces for implicit AND matching
+  return query
+    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}