wotann 0.5.0

package/dist/memory/omega-layers.js

@@ -0,0 +1,263 @@
+/**
+ * OMEGA 3-layer memory orchestrator — Phase 2 P1-M2.
+ *
+ * OMEGA (omegamax.co, Apache-2.0, 95.4% LongMemEval) organizes memory
+ * in three canonical layers:
+ *
+ *   Layer 1 — Raw event log (immutable, append-only)
+ *   Layer 2 — Extracted facts (structured entries + KG relations)
+ *   Layer 3 — Compressed summaries (periodic L1→L3 distillation)
+ *
+ * This module is a read/write FACADE. It does not duplicate state;
+ * rather it composes the 3-layer abstraction over the existing WOTANN
+ * tables:
+ *
+ *   L1 → `auto_capture` (exists; append-only event log)
+ *   L2 → `memory_entries` + `knowledge_nodes` + `knowledge_edges` (exist)
+ *   L3 → `memory_summaries` (NEW: created by this module)
+ *
+ * Regression-lock: we do NOT change the existing `memory_entries.layer`
+ * enum (8 values) or the `auto_capture` schema. The OMEGA semantics
+ * live in the facade, not the tables.
+ *
+ * Design principles:
+ *
+ *   1. The facade is STATELESS. `createOmegaLayers` returns plain
+ *      objects over a MemoryStore — the store is the ground truth.
+ *      Two facades over the same store see the same data (per-session
+ *      isolation is handled by the store's session_id column, not the
+ *      facade).
+ *
+ *   2. Compression is INJECTION-DRIVEN. The LLM call is passed in via
+ *      `OmegaLlmQuery` (same shape as other WOTANN modules). Tests
+ *      mock the LLM; production wires it to the runtime provider.
+ *
+ *   3. Lineage is preserved. Every L3 summary references the L1 event
+ *      ids it was distilled from (`source_event_ids` JSON column).
+ *      This means a caller can always ask "what raw events back this
+ *      summary?" and follow the chain back to ground truth.
+ *
+ *   4. Honest failure. If the LLM throws or returns empty, `compress`
+ *      returns null (no stub summary, no fake content). Callers
+ *      decide whether to retry, skip, or surface the failure.
+ */
+// ── Facade factory ─────────────────────────────────────
+export function createOmegaLayers(config) {
+    const summariesTable = config.summariesTable ?? "memory_summaries";
+    // Identifier safety
+    if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(summariesTable)) {
+        throw new Error(`omega-layers: invalid summariesTable "${summariesTable}" (alnum + _ only)`);
+    }
+    const db = config.store.db;
+    if (!db || typeof db.prepare !== "function") {
+        throw new Error("omega-layers: MemoryStore missing expected db handle");
+    }
+    // Ensure the L3 summaries table exists. Idempotent.
+    ensureSummariesTable(db, summariesTable);
+    // Prepared statements for L3 — reused across calls.
+    const stmts = prepareSummaryStmts(db, summariesTable);
+    return {
+        layer1: createLayer1(db, config.store),
+        layer2: createLayer2(config.store),
+        layer3: createLayer3(db, stmts),
+    };
+}
+// ── Layer 1 ────────────────────────────────────────────
+function createLayer1(db, store) {
+    const insertStmt = db.prepare(`INSERT INTO auto_capture (event_type, tool_name, content, session_id)
+     VALUES (?, ?, ?, ?)`);
+    const countStmt = db.prepare(`SELECT COUNT(*) AS c FROM auto_capture`);
+    return {
+        append: (args) => {
+            // Reuse store.captureEvent semantics (content is truncated to 2000)
+            // to stay consistent with the rest of the codebase. But we insert
+            // directly here so we can capture the lastInsertRowid for lineage.
+            const info = insertStmt.run(args.eventType, args.toolName ?? null, args.content.slice(0, 2000), args.sessionId ?? null);
+            return Number(info.lastInsertRowid);
+        },
+        query: (args) => {
+            const conds = [];
+            const params = [];
+            if (args.sessionId !== undefined) {
+                conds.push("session_id = ?");
+                params.push(args.sessionId);
+            }
+            if (args.since !== undefined) {
+                conds.push("created_at >= ?");
+                params.push(args.since);
+            }
+            if (args.until !== undefined) {
+                conds.push("created_at <= ?");
+                params.push(args.until);
+            }
+            const where = conds.length > 0 ? `WHERE ${conds.join(" AND ")}` : "";
+            const limit = Math.max(1, Math.floor(args.limit ?? 500));
+            params.push(limit);
+            const sql = `SELECT id, event_type, tool_name, content, session_id, created_at
+                   FROM auto_capture ${where}
+                   ORDER BY id ASC
+                   LIMIT ?`;
+            const rows = db.prepare(sql).all(...params);
+            return rows.map((r) => ({
+                id: Number(r["id"] ?? 0),
+                eventType: String(r["event_type"] ?? ""),
+                ...(r["tool_name"] != null ? { toolName: String(r["tool_name"]) } : {}),
+                content: String(r["content"] ?? ""),
+                ...(r["session_id"] != null ? { sessionId: String(r["session_id"]) } : {}),
+                createdAt: String(r["created_at"] ?? ""),
+            }));
+        },
+        count: () => {
+            // store.getAutoCaptureEntries is lossy (caps); use direct count.
+            void store; // silence unused
+            const row = countStmt.get();
+            return row.c;
+        },
+    };
+}
+// ── Layer 2 ────────────────────────────────────────────
+function createLayer2(store) {
+    return {
+        search: (query, limit = 20) => {
+            // Reuse store.search which hits memory_fts. Projection to
+            // Layer2Fact keeps the OMEGA surface narrow.
+            const results = store.search(query, limit);
+            return results.map((r) => ({
+                id: r.entry.id,
+                layer: r.entry.layer,
+                blockType: r.entry.blockType,
+                key: r.entry.key,
+                value: r.entry.value,
+                score: r.score,
+            }));
+        },
+        getById: (id) => {
+            return store.getById(id);
+        },
+    };
+}
+function ensureSummariesTable(db, table) {
+    db.prepare(`CREATE TABLE IF NOT EXISTS ${table} (
+       id TEXT PRIMARY KEY,
+       content TEXT NOT NULL,
+       session_id TEXT,
+       source_event_ids TEXT NOT NULL DEFAULT '[]',
+       source_event_count INTEGER NOT NULL DEFAULT 0,
+       created_at TEXT NOT NULL DEFAULT (datetime('now'))
+     )`).run();
+    db.prepare(`CREATE INDEX IF NOT EXISTS idx_${table}_session ON ${table}(session_id)`).run();
+    db.prepare(`CREATE INDEX IF NOT EXISTS idx_${table}_created ON ${table}(created_at DESC)`).run();
+}
+function prepareSummaryStmts(db, table) {
+    return {
+        insert: db.prepare(`INSERT INTO ${table} (id, content, session_id, source_event_ids, source_event_count)
+       VALUES (?, ?, ?, ?, ?)`),
+        list: db.prepare(`SELECT id, content, session_id, source_event_ids, source_event_count, created_at
+       FROM ${table} ORDER BY created_at DESC LIMIT ?`),
+        getById: db.prepare(`SELECT id, content, session_id, source_event_ids, source_event_count, created_at
+       FROM ${table} WHERE id = ?`),
+        count: db.prepare(`SELECT COUNT(*) AS c FROM ${table}`),
+    };
+}
+function rowToSummary(row) {
+    let ids = [];
+    try {
+        const parsed = JSON.parse(String(row["source_event_ids"] ?? "[]"));
+        if (Array.isArray(parsed)) {
+            ids = parsed.filter((n) => typeof n === "number");
+        }
+    }
+    catch {
+        ids = [];
+    }
+    return {
+        id: String(row["id"]),
+        content: String(row["content"] ?? ""),
+        ...(row["session_id"] != null ? { sessionId: String(row["session_id"]) } : {}),
+        sourceEventIds: ids,
+        sourceEventCount: Number(row["source_event_count"] ?? ids.length),
+        createdAt: String(row["created_at"] ?? ""),
+    };
+}
+function createLayer3(db, stmts) {
+    return {
+        compress: async (args) => {
+            // 1. Gather L1 events in scope.
+            const conds = [];
+            const params = [];
+            if (args.sessionId !== undefined) {
+                conds.push("session_id = ?");
+                params.push(args.sessionId);
+            }
+            if (args.since !== undefined) {
+                conds.push("created_at >= ?");
+                params.push(args.since);
+            }
+            if (args.until !== undefined) {
+                conds.push("created_at <= ?");
+                params.push(args.until);
+            }
+            const where = conds.length > 0 ? `WHERE ${conds.join(" AND ")}` : "";
+            const maxEvents = Math.max(1, Math.floor(args.maxEvents ?? 100));
+            params.push(maxEvents);
+            const sql = `SELECT id, event_type, tool_name, content, created_at
+                   FROM auto_capture ${where}
+                   ORDER BY id ASC
+                   LIMIT ?`;
+            const rows = db.prepare(sql).all(...params);
+            if (rows.length === 0)
+                return null;
+            // 2. Build compression prompt (deterministic, LLM-friendly).
+            const eventIds = rows.map((r) => Number(r["id"] ?? 0));
+            const prompt = buildCompressionPrompt(rows);
+            // 3. Invoke LLM. Honest failure: null on throw or empty.
+            let summaryText;
+            try {
+                summaryText = await args.llmQuery(prompt, { maxTokens: 400, temperature: 0.2 });
+            }
+            catch {
+                return null;
+            }
+            if (!summaryText || summaryText.trim().length === 0)
+                return null;
+            // 4. Persist with lineage.
+            const id = `summary-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
+            const sessionId = args.sessionId ?? null;
+            stmts.insert.run(id, summaryText.trim(), sessionId, JSON.stringify(eventIds), eventIds.length);
+            const saved = stmts.getById.get(id);
+            if (!saved)
+                return null;
+            return rowToSummary(saved);
+        },
+        list: (limit = 50) => {
+            const rows = stmts.list.all(Math.max(1, Math.floor(limit)));
+            return rows.map(rowToSummary);
+        },
+        getById: (id) => {
+            const row = stmts.getById.get(id);
+            if (!row)
+                return null;
+            return rowToSummary(row);
+        },
+        count: () => {
+            const row = stmts.count.get();
+            return row.c;
+        },
+    };
+}
+// ── Helpers ────────────────────────────────────────────
+function buildCompressionPrompt(rows) {
+    const header = "Compress the following raw agent events into a single concise summary. " +
+        "Preserve: decisions made, tools used, outcomes, errors. Reject: tool " +
+        "argument noise, repetitive details. Return ONLY the summary text.\n\n" +
+        "Events:\n";
+    const body = rows
+        .map((r, i) => {
+        const tool = r["tool_name"] ? `[${String(r["tool_name"])}] ` : "";
+        const content = String(r["content"] ?? "").slice(0, 500);
+        return `${i + 1}. ${tool}${content}`;
+    })
+        .join("\n");
+    return header + body + "\n\nSummary:";
+}
+//# sourceMappingURL=omega-layers.js.map

package/dist/memory/omega-layers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"omega-layers.js","sourceRoot":"","sources":["../../src/memory/omega-layers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AA6GH,0DAA0D;AAE1D,MAAM,UAAU,iBAAiB,CAAC,MAAyB;IACzD,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,kBAAkB,CAAC;IACnE,oBAAoB;IACpB,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,yCAAyC,cAAc,oBAAoB,CAAC,CAAC;IAC/F,CAAC;IASD,MAAM,EAAE,GAAI,MAAM,CAAC,KAAgC,CAAC,EAAE,CAAC;IACvD,IAAI,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,oDAAoD;IACpD,oBAAoB,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC;IAEzC,oDAAoD;IACpD,MAAM,KAAK,GAAG,mBAAmB,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC;IAEtD,OAAO;QACL,MAAM,EAAE,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC;QACtC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;QAClC,MAAM,EAAE,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,0DAA0D;AAE1D,SAAS,YAAY,CAAC,EAAqB,EAAE,KAAkB;IAC7D,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B;yBACqB,CACtB,CAAC;IACF,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;IAEvE,OAAO;QACL,MAAM,EAAE,CAAC,IAAsB,EAAU,EAAE;YACzC,oEAAoE;YACpE,kEAAkE;YAClE,mEAAmE;YACnE,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CACzB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,QAAQ,IAAI,IAAI,EACrB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAC3B,IAAI,CAAC,SAAS,IAAI,IAAI,CACvB,CAAC;YACF,OAAO,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACtC,CAAC;QAED,KAAK,EAAE,CAAC,IAAqB,EAA0B,EAAE;YACvD,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAwB,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,MAAM,GAAG,GAAG;uCACqB,KAAK;;2BAEjB,CAAC;YACtB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAA8B,CAAC;YACzE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;gBACxC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1E,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;aACzC,CAAC,CAAC,CAAC;QACN,CAAC;QAED,KAAK,EAAE,GAAW,EAAE;YAClB,iEAAiE;YACjE,KAAK,KAAK,CAAC,CAAC,iBAAiB;YAC7B,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,EAAmB,CAAC;YAC7C,OAAO,GAAG,CAAC,CAAC,CAAC;QACf,CAAC;KACF,CAAC;AACJ,CAAC;AAED,0DAA0D;AAE1D,SAAS,YAAY,CAAC,KAAkB;IACtC,OAAO;QACL,MAAM,EAAE,CAAC,KAAa,EAAE,QAAgB,EAAE,EAAyB,EAAE;YACnE,0DAA0D;YAC1D,6CAA6C;YAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAC3C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE;gBACd,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK;gBACpB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS;gBAC5B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK;gBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;aACf,CAAC,CAAC,CAAC;QACN,CAAC;QACD,OAAO,EAAE,CAAC,EAAU,EAAsB,EAAE;YAC1C,OAAO,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;KACF,CAAC;AACJ,CAAC;AAWD,SAAS,oBAAoB,CAAC,EAAqB,EAAE,KAAa;IAChE,EAAE,CAAC,OAAO,CACR,8BAA8B,KAAK;;;;;;;OAOhC,CACJ,CAAC,GAAG,EAAE,CAAC;IACR,EAAE,CAAC,OAAO,CAAC,kCAAkC,KAAK,eAAe,KAAK,cAAc,CAAC,CAAC,GAAG,EAAE,CAAC;IAC5F,EAAE,CAAC,OAAO,CAAC,kCAAkC,KAAK,eAAe,KAAK,mBAAmB,CAAC,CAAC,GAAG,EAAE,CAAC;AACnG,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,KAAa;IAC/D,OAAO;QACL,MAAM,EAAE,EAAE,CAAC,OAAO,CAChB,eAAe,KAAK;8BACI,CACzB;QACD,IAAI,EAAE,EAAE,CAAC,OAAO,CACd;cACQ,KAAK,mCAAmC,CACjD;QACD,OAAO,EAAE,EAAE,CAAC,OAAO,CACjB;cACQ,KAAK,eAAe,CAC7B;QACD,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,6BAA6B,KAAK,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAA4B;IAChD,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACnE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,GAAG,EAAE,CAAC;IACX,CAAC;IACD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACrC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC;QACjE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;KAC3C,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,EAAqB,EAAE,KAAmB;IAC9D,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,IAAwB,EAAsC,EAAE;YAC/E,gCAAgC;YAChC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAwB,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvB,MAAM,GAAG,GAAG;uCACqB,KAAK;;2BAEjB,CAAC;YACtB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAA8B,CAAC;YAEzE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAEnC,6DAA6D;YAC7D,MAAM,QAAQ,GAAa,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAE5C,yDAAyD;YACzD,IAAI,WAAmB,CAAC;YACxB,IAAI,CAAC;gBACH,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YAClF,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAEjE,2BAA2B;YAC3B,MAAM,EAAE,GAAG,WAAW,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC9E,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC;YACzC,KAAK,CAAC,MAAM,CAAC,GAAG,CACd,EAAE,EACF,WAAW,CAAC,IAAI,EAAE,EAClB,SAAS,EACT,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EACxB,QAAQ,CAAC,MAAM,CAChB,CAAC;YACF,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAwC,CAAC;YAC3E,IAAI,CAAC,KAAK;gBAAE,OAAO,IAAI,CAAC;YACxB,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,EAAE,CAAC,QAAgB,EAAE,EAAiC,EAAE;YAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAA8B,CAAC;YACzF,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,EAAE,CAAC,EAAU,EAA6B,EAAE;YACjD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAwC,CAAC;YACzE,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAC;YACtB,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,KAAK,EAAE,GAAW,EAAE;YAClB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAmB,CAAC;YAC/C,OAAO,GAAG,CAAC,CAAC,CAAC;QACf,CAAC;KACF,CAAC;AACJ,CAAC;AAED,0DAA0D;AAE1D,SAAS,sBAAsB,CAAC,IAA+B;IAC7D,MAAM,MAAM,GACV,yEAAyE;QACzE,uEAAuE;QACvE,uEAAuE;QACvE,WAAW,CAAC;IACd,MAAM,IAAI,GAAG,IAAI;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACzD,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,OAAO,EAAE,CAAC;IACvC,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,OAAO,MAAM,GAAG,IAAI,GAAG,cAAc,CAAC;AACxC,CAAC"}

package/dist/memory/onnx-cross-encoder.d.ts

@@ -0,0 +1,239 @@
+/**
+ * ONNX cross-encoder — Phase 2 P1-M2 (OMEGA port) + V9 Wave 4-CC
+ * real WordPiece tokenizer.
+ *
+ * Cross-encoders score (query, doc) PAIRS for relevance and are the
+ * rerank standard after dense+sparse retrieval. OMEGA uses
+ * `cross-encoder/ms-marco-MiniLM-L-6-v2` (Apache-2.0, ~90MB) via
+ * onnxruntime-node. This module is the WOTANN port.
+ *
+ * Design notes:
+ *
+ *   1. ONNX runtime + model are OPTIONAL. The module surfaces two
+ *      probe functions:
+ *        - isOnnxRuntimeAvailable(): boolean — is the native runtime
+ *          importable?
+ *        - isMiniLmModelAvailable(path?): boolean — is the specific
+ *          model file on disk?
+ *      Callers (store.ts, temprSearch) use these to decide whether
+ *      to wire the ONNX session or stay on M4's heuristic encoder.
+ *
+ *   2. Model files are LARGE (~90MB) and NEVER committed. A dedicated
+ *      `scripts/download-minilm.mjs` (like the LongMemEval script)
+ *      handles opt-in download into `.wotann/models/`. The companion
+ *      `vocab.txt` (~226KB BERT WordPiece vocab) is downloaded by the
+ *      same script and lives next to the model file.
+ *
+ *   3. Honest fallback: if the session throws during rerank, we fall
+ *      back to the M4 heuristic encoder per-batch. The caller gets
+ *      a result (never a crash) but `rerankerApplied: false` in the
+ *      TEMPR result signals the graceful degradation.
+ *
+ *   4. Tokenization (V9 Wave 4-CC): MiniLM uses a 30522-token
+ *      WordPiece vocab (BERT-base uncased). This module ships TWO
+ *      tokenizers:
+ *        - tokenizePair: SIMPLIFIED byte-level fallback. Produces
+ *          a stable input shape but scores are not semantically
+ *          equivalent to real WordPiece. Honest stub when vocab is
+ *          unavailable.
+ *        - WordPieceTokenizer + createWordPieceTokenizePair: real
+ *          BERT-style tokenization (BasicTokenizer + greedy WordPiece
+ *          subword). Loads vocab.txt lazily; per-instance state (no
+ *          module globals) so multiple models can co-exist.
+ *      Use `loadMiniLmTokenizer(vocabPath?)` to get a `PairTokenizer`
+ *      that auto-falls-back to byte-level when vocab.txt is missing.
+ *
+ *   5. Injection: the `session` config accepts any object conforming
+ *      to the OnnxSession interface. Tests use a mock; production
+ *      uses onnxruntime-node's `InferenceSession`. The interface is
+ *      intentionally NARROW so other runtimes (WebGL, ORT WebAssembly)
+ *      can be swapped without touching callers.
+ */
+import { type CrossEncoder } from "./cross-encoder.js";
+/**
+ * Narrow interface over onnxruntime-node's InferenceSession. We only
+ * need `run(feeds)` returning output tensors keyed by output name.
+ * This lets tests mock the session without dragging in the native
+ * runtime.
+ */
+export interface OnnxSession {
+    readonly run: (feeds: Record<string, OnnxTensor>) => Promise<Record<string, OnnxTensor>>;
+}
+export interface OnnxTensor {
+    readonly data: BigInt64Array | Float32Array | Int32Array;
+    readonly dims?: readonly number[];
+}
+/**
+ * Tokenized (query, doc) pair ready for ONNX input. Values mirror
+ * MiniLM's 3-input schema: input_ids, attention_mask, token_type_ids.
+ */
+export interface TokenizedPair {
+    readonly inputIds: BigInt64Array;
+    readonly attentionMask: BigInt64Array;
+    readonly tokenTypeIds: BigInt64Array;
+}
+/**
+ * Custom tokenizer signature for callers that want real WordPiece.
+ * Default tokenizer is deterministic but not semantically optimal.
+ */
+export type PairTokenizer = (query: string, doc: string, maxLen?: number) => TokenizedPair;
+export interface OnnxCrossEncoderConfig {
+    /**
+     * ONNX inference session. When undefined, rerank() ALWAYS falls back
+     * to the M4 heuristic encoder. Pass a real session (from
+     * onnxruntime-node's InferenceSession.create()) for production.
+     */
+    readonly session?: OnnxSession;
+    /** Override the input tokenizer. Default: tokenizePair. */
+    readonly tokenize?: PairTokenizer;
+    /** Max token sequence length. Default 128 (MiniLM's typical ceiling). */
+    readonly maxLength?: number;
+    /** Output tensor name on the ONNX graph. Default "logits". */
+    readonly outputName?: string;
+}
+/**
+ * Is onnxruntime-node loadable in this process? Cached. Does NOT
+ * require a model file — only that the native runtime binary is
+ * available. Callers still need to verify the model file separately.
+ */
+export declare function isOnnxRuntimeAvailable(): boolean;
+/**
+ * Is a MiniLM ONNX model file present on disk? Defaults to looking at
+ * `.wotann/models/ms-marco-MiniLM-L-6-v2.onnx` (set by the download
+ * script). Callers can pass an explicit path.
+ */
+export declare function isMiniLmModelAvailable(path?: string): boolean;
+/**
+ * Is a BERT vocab.txt present on disk for MiniLM? Independent of the
+ * .onnx model probe — the runtime can load the model without vocab
+ * (it just shapes inputs poorly), and a vocab without a model is
+ * still useful for offline tokenization tests.
+ */
+export declare function isMiniLmVocabAvailable(path?: string): boolean;
+/**
+ * WordPiece tokenizer state. Per-instance (NOT module-global per
+ * QB#7) so multiple vocabs can coexist (e.g., a MiniLM vocab AND
+ * a fine-tuned reranker vocab in the same process).
+ *
+ * Construct via `loadWordPieceTokenizer(path)`. The class is exported
+ * so callers can build a tokenizer from a vocab they already have in
+ * memory (e.g., bundled, fetched, generated for tests).
+ */
+export declare class WordPieceTokenizer {
+    /** Token → id lookup. Built once at construction; never mutated. */
+    private readonly vocab;
+    /** Cached id for [UNK]. */
+    private readonly unkId;
+    /** Cached id for [CLS]. */
+    private readonly clsId;
+    /** Cached id for [SEP]. */
+    private readonly sepId;
+    /** Cached id for [PAD]. */
+    private readonly padId;
+    /**
+     * BERT base uncased lowercases inputs (and strips accents). Some
+     * vocabs (`bert-base-cased`) keep case. Default true matches MiniLM.
+     */
+    private readonly doLowerCase;
+    /** Max characters per word — WordPiece splits long words to UNK. */
+    private readonly maxWordCharsForUnk;
+    constructor(vocab: ReadonlyMap<string, number>, options?: {
+        readonly doLowerCase?: boolean;
+        readonly maxWordCharsForUnk?: number;
+    });
+    /** Special-token ids exposed for downstream pair builders. */
+    readonly specials: () => {
+        cls: number;
+        sep: number;
+        pad: number;
+        unk: number;
+    };
+    /**
+     * Tokenize a single string into vocab ids. Implements the standard
+     * BERT pipeline: BasicTokenizer (clean → lowercase+strip-accents →
+     * punctuation split) → WordPiece (greedy longest-match subword).
+     */
+    encode(text: string): number[];
+    /**
+     * BasicTokenizer: whitespace + punctuation split, optional
+     * lowercase + accent stripping. Returns the word stream.
+     */
+    private basicTokenize;
+    /**
+     * Greedy longest-match WordPiece subword tokenization. For each
+     * word, peel off the longest prefix that exists in the vocab,
+     * then continue from the remainder with "##" prefix. If any piece
+     * misses, the entire word becomes [UNK].
+     */
+    private wordPieceEncode;
+}
+/**
+ * Load a WordPiece tokenizer from a BERT vocab.txt file. Returns
+ * undefined when the file is missing or malformed — caller should
+ * fall back to the byte-level tokenizer (honest-stub principle, QB#6).
+ *
+ * vocab.txt format: one token per line, line index = token id.
+ * Empty lines are PRESERVED as ids (BERT vocab has unused [unusedN]
+ * slots; their ids must align with the model's embedding table).
+ */
+export declare function loadWordPieceTokenizer(vocabPath?: string, options?: {
+    readonly doLowerCase?: boolean;
+}): WordPieceTokenizer | undefined;
+/**
+ * Build a `PairTokenizer` (the config slot type) backed by a real
+ * WordPiece tokenizer. The output mirrors BERT's pair-input schema:
+ *
+ *   [CLS] query-tokens [SEP] doc-tokens [SEP] [PAD...]
+ *
+ * with token_type_ids 0 for the query side and 1 for the doc side
+ * (same as the existing byte-level `tokenizePair`).
+ *
+ * Returned function is pure on the captured tokenizer — no shared
+ * mutable state across calls.
+ */
+export declare function createWordPieceTokenizePair(tokenizer: WordPieceTokenizer): PairTokenizer;
+/**
+ * High-level loader: returns a `PairTokenizer` backed by real
+ * WordPiece when vocab.txt is present, or `undefined` when missing.
+ * Callers should fall back to `tokenizePair` (the byte-level stub)
+ * on undefined per QB#6 honest-fallback.
+ *
+ * Example wiring in store.ts (T1.4 attach point):
+ *   const pairTokenizer =
+ *     loadMiniLmTokenizer(vocabPath) ?? tokenizePair;
+ *   const enc = createOnnxCrossEncoder({ session, tokenize: pairTokenizer });
+ */
+export declare function loadMiniLmTokenizer(vocabPath?: string): PairTokenizer | undefined;
+/**
+ * Byte-level deterministic tokenizer for (query, doc) pairs. HONEST
+ * STUB (QB#6): produces a stable input shape so the ONNX runtime can
+ * process the pair, but the scores it produces are NOT semantically
+ * equivalent to real WordPiece on MiniLM.
+ *
+ * Use the real path when vocab.txt is on disk:
+ *   const tok = loadMiniLmTokenizer();              // PairTokenizer | undefined
+ *   const enc = createOnnxCrossEncoder({
+ *     session,
+ *     tokenize: tok ?? tokenizePair,                // honest fallback
+ *   });
+ *
+ * Schema mirrors BERT-family:
+ *   [CLS] query-tokens [SEP] doc-tokens [SEP] [PAD...]
+ *
+ * Special token ids (matched to BERT-base-uncased so a real model
+ * sees the right specials even on the stub path):
+ *   CLS=101, SEP=102, PAD=0
+ */
+export declare function tokenizePair(query: string, doc: string, maxLen?: number): TokenizedPair;
+export declare function createOnnxCrossEncoder(config: OnnxCrossEncoderConfig): CrossEncoder;
+/**
+ * Load the ms-marco-MiniLM-L-6-v2 ONNX model into an
+ * onnxruntime-node session. Returns undefined if the runtime or the
+ * model file is missing. Callers SHOULD handle undefined by calling
+ * `createOnnxCrossEncoder({})` (no session → heuristic fallback).
+ *
+ * This is an async loader — the caller awaits once at startup and
+ * reuses the session across queries.
+ */
+export declare function loadMiniLmSession(modelPath?: string): Promise<OnnxSession | undefined>;
+//# sourceMappingURL=onnx-cross-encoder.d.ts.map

package/dist/memory/onnx-cross-encoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"onnx-cross-encoder.d.ts","sourceRoot":"","sources":["../../src/memory/onnx-cross-encoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AAGH,OAAO,EAEL,KAAK,YAAY,EAGlB,MAAM,oBAAoB,CAAC;AAI5B;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;CAC1F;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,aAAa,GAAG,YAAY,GAAG,UAAU,CAAC;IACzD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACnC;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC;CACtC;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAE3F,MAAM,WAAW,sBAAsB;IACrC;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC;IAC/B,2DAA2D;IAC3D,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAClC,yEAAyE;IACzE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,8DAA8D;IAC9D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAMD;;;;GAIG;AACH,wBAAgB,sBAAsB,IAAI,OAAO,CAShD;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAS7D;AAeD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAS7D;AAoBD;;;;;;;;GAQG;AACH,qBAAa,kBAAkB;IAC7B,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IACpD,2BAA2B;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,2BAA2B;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,2BAA2B;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,2BAA2B;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;gBAG1C,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;KAAE;IAoBpF,8DAA8D;IAC9D,QAAQ,CAAC,QAAQ,QAAO;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAK3E;IAEH;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAS9B;;;OAGG;IACH,OAAO,CAAC,aAAa;IAcrB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CA+BxB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,GAC3C,kBAAkB,GAAG,SAAS,CAoBhC;AAqFD;;;;;;;;;;;GAWG;AACH,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,kBAAkB,GAAG,aAAa,CAwDxF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAIjF;AAID;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,MAAY,GAAG,aAAa,CA2D5F;AAgBD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,sBAAsB,GAAG,YAAY,CAyDnF;AAID;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAe5F"}