pi-mega-compact 0.4.24 → 0.4.25

package/dist/extensions/openclaw-mega-compact.js

@@ -0,0 +1,291 @@
+/**
+ * openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
+ *
+ * Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
+ *  - Registers a CompactionProvider that replaces the built-in summarizeInStages.
+ *  - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
+ *  - Hooks into `before_compaction` / `after_compaction` for diagnostics.
+ *
+ * Design constraints:
+ *  - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
+ *  - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
+ *  - No network at runtime — everything is local (stores + extractive summarizer).
+ */
+import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
+import { compactSession, setDefaultStore, } from "../src/engine.js";
+import { recallAndInline } from "../src/recall.js";
+import { VectorStore } from "../src/vectorStore.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const PLUGIN_ID = "mega-compact";
+const PLUGIN_LABEL = "Mega Compact (Trident)";
+/** Default state directory for vector store persistence. */
+const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
+/** Minimum messages before we bother compacting. */
+const MIN_MESSAGES_FOR_COMPACT = 6;
+// ---------------------------------------------------------------------------
+// Message conversion — OpenClaw unknown[] → EngineMessage[]
+// ---------------------------------------------------------------------------
+/**
+ * Best-effort conversion from OpenClaw's opaque message array to our
+ * EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
+ * handle whatever shape comes through gracefully.
+ */
+function toEngineMessages(messages) {
+    return messages.map((msg) => {
+        if (!msg || typeof msg !== "object") {
+            // Primitive fallback — treat as custom text.
+            return {
+                role: "custom",
+                text: String(msg ?? ""),
+            };
+        }
+        const m = msg;
+        const role = typeof m.role === "string" ? m.role : "custom";
+        // Normalize role to one of our four engine roles.
+        let engineRole;
+        switch (role) {
+            case "user":
+                engineRole = "user";
+                break;
+            case "assistant":
+                engineRole = "assistant";
+                break;
+            case "tool":
+            case "function":
+                engineRole = "tool";
+                break;
+            default:
+                engineRole = "custom";
+                break;
+        }
+        // Extract text content from common message shapes.
+        const text = typeof m.content === "string"
+            ? m.content
+            : typeof m.text === "string"
+                ? m.text
+                : Array.isArray(m.content)
+                    ? m.content
+                        .filter((part) => part.type === "text" && typeof part.text === "string")
+                        .map((part) => part.text)
+                        .join("\n")
+                    : "";
+        // Preserve tool metadata when present.
+        const toolName = typeof m.name === "string"
+            ? m.name
+            : typeof m.toolName === "string"
+                ? m.toolName
+                : undefined;
+        const input = typeof m.input === "string"
+            ? m.input
+            : typeof m.arguments === "string"
+                ? m.arguments
+                : m.arguments !== undefined
+                    ? JSON.stringify(m.arguments)
+                    : undefined;
+        const output = typeof m.output === "string"
+            ? m.output
+            : engineRole === "tool" && typeof m.content === "string"
+                ? m.content
+                : undefined;
+        return { role: engineRole, text, toolName, input, output };
+    });
+}
+// ---------------------------------------------------------------------------
+// Compaction provider
+// ---------------------------------------------------------------------------
+function createCompactionProvider(store) {
+    return {
+        id: PLUGIN_ID,
+        label: PLUGIN_LABEL,
+        async summarize({ messages, signal, compressionRatio, }) {
+            // Abort check — bail early if the caller cancelled.
+            if (signal?.aborted) {
+                throw new DOMException("Aborted", "AbortError");
+            }
+            const engineMessages = toEngineMessages(messages);
+            // Nothing meaningful to compact.
+            if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
+                return "";
+            }
+            // Map compression ratio → keepFrom boundary.
+            // compressionRatio=0.5 means "compact the oldest 50%".
+            // Default to compacting the oldest half if not specified.
+            const ratio = compressionRatio ?? 0.5;
+            const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
+            // Abort check after conversion (conversion is cheap but check anyway).
+            if (signal?.aborted) {
+                throw new DOMException("Aborted", "AbortError");
+            }
+            const sessionId = `openclaw-${Date.now()}`;
+            const input = {
+                sessionId,
+                messages: engineMessages,
+                keepFrom,
+            };
+            const result = compactSession(input, store);
+            if (result.skipped) {
+                return "";
+            }
+            return result.summary;
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Plugin entry
+// ---------------------------------------------------------------------------
+export default definePluginEntry({
+    id: PLUGIN_ID,
+    name: "Mega Compact",
+    description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
+    register(api) {
+        const logger = api.logger;
+        // Resolve state directory — prefer plugin config override.
+        const pluginCfg = (api.pluginConfig ?? {});
+        const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
+            ? pluginCfg.stateDir
+            : STATE_DIR;
+        // Initialize vector store.
+        let store;
+        try {
+            store = new VectorStore({ stateDir });
+            setDefaultStore(store);
+            logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
+        }
+        catch (err) {
+            logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
+            return; // Hard bail — no point registering if store is broken.
+        }
+        // -----------------------------------------------------------------------
+        // Register compaction provider
+        // -----------------------------------------------------------------------
+        const provider = createCompactionProvider(store);
+        api.registerCompactionProvider(provider);
+        logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
+        // -----------------------------------------------------------------------
+        // Hooks — before / after compaction diagnostics
+        // -----------------------------------------------------------------------
+        api.registerHook({
+            event: "before_compaction",
+            handler: async (ctx) => {
+                const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
+                logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
+            },
+        });
+        api.registerHook({
+            event: "after_compaction",
+            handler: async (ctx) => {
+                const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
+                logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
+            },
+        });
+        // -----------------------------------------------------------------------
+        // Tool: mega_status
+        // -----------------------------------------------------------------------
+        api.registerTool({
+            name: "mega_status",
+            description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
+            parameters: {
+                type: "object",
+                properties: {
+                    sessionId: {
+                        type: "string",
+                        description: "Optional session ID to scope stats to.",
+                    },
+                },
+                additionalProperties: false,
+            },
+            handler: async (args) => {
+                const sessionId = args?.sessionId ?? "global";
+                try {
+                    const stats = store.stats(sessionId);
+                    const parts = [
+                        `**Mega Compact Status**`,
+                        `Session: ${sessionId}`,
+                        `Checkpoints: ${stats.checkpointCount}`,
+                        `Total tokens saved: ${stats.totalTokenEstimate}`,
+                        `Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
+                        `Injected count: ${stats.injectedCount}`,
+                        `Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
+                    ];
+                    if (stats.lastSummary) {
+                        parts.push(`\nLast summary (truncated):\n  ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
+                    }
+                    return { content: [{ type: "text", text: parts.join("\n") }] };
+                }
+                catch (err) {
+                    return {
+                        content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
+                        isError: true,
+                    };
+                }
+            },
+        });
+        // -----------------------------------------------------------------------
+        // Tool: mega_recall
+        // -----------------------------------------------------------------------
+        api.registerTool({
+            name: "mega_recall",
+            description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
+            parameters: {
+                type: "object",
+                properties: {
+                    sessionId: {
+                        type: "string",
+                        description: "Session ID to recall context for.",
+                    },
+                    query: {
+                        type: "string",
+                        description: "Natural language query for relevant context.",
+                    },
+                    limit: {
+                        type: "number",
+                        description: "Max checkpoints to recall (default 3).",
+                    },
+                },
+                required: ["sessionId", "query"],
+                additionalProperties: false,
+            },
+            handler: async (args) => {
+                const { sessionId, query, limit } = args;
+                if (!sessionId || !query) {
+                    return {
+                        content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
+                        isError: true,
+                    };
+                }
+                try {
+                    const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
+                    if (result.toInject.length === 0) {
+                        return {
+                            content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
+                        };
+                    }
+                    const parts = [
+                        `**Recalled ${result.toInject.length} checkpoint(s):**`,
+                        ...result.report,
+                        "",
+                        "---",
+                        result.block,
+                    ];
+                    return { content: [{ type: "text", text: parts.join("\n") }] };
+                }
+                catch (err) {
+                    return {
+                        content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
+                        isError: true,
+                    };
+                }
+            },
+        });
+        // -----------------------------------------------------------------------
+        // Cleanup on shutdown
+        // -----------------------------------------------------------------------
+        api.on("shutdown", () => {
+            logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
+            setDefaultStore(undefined);
+        });
+        logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
+    },
+});

package/dist/src/minilm.js

@@ -0,0 +1,92 @@
+/**
+ * minilm.ts — local MiniLM (all-MiniLM-L6-v2) sentence embedder (Sprint 12).
+ *
+ * Implements the `Embedder` interface so it drops into the existing VectorStore
+ * dedup cascade and search with no call-site changes. Inference is 100% local:
+ * the ONNX model + WordPiece vocab are on-disk artifacts fetched once by
+ * scripts/setup-minilm.mjs. There is NO network call at runtime (PREVENT-PI-004).
+ *
+ * Inputs (dynamic): input_ids, attention_mask, token_type_ids (int64).
+ * Output: last_hidden_state (batch, seq, 384). We mean-pool over non-padded
+ * tokens (attention_mask == 1) and L2-normalize → 384-dim unit vector.
+ *
+ * The ONNX session + tokenizer are loaded LAZILY on first embed() so the default
+ * TrigramEmbedder path (and its zero native-init cost) is untouched unless
+ * MEGACOMPACT_EMBEDDER=minilm is selected.
+ */
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { existsSync } from "node:fs";
+import { l2Normalize, awaitSync } from "./embedder.js";
+import { WordPieceTokenizer } from "./wordpiece.js";
+export const MINILM_DIM = 384;
+export const MINILM_MAX_LEN = 256;
+/** Resolve the model directory: MEGACOMPACT_MINILM_DIR > ./models/minilm > ~/.pi … */
+function resolveModelDir() {
+    if (process.env.MEGACOMPACT_MINILM_DIR)
+        return process.env.MEGACOMPACT_MINILM_DIR;
+    // Repo-local vendored path (gitignored).
+    const local = join(process.cwd(), "models", "minilm");
+    if (existsSync(local))
+        return local;
+    return join(homedir(), ".pi", "agent", "extensions", "mega-compact", "models", "minilm");
+}
+export class MiniLMEmbedder {
+    dim = MINILM_DIM;
+    session = null;
+    tokenizer = null;
+    modelDir;
+    loadPromise = null;
+    constructor(modelDir = resolveModelDir()) {
+        this.modelDir = modelDir;
+    }
+    async ensureLoaded() {
+        if (this.session && this.tokenizer)
+            return;
+        if (this.loadPromise)
+            return this.loadPromise;
+        this.loadPromise = (async () => {
+            const ort = await import("onnxruntime-node");
+            const modelPath = join(this.modelDir, "model_quantized.onnx");
+            const vocabPath = join(this.modelDir, "vocab.txt");
+            if (!existsSync(modelPath) || !existsSync(vocabPath)) {
+                throw new Error(`MiniLM artifacts missing in ${this.modelDir}. Run: node scripts/setup-minilm.mjs`);
+            }
+            // 1 thread is plenty for a single short-region embed and bounds CPU.
+            this.session = await ort.InferenceSession.create(modelPath, {
+                executionProviders: ["cpu"],
+                graphOptimizationLevel: "all",
+            });
+            this.tokenizer = WordPieceTokenizer.fromVocabFile(vocabPath);
+        })();
+        return this.loadPromise;
+    }
+    embed(text) {
+        awaitSync(this.ensureLoaded());
+        const enc = this.tokenizer.encode(text, MINILM_MAX_LEN);
+        const n = enc.inputIds.length;
+        const BigInt64 = (arr) => arr.map((x) => BigInt(x));
+        const ort = awaitSync(import("onnxruntime-node"));
+        const tensors = {
+            input_ids: new ort.Tensor("int64", BigInt64(enc.inputIds), [1, n]),
+            attention_mask: new ort.Tensor("int64", BigInt64(enc.attentionMask), [1, n]),
+            token_type_ids: new ort.Tensor("int64", BigInt64(enc.tokenTypeIds), [1, n]),
+        };
+        const out = awaitSync(this.session.run(tensors));
+        const hidden = out.last_hidden_state.data;
+        // hidden shape: [1, n, 384]. Mean-pool over non-padded positions.
+        const pooled = new Array(MINILM_DIM).fill(0);
+        let count = 0;
+        for (let i = 0; i < n; i++) {
+            if (enc.attentionMask[i] === 0)
+                continue;
+            const base = i * MINILM_DIM;
+            for (let d = 0; d < MINILM_DIM; d++)
+                pooled[d] += hidden[base + d];
+            count++;
+        }
+        if (count === 0)
+            return l2Normalize(new Array(MINILM_DIM).fill(0));
+        return l2Normalize(pooled.map((x) => x / count));
+    }
+}

package/dist/src/recall.js

@@ -84,3 +84,58 @@ export function recallAndInline(opts, store) {
         empty: toInject.length === 0,
     };
 }
+/**
+ * Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
+ * `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
+ * across repos (HNSW NN over the global PGlite index) when `opts.crossRepo` is
+ * set. The synchronous `recallAndInline` is unchanged and remains the default
+ * per-session path. Inline-window dedupe + token cap (Fix C) apply here too.
+ *
+ * `store` must provide `searchAsync` (the live VectorStore does). Errors fall
+ * back to an empty result — recall is a bonus, never a hard dependency.
+ */
+export async function recallAndInlineAsync(opts, store) {
+    const limit = opts.limit ?? 3;
+    const skip = opts.skipInjected ?? true;
+    const maxTokens = opts.recallMaxTokens ?? 0;
+    const doWindowDedupe = opts.windowDedupe ?? false;
+    const dedupSim = opts.dedupSim ?? 0.9;
+    let hits = [];
+    try {
+        hits = await store.searchAsync(opts.sessionId, opts.query, limit, {
+            crossRepo: opts.crossRepo,
+            repoId: opts.repoId,
+        });
+    }
+    catch {
+        hits = [];
+    }
+    let liveEmbeddings = [];
+    if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+        const embedder = defaultEmbedder();
+        liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+    }
+    const toInject = [];
+    const parts = [];
+    let blockTokens = 0;
+    for (const h of hits) {
+        if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
+            continue;
+        if (doWindowDedupe && liveEmbeddings.length > 0) {
+            const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
+            if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim))
+                continue;
+        }
+        const part = formatRecallBlock([h]);
+        const partTokens = estimateBlockTokens(part);
+        if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
+            break;
+        parts.push(part);
+        toInject.push(h);
+        blockTokens += partTokens;
+        store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
+    }
+    const block = parts.join("\n");
+    const report = toInject.map((h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
+    return { toInject, report, block, empty: toInject.length === 0 };
+}

package/dist/src/store/sqlite.js

@@ -705,6 +705,14 @@ export function hasCheckpoint(sessionId, checkpointId, stateDir = getStateDir())
         .get(normalizeSessionId(sessionId), checkpointId);
     return row !== undefined;
 }
+/** Fetch a single checkpoint by (session, id), or undefined if absent. */
+export function getCheckpoint(sessionId, checkpointId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const row = db
+        .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
+        .get(normalizeSessionId(sessionId), checkpointId);
+    return row ? rowToCheckpoint(row) : undefined;
+}
 /** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
 export function setDedupStatus(checkpointId, sessionId, status, stateDir = getStateDir()) {
     const db = openStore(stateDir);

package/dist/src/store/vectorIndex.js

@@ -0,0 +1,210 @@
+/**
+ * vectorIndex.ts — Slice 2 async vector index (PGlite/pgvector HNSW).
+ *
+ * A REDUNDANT, additive, ASYNC index layered over the synchronous node:sqlite
+ * store (which remains the authoritative source of truth). The sync linear
+ * cosine scan over `embedding_blob` stays the DEFAULT recall path; this index
+ * exists only to provide real cross-repo / cross-session HNSW nearest-neighbor
+ * recall. It is best-effort and non-fatal: any init/write failure degrades to
+ * the sync scan and must NEVER break add(), compaction, or extension load.
+ *
+ * PREVENT-PI-004: PGlite is WASM Postgres — fully local, zero network.
+ *
+ * Index topology (decision 2026-07-15): ONE global PGlite DB, `repo_id` is a
+ * first-class column. `searchAsync(q, k, {repoId?})` → omit repoId for cross-repo
+ * NN, pass repoId to scope to a single repo. The sync store is per-repo (state
+ * dir); this global index is the thing that makes cross-repo recall possible.
+ */
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { mkdirSync } from "node:fs";
+// PGlite + pgvector are script-free WASM (no native build) → survive pi's
+// install-script block. Imported lazily so a missing/broken package degrades
+// gracefully instead of crashing module load.
+import { PGlite } from "@electric-sql/pglite";
+import { vector } from "@electric-sql/pglite-pgvector";
+/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
+export const EMBEDDING_DIM = 512;
+let db;
+let initPromise;
+let disabled = false;
+let warned = false;
+function indexDir() {
+    const override = process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
+    if (override && override.trim() !== "")
+        return override;
+    try {
+        return join(homedir(), ".pi", "mega-compact-vector");
+    }
+    catch {
+        return join("/tmp", ".mega-compact-vector");
+    }
+}
+function logWarn(msg) {
+    // Never throw — degradation is the whole point. One warning per process.
+    if (warned)
+        return;
+    warned = true;
+    try {
+        console.warn(`[mega-compact:vectorIndex] ${msg} (falling back to sync scan)`);
+    }
+    catch {
+        /* ignore */
+    }
+}
+/** Honor the emergency kill-switch. When set, the index is fully disabled. */
+export function isVectorIndexDisabled() {
+    return (disabled ||
+        process.env.MEGACOMPACT_PGLITE_DISABLED === "true" ||
+        process.env.MEGACOMPACT_PGLITE_DISABLED === "1");
+}
+/**
+ * Lazily open + schema-init the global PGlite DB. Idempotent and safe to call
+ * from many places. Returns undefined when disabled/unavailable so callers can
+ * fall back to the synchronous scan. Never throws.
+ */
+export function initVectorIndex() {
+    if (isVectorIndexDisabled())
+        return Promise.resolve(undefined);
+    if (db)
+        return Promise.resolve(db);
+    if (initPromise)
+        return initPromise;
+    initPromise = (async () => {
+        try {
+            const dir = indexDir();
+            mkdirSync(dir, { recursive: true });
+            const pg = await new PGlite({
+                dataDir: dir,
+                extensions: { vector },
+            });
+            await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
+            await pg.exec(`
+        CREATE TABLE IF NOT EXISTS vector_index (
+          repo_id       TEXT NOT NULL,
+          session_id    TEXT NOT NULL,
+          checkpoint_id TEXT NOT NULL,
+          embedding     vector(${EMBEDDING_DIM}) NOT NULL,
+          PRIMARY KEY (repo_id, session_id, checkpoint_id)
+        );
+      `);
+            // HNSW index over cosine distance for fast NN. Created idempotently.
+            await pg.exec("CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);");
+            db = pg;
+            return pg;
+        }
+        catch (err) {
+            disabled = true;
+            logWarn(`init failed: ${err instanceof Error ? err.message : String(err)}`);
+            return undefined;
+        }
+    })();
+    return initPromise;
+}
+function toVectorLiteral(v) {
+    // pgvector text form: [a,b,c]. Guard against NaN/Inf for a clean literal.
+    const parts = v.map((x) => (Number.isFinite(x) ? x : 0));
+    return `[${parts.join(",")}]`;
+}
+/**
+ * Best-effort upsert of one checkpoint embedding into the global index.
+ * Dimension-mismatched vectors (e.g. a BYO embedder with dim ≠ 512) are skipped
+ * rather than corrupting the index. Fire-and-forget: resolved promise only;
+ * callers must NOT await this on the sync path. Never throws.
+ */
+export async function upsertEmbedding(repoId, sessionId, checkpointId, embedding) {
+    if (isVectorIndexDisabled())
+        return;
+    if (!embedding || embedding.length !== EMBEDDING_DIM) {
+        // Dimension guard: skip without corrupting the fixed-dim index.
+        return;
+    }
+    try {
+        const pg = await initVectorIndex();
+        if (!pg)
+            return;
+        const lit = toVectorLiteral(embedding);
+        await pg.query(`INSERT INTO vector_index (repo_id, session_id, checkpoint_id, embedding)
+       VALUES ($1, $2, $3, $4::vector)
+       ON CONFLICT (repo_id, session_id, checkpoint_id)
+       DO UPDATE SET embedding = EXCLUDED.embedding;`, [repoId, sessionId, checkpointId, lit]);
+    }
+    catch (err) {
+        disabled = true;
+        logWarn(`upsert failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+/**
+ * Cross-repo (or single-repo) HNSW nearest-neighbor search. Returns hits sorted
+ * by descending similarity. Never throws — on any failure returns [].
+ */
+export async function searchAsync(query, opts = {}) {
+    if (isVectorIndexDisabled() || !query || query.length !== EMBEDDING_DIM)
+        return [];
+    const k = opts.k ?? 3;
+    const repoId = opts.repoId;
+    try {
+        const pg = await initVectorIndex();
+        if (!pg)
+            return [];
+        const lit = toVectorLiteral(query);
+        const params = [lit, k];
+        let sql = "SELECT repo_id, session_id, checkpoint_id, 1 - (embedding <=> $1::vector) AS score " +
+            "FROM vector_index";
+        if (repoId) {
+            sql += " WHERE repo_id = $3";
+            params.push(repoId);
+        }
+        sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
+        const res = await pg.query(sql, params);
+        return res.rows.map((r) => ({
+            repoId: r.repo_id,
+            sessionId: r.session_id,
+            checkpointId: r.checkpoint_id,
+            score: r.score,
+        }));
+    }
+    catch (err) {
+        disabled = true;
+        logWarn(`search failed: ${err instanceof Error ? err.message : String(err)}`);
+        return [];
+    }
+}
+/** Close the index (test teardown / shutdown). Safe to call when unopened. */
+export async function closeVectorIndex() {
+    if (db) {
+        try {
+            await db.close();
+        }
+        catch {
+            /* ignore */
+        }
+    }
+    db = undefined;
+    initPromise = undefined;
+}
+/**
+ * Rebuild the entire index from the authoritative node:sqlite store. Used for
+ * backfill + DR. `enumerateRepoStateDirs` yields each repo's state dir; we read
+ * its checkpoint embeddings and bulk upsert. Best-effort: counts successes and
+ * skips failures. Returns {upserted, errors}.
+ */
+export async function rebuildFromSqlite(enumerateRepoStateDirs, readCheckpoints) {
+    let upserted = 0;
+    let errors = 0;
+    const pg = await initVectorIndex();
+    if (!pg)
+        return { upserted, errors: 1 };
+    for (const repo of enumerateRepoStateDirs()) {
+        for (const cp of readCheckpoints(repo.stateDir)) {
+            try {
+                await upsertEmbedding(repo.repoId, cp.sessionId, cp.checkpointId, cp.embedding);
+                upserted++;
+            }
+            catch {
+                errors++;
+            }
+        }
+    }
+    return { upserted, errors };
+}