sanook-cli 0.4.0 → 0.5.1

package/dist/search/engine.js

@@ -0,0 +1,203 @@
+// ============================================================================
+// src/search/engine.ts — the search orchestrator (the one module callers use).
+//
+// Implements the degradation ladder as a single search() call:
+//   mode='fts'      → pure BM25 (the always-on floor)
+//   mode='semantic' → cosine over BYOK vectors (full recall)
+//   mode='hybrid'   → BM25 ⊕ cosine ⊕ memory-importance prior, fused by RRF
+//   mode='auto'     → hybrid when vectors are usable, else fts (the smart default)
+//
+// rankSearch() is the PURE core (index + optional vectors + optional query vector
+// in, ranked hits out) so the whole ranking pipeline unit-tests with zero disk
+// and zero network. search() is the thin disk/embedding wrapper: it caches the
+// index by mtime, caches query embeddings in an LRU, resolves a BYOK embedder
+// lazily, and on ANY embedding error degrades to BM25 with a `degraded` flag —
+// search must never throw at the floor.
+// ============================================================================
+import { readFile } from 'node:fs/promises';
+import { appHomePath } from '../brand.js';
+import { bm25Search, termList } from './index-core.js';
+import { rrfFuse } from './fuse.js';
+import { cosineTopK, embedQuery, getEmbedder, loadVectors, vectorsMtimeMs, } from './embed-store.js';
+import { indexMtimeMs, loadIndex } from './store.js';
+const CAND = 60; // candidate pool depth per leg before fusion/limit
+const SNIPPET_WIDTH = 64;
+/** ±width snippet around the first matched query term; falls back to the head for semantic-only hits. */
+function makeSnippet(text, qTerms, width = SNIPPET_WIDTH) {
+    const flat = text.replace(/\s+/g, ' ').trim();
+    const lower = flat.toLowerCase();
+    let pos = -1;
+    for (const t of qTerms) {
+        const i = lower.indexOf(t);
+        if (i >= 0 && (pos < 0 || i < pos))
+            pos = i;
+    }
+    if (pos < 0)
+        return flat.length > width * 2 ? `${flat.slice(0, width * 2).trim()}…` : flat;
+    const start = Math.max(0, pos - width);
+    const end = Math.min(flat.length, pos + width);
+    return `${start > 0 ? '…' : ''}${flat.slice(start, end).trim()}${end < flat.length ? '…' : ''}`;
+}
+/** ids of docs whose source is allowed (or all if no filter). */
+function sourceFilteredIds(index, sources) {
+    if (!sources)
+        return undefined;
+    const out = new Set();
+    for (const m of index.docs.values())
+        if (sources.has(m.source))
+            out.add(m.id);
+    return out;
+}
+/**
+ * PURE ranking core. Given the index, optional vectors, and an optional query
+ * vector, produce ranked hits per the requested mode. No disk, no network.
+ */
+export function rankSearch(index, query, opts = {}, vectors, queryVec) {
+    const mode = opts.mode ?? 'auto';
+    const limit = opts.limit ?? 8;
+    const sources = opts.sources?.length ? new Set(opts.sources) : undefined;
+    const qTerms = [...new Set(termList(query))];
+    const bm25 = bm25Search(index, query, CAND, sources);
+    const bm25Ids = bm25.map((h) => h.id);
+    const semanticPossible = !!(vectors && vectors.dim && queryVec && queryVec.length === vectors.dim);
+    const wantsSemantic = mode === 'semantic' || mode === 'hybrid' || mode === 'auto';
+    // resolve the executed mode + a degraded reason if the request can't be honored
+    let exec;
+    let degraded;
+    if (!wantsSemantic)
+        exec = 'fts';
+    else if (semanticPossible)
+        exec = mode === 'auto' ? 'hybrid' : mode;
+    else {
+        exec = 'fts';
+        if (mode === 'semantic' || mode === 'hybrid')
+            degraded = 'semantic-unavailable';
+    }
+    let orderedIds;
+    if (exec === 'fts') {
+        orderedIds = bm25Ids;
+    }
+    else {
+        const allowed = sourceFilteredIds(index, sources);
+        const cosine = cosineTopK(vectors, queryVec, CAND, allowed).filter((h) => index.docs.has(h.id));
+        const cosineIds = cosine.map((h) => h.id);
+        if (exec === 'semantic') {
+            orderedIds = cosineIds;
+        }
+        else {
+            // hybrid: BM25 ⊕ cosine ⊕ memory-importance prior, fused by rank (scale-free)
+            const priorIds = [...new Set([...bm25Ids, ...cosineIds])]
+                .map((id) => index.docs.get(id))
+                .filter((m) => !!m && m.source === 'memory' && m.importance != null)
+                .sort((a, b) => (b.importance ?? 0) - (a.importance ?? 0))
+                .map((m) => m.id);
+            orderedIds = rrfFuse([
+                { ids: bm25Ids },
+                { ids: cosineIds },
+                { ids: priorIds, weight: 0.4 },
+            ]);
+        }
+    }
+    const snippets = opts.snippets !== false;
+    const hits = [];
+    for (const id of orderedIds.slice(0, limit)) {
+        const m = index.docs.get(id);
+        if (!m)
+            continue;
+        hits.push({
+            id: m.id,
+            source: m.source,
+            title: m.title,
+            path: m.path,
+            noteType: m.noteType,
+            tags: m.tags,
+            score: 0, // rank-based; fused score isn't meaningful cross-mode, so we expose rank order
+            snippet: snippets ? makeSnippet(m.text, qTerms) : '',
+            importance: m.importance,
+        });
+    }
+    return { hits, mode: exec, degraded, total: new Set(orderedIds).size };
+}
+// ---- disk/embedding wrapper (the only impure part) -------------------------
+let indexCache = null;
+let vectorCache = null;
+const queryVecLRU = new Map(); // key = `${tag}\n${query}`
+const LRU_CAP = 100;
+/** cached index load — re-reads only when the on-disk index.json mtime changes. */
+async function cachedIndex() {
+    const mtime = await indexMtimeMs();
+    if (!indexCache || indexCache.mtime !== mtime) {
+        indexCache = { index: (await loadIndex()).index, mtime };
+    }
+    return indexCache.index;
+}
+async function cachedVectors() {
+    const mtime = await vectorsMtimeMs();
+    if (!vectorCache || vectorCache.mtime !== mtime) {
+        vectorCache = { vectors: await loadVectors(), mtime };
+    }
+    return vectorCache.vectors;
+}
+/** read an optional embeddingModel spec from ~/.sanook/config.json. */
+async function configEmbeddingModel() {
+    try {
+        const cfg = JSON.parse(await readFile(appHomePath('config.json'), 'utf8'));
+        return cfg.embeddingModel;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** drop in-process caches (tests + after a reindex in the same process). */
+export function resetSearchCaches() {
+    indexCache = null;
+    vectorCache = null;
+    queryVecLRU.clear();
+}
+/**
+ * The public search entrypoint. Loads the cached index, lazily resolves a BYOK
+ * embedder (opts → env SANOOK_EMBEDDING_MODEL → config → auto-detect), embeds the
+ * query (LRU-cached) only when semantic is wanted AND a usable same-tag vector set
+ * exists, then delegates to rankSearch. Any embedding failure degrades to BM25.
+ */
+export async function search(query, opts = {}) {
+    const index = await cachedIndex();
+    const mode = opts.mode ?? 'auto';
+    if (mode === 'fts')
+        return rankSearch(index, query, opts);
+    const spec = opts.embeddingModel ?? process.env.SANOOK_EMBEDDING_MODEL ?? (await configEmbeddingModel());
+    const embedder = getEmbedder(spec);
+    if (!embedder) {
+        const res = rankSearch(index, query, opts);
+        if (mode === 'semantic' || mode === 'hybrid')
+            res.degraded = 'no-embedder';
+        return res;
+    }
+    const vectors = await cachedVectors();
+    // a model change (different tag) invalidates the cache → behave as no-vectors until reindex
+    if (!vectors.dim || vectors.tag !== embedder.tag) {
+        const res = rankSearch(index, query, opts);
+        res.degraded = vectors.dim ? 'embedding-model-changed' : 'no-vectors';
+        return mode === 'auto' ? { ...res, degraded: undefined } : res;
+    }
+    let queryVec;
+    try {
+        const key = `${embedder.tag}\n${query}`;
+        const cached = queryVecLRU.get(key);
+        if (cached) {
+            queryVec = cached;
+        }
+        else {
+            queryVec = await embedQuery(embedder, query);
+            queryVecLRU.set(key, queryVec);
+            if (queryVecLRU.size > LRU_CAP)
+                queryVecLRU.delete(queryVecLRU.keys().next().value);
+        }
+    }
+    catch {
+        const res = rankSearch(index, query, opts); // embedding failed mid-query → BM25 floor
+        res.degraded = 'semantic-unavailable';
+        return res;
+    }
+    return rankSearch(index, query, opts, vectors, queryVec);
+}

package/dist/search/fuse.js

@@ -0,0 +1,35 @@
+// ============================================================================
+// src/search/fuse.ts — Reciprocal Rank Fusion (RRF).
+//
+// arra-oracle blends results with a hand-tuned linear formula
+// (fts*0.7 + vec*0.65 + 0.12*overlap) that mixes BM25 magnitudes with cosine
+// distances — two scales that are not comparable, so the weights are fragile and
+// corpus-dependent. RRF sidesteps the whole problem: it fuses on RANK, not score,
+// so a document's contribution depends only on where it placed in each list, not
+// on the (incomparable) raw numbers. A doc that ranks well in two lists naturally
+// sums two reciprocals and outranks a doc strong in only one. k=60 is the
+// standard Cormack et al. constant. Pure, deterministic, parameter-light.
+// ============================================================================
+const RRF_K = 60;
+/**
+ * Fuse N ranked id-lists into a single score map (higher = better).
+ * score(d) = Σ_lists weight / (k + rank_in_list(d)), rank 0-based.
+ */
+export function rrf(lists, k = RRF_K) {
+    const scores = new Map();
+    for (const list of lists) {
+        const w = list.weight ?? 1;
+        for (let rank = 0; rank < list.ids.length; rank++) {
+            const id = list.ids[rank];
+            scores.set(id, (scores.get(id) ?? 0) + w / (k + rank));
+        }
+    }
+    return scores;
+}
+/** RRF then sort → fused id list (best first), deterministic tie-break by id. */
+export function rrfFuse(lists, limit, k = RRF_K) {
+    const scores = rrf(lists, k);
+    const out = [...scores.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+    const ids = out.map(([id]) => id);
+    return limit == null ? ids : ids.slice(0, limit);
+}

package/dist/search/index-core.js

@@ -0,0 +1,187 @@
+// ============================================================================
+// src/search/index-core.ts — the zero-dependency search FLOOR.
+//
+// A pure-TS inverted index with REAL BM25 (k1=1.2, b=0.75, genuine corpus-stat
+// IDF via df/N). No SQLite, no Bun, no native binary, no network — it works the
+// instant a corpus exists, on any OS Node 22 runs on. This is deliberately NOT
+// node:sqlite FTS5: that is experimental, its FTS5 build is not guaranteed across
+// platforms, and it reintroduces a quasi-native dependency that fights the
+// zero-config/portability contract. A few hundred lines of TS give us a real
+// ranking model that FTS5's bm25() only approximates without true global IDF.
+//
+// Tokenization REUSES memory-store.ts normalize() (the canonical, Thai-safe,
+// stopword-aware tokenizer) so memory matching and search matching never drift.
+//
+// addDoc/removeDoc MUTATE the index in place and return it — an index over a
+// large vault must not deep-copy its postings map on every chunk (that is the
+// one place we diverge from memory-store's small-array immutability). bm25Search
+// is pure and read-only. Re-adding the same doc id replaces its postings, so the
+// index can never accumulate duplicate postings the way arra's FTS5
+// delete-then-insert can drift.
+// ============================================================================
+import { normalize } from '../memory-store.js';
+export const SEARCH_SOURCES = ['memory', 'vault', 'session', 'skill'];
+/** BM25 params — Robertson/Spärck-Jones defaults; title terms get weighted tf. */
+const K1 = 1.2;
+const B = 0.75;
+const TITLE_BOOST = 2; // a term in a doc's title counts this many times toward tf
+const WORD_SEG = new Intl.Segmenter(undefined, { granularity: 'word' });
+export const INDEX_VERSION = 1;
+export function emptyIndex() {
+    return { version: INDEX_VERSION, postings: new Map(), docs: new Map(), totalDl: 0 };
+}
+/**
+ * Ordered tokens WITH repeats — BM25 needs term frequencies, so unlike
+ * memory-store's tokens() (a deduped Set) we keep counts. Builds on the SAME
+ * canonical normalize() (lowercase, punctuation→space, Thai preserved), then
+ * segments with Intl.Segmenter at word granularity so Thai (which has no spaces)
+ * splits into real words instead of one coarse blob, giving BM25 genuine Thai
+ * term frequencies while preserving repeats.
+ */
+export function termList(text) {
+    const out = [];
+    for (const seg of WORD_SEG.segment(normalize(text))) {
+        if (!seg.isWordLike)
+            continue;
+        const token = seg.segment.trim();
+        if (token.length > 1)
+            out.push(token);
+    }
+    return out;
+}
+/** combined term-frequency map for a doc, with title terms weighted, + the token length. */
+function termFreqs(title, text) {
+    const tf = new Map();
+    const body = termList(text);
+    const head = termList(title);
+    for (const t of body)
+        tf.set(t, (tf.get(t) ?? 0) + 1);
+    for (const t of head)
+        tf.set(t, (tf.get(t) ?? 0) + TITLE_BOOST);
+    return { tf, dl: body.length + head.length };
+}
+/** add (or REPLACE, if id already present) a document. Mutates + returns idx. */
+export function addDoc(idx, doc) {
+    if (idx.docs.has(doc.id))
+        removeDoc(idx, doc.id); // replace → no posting creep
+    const { tf, dl } = termFreqs(doc.title, doc.text);
+    const meta = {
+        id: doc.id,
+        source: doc.source,
+        title: doc.title,
+        text: doc.text,
+        path: doc.path,
+        noteType: doc.noteType,
+        tags: doc.tags ?? [],
+        links: doc.links ?? [],
+        importance: doc.importance,
+        updatedMs: doc.updatedMs,
+        dl,
+    };
+    idx.docs.set(doc.id, meta);
+    idx.totalDl += dl;
+    for (const [term, freq] of tf) {
+        const plist = idx.postings.get(term);
+        if (plist)
+            plist.push({ docId: doc.id, tf: freq });
+        else
+            idx.postings.set(term, [{ docId: doc.id, tf: freq }]);
+    }
+    return idx;
+}
+/** remove a document and all its postings. Mutates + returns idx. No-op if absent. */
+export function removeDoc(idx, id) {
+    const meta = idx.docs.get(id);
+    if (!meta)
+        return idx;
+    const { tf } = termFreqs(meta.title, meta.text);
+    for (const term of tf.keys()) {
+        const plist = idx.postings.get(term);
+        if (!plist)
+            continue;
+        const next = plist.filter((p) => p.docId !== id);
+        if (next.length)
+            idx.postings.set(term, next);
+        else
+            idx.postings.delete(term);
+    }
+    idx.totalDl -= meta.dl;
+    idx.docs.delete(id);
+    return idx;
+}
+/**
+ * BM25 ranking — pure, read-only. Genuine IDF from df/N (the always-positive
+ * BM25+ form ln(1 + (N-df+0.5)/(df+0.5))), length-normalized by avgdl. Optional
+ * source allow-list keeps cross-corpus queries cheap. Deterministic tie-break by id.
+ */
+export function bm25Search(idx, query, limit = 50, sources) {
+    const n = idx.docs.size;
+    if (!n)
+        return [];
+    const avgdl = idx.totalDl / n || 1;
+    const qTerms = [...new Set(termList(query))];
+    if (!qTerms.length)
+        return [];
+    const scores = new Map();
+    for (const term of qTerms) {
+        const plist = idx.postings.get(term);
+        if (!plist)
+            continue;
+        const df = plist.length;
+        const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
+        for (const p of plist) {
+            const meta = idx.docs.get(p.docId);
+            if (!meta)
+                continue;
+            if (sources && !sources.has(meta.source))
+                continue;
+            const denom = p.tf + K1 * (1 - B + B * (meta.dl / avgdl));
+            const contrib = idf * ((p.tf * (K1 + 1)) / denom);
+            scores.set(p.docId, (scores.get(p.docId) ?? 0) + contrib);
+        }
+    }
+    return [...scores.entries()]
+        .map(([id, score]) => ({ id, score }))
+        .sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
+        .slice(0, limit);
+}
+/** remove every doc of a given source (used to refresh the live memory/session/skill corpora). Returns count removed. */
+export function removeSource(idx, source) {
+    const ids = [];
+    for (const m of idx.docs.values())
+        if (m.source === source)
+            ids.push(m.id);
+    for (const id of ids)
+        removeDoc(idx, id);
+    return ids.length;
+}
+export function indexStats(idx) {
+    const bySource = {};
+    for (const m of idx.docs.values())
+        bySource[m.source] = (bySource[m.source] ?? 0) + 1;
+    return {
+        docs: idx.docs.size,
+        terms: idx.postings.size,
+        bySource,
+        avgdl: idx.docs.size ? idx.totalDl / idx.docs.size : 0,
+    };
+}
+export function indexToJSON(idx) {
+    const postings = {};
+    for (const [term, plist] of idx.postings)
+        postings[term] = plist;
+    return { version: idx.version, totalDl: idx.totalDl, postings, docs: [...idx.docs.values()] };
+}
+export function indexFromJSON(raw) {
+    const obj = raw;
+    if (!obj || obj.version !== INDEX_VERSION || !obj.postings || !Array.isArray(obj.docs)) {
+        return emptyIndex(); // unknown/old shape degrades to empty rather than throwing
+    }
+    const idx = emptyIndex();
+    idx.totalDl = obj.totalDl ?? 0;
+    for (const [term, plist] of Object.entries(obj.postings))
+        idx.postings.set(term, plist);
+    for (const m of obj.docs)
+        idx.docs.set(m.id, m);
+    return idx;
+}

package/dist/search/indexer.js

@@ -0,0 +1,241 @@
+// ============================================================================
+// src/search/indexer.ts — incremental, O(delta) vault indexer.
+//
+// Beats arra-oracle's indexer on three axes:
+//   1. NO directory convention. arra requires a `ψ/memory/…` tree; we index the
+//      user's EXISTING second-brain vault via getBrainPath(), any layout.
+//   2. TRUE incremental. arra full-re-indexes every pass (guarded only by a >50%
+//      delete abort). We diff a per-file manifest: an unchanged file costs ONE
+//      stat(); only changed files are read+sha256+re-chunked; deleted files have
+//      their chunks evicted precisely (manifest stores each file's chunk ids).
+//   3. ONE unified surface. Vault chunks, active memory Facts, recent session
+//      turns, and skills all land in the SAME ranked index — the unification arra
+//      never did (its memory store and search index use divorced formats).
+//
+// The file-walk is injected (VaultFS) so the core logic unit-tests against an
+// in-memory fs + clock with zero disk, exactly like memory-store.ts.
+// ============================================================================
+import { createHash } from 'node:crypto';
+import { readFile, readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { appHomePath } from '../brand.js';
+import { getBrainPath } from '../memory.js';
+import { loadSkills } from '../skills.js';
+import { activeFacts, effImportance, loadStore } from '../memory-store.js';
+import { chunkMarkdown } from './chunk.js';
+import { addDoc, removeDoc, removeSource } from './index-core.js';
+import { loadIndex, saveIndex } from './store.js';
+/** strip a .md path to a human title fallback when a chunk has no heading. */
+function fileTitle(rel) {
+    return (rel.split('/').pop() ?? rel).replace(/\.md$/i, '');
+}
+/**
+ * Incremental vault pass. Mutates `index`, returns the NEXT manifest + a diff.
+ * Pure w.r.t. the injected fs/clock — no disk access of its own.
+ */
+export async function indexVaultFiles(index, manifest, fs) {
+    const next = {};
+    const diff = { added: 0, updated: 0, removed: 0, skipped: 0 };
+    const paths = await fs.listMarkdown();
+    const seenExisting = new Set();
+    for (const rel of paths) {
+        const fp = await fs.fingerprint(rel);
+        if (!fp)
+            continue; // vanished between listing and stat → treat as deletion below
+        seenExisting.add(rel);
+        const prev = manifest[rel];
+        // cheap path: mtime + size unchanged ⇒ skip without reading the file
+        if (prev && prev.mtimeMs === fp.mtimeMs && prev.size === fp.size) {
+            next[rel] = prev;
+            diff.skipped++;
+            continue;
+        }
+        const content = await fs.read(rel);
+        const sha = fs.hash(content);
+        // touched but content identical (mtime bumped by a sync) ⇒ refresh fingerprint, keep chunks
+        if (prev && prev.sha === sha) {
+            next[rel] = { ...prev, mtimeMs: fp.mtimeMs, size: fp.size };
+            diff.skipped++;
+            continue;
+        }
+        // changed or new ⇒ evict old chunks, re-chunk, re-add
+        if (prev)
+            for (const id of prev.ids)
+                removeDoc(index, id);
+        const parsed = chunkMarkdown(rel, content);
+        const title0 = fileTitle(rel);
+        const ids = [];
+        for (const c of parsed.chunks) {
+            const doc = {
+                id: c.id,
+                source: 'vault',
+                title: c.heading || title0,
+                text: c.text,
+                path: rel,
+                noteType: parsed.frontmatter.noteType,
+                tags: parsed.frontmatter.tags,
+                links: parsed.links,
+                updatedMs: fp.mtimeMs,
+            };
+            addDoc(index, doc);
+            ids.push(c.id);
+        }
+        next[rel] = { mtimeMs: fp.mtimeMs, size: fp.size, sha, ids };
+        if (prev)
+            diff.updated++;
+        else
+            diff.added++;
+    }
+    // deletions: present last time, absent now ⇒ evict their chunks
+    for (const rel of Object.keys(manifest)) {
+        if (seenExisting.has(rel))
+            continue;
+        for (const id of manifest[rel].ids)
+            removeDoc(index, id);
+        diff.removed++;
+    }
+    return { manifest: next, diff };
+}
+/** refresh the live memory corpus: drop old memory docs, re-add active Facts with an importance prior. */
+export function foldFacts(index, facts, now) {
+    removeSource(index, 'memory');
+    const searchable = facts.filter((f) => f.status === 'active' && f.tier !== 'inbox');
+    for (const f of searchable) {
+        addDoc(index, {
+            id: f.id, // memory-store deriveId — stable, dedups against itself
+            source: 'memory',
+            title: '',
+            text: f.text,
+            noteType: f.noteType,
+            tags: f.tags,
+            importance: effImportance(f, now),
+            updatedMs: f.updated,
+        });
+    }
+    return searchable.length;
+}
+/** refresh the session corpus (first-user-message per recent session). */
+export function foldSessions(index, sessions) {
+    removeSource(index, 'session');
+    for (const s of sessions) {
+        addDoc(index, { id: s.id, source: 'session', title: '', text: s.text, updatedMs: s.updatedMs });
+    }
+    return sessions.length;
+}
+/** refresh the skill corpus (name + description + whenToUse). */
+export function foldSkills(index, skills) {
+    removeSource(index, 'skill');
+    for (const s of skills) {
+        addDoc(index, { id: s.id, source: 'skill', title: s.name, text: s.text });
+    }
+    return skills.length;
+}
+// ---- real-filesystem wiring ------------------------------------------------
+const IGNORE_DIRS = new Set([
+    'node_modules', 'dist', 'build', 'coverage', '.next', '.cache', '.git',
+    '.obsidian', 'vendor', '.turbo', '.vercel',
+]);
+/** node:fs implementation of VaultFS — recursive .md walk with the default-ignore set. */
+export function nodeVaultFS(root) {
+    async function walk(dir, rel, out) {
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (e.isDirectory()) {
+                if (IGNORE_DIRS.has(e.name) || e.name.startsWith('.'))
+                    continue;
+                await walk(join(dir, e.name), rel ? `${rel}/${e.name}` : e.name, out);
+            }
+            else if (e.isFile() && e.name.toLowerCase().endsWith('.md')) {
+                out.push(rel ? `${rel}/${e.name}` : e.name);
+            }
+        }
+    }
+    return {
+        async listMarkdown() {
+            const out = [];
+            await walk(root, '', out);
+            return out.sort();
+        },
+        async fingerprint(relPath) {
+            try {
+                const s = await stat(join(root, relPath));
+                return { mtimeMs: s.mtimeMs, size: s.size };
+            }
+            catch {
+                return null;
+            }
+        },
+        read: (relPath) => readFile(join(root, relPath), 'utf8'),
+        hash: (content) => createHash('sha256').update(content).digest('hex'),
+    };
+}
+const SESSIONS_DIR = appHomePath('sessions');
+/** load first-user-message of the most recent sessions (bounded) for the session corpus. */
+export async function loadRecentSessions(limit = 60) {
+    const out = [];
+    let candidates;
+    try {
+        const files = (await readdir(SESSIONS_DIR)).filter((f) => f.endsWith('.json'));
+        const withStats = await Promise.all(files.map(async (file) => {
+            const full = join(SESSIONS_DIR, file);
+            try {
+                return { file, full, mtimeMs: (await stat(full)).mtimeMs };
+            }
+            catch {
+                return null;
+            }
+        }));
+        candidates = withStats
+            .filter((c) => c !== null)
+            .sort((a, b) => b.mtimeMs - a.mtimeMs || b.file.localeCompare(a.file))
+            .slice(0, limit);
+    }
+    catch {
+        return out;
+    }
+    for (const { file, full, mtimeMs } of candidates) {
+        try {
+            const s = JSON.parse(await readFile(full, 'utf8'));
+            const firstUser = (s.messages ?? []).find((m) => m.role === 'user');
+            const text = typeof firstUser?.content === 'string' ? firstUser.content : '';
+            if (!text.trim())
+                continue;
+            out.push({ id: `sess:${s.id ?? file}`, text: text.slice(0, 2000), updatedMs: mtimeMs });
+        }
+        catch {
+            /* skip a corrupt session file */
+        }
+    }
+    return out;
+}
+/**
+ * Full incremental reindex: vault (via getBrainPath) + memory + sessions + skills,
+ * persisted atomically. Returns a change report. This is what `sanook index` and
+ * the MCP `sanook_index` tool call.
+ */
+export async function reindex(now = Date.now()) {
+    const { index, manifest } = await loadIndex();
+    let diff = { added: 0, updated: 0, removed: 0, skipped: 0 };
+    let nextManifest = manifest;
+    const brain = await getBrainPath();
+    if (brain) {
+        const r = await indexVaultFiles(index, manifest, nodeVaultFS(brain));
+        nextManifest = r.manifest;
+        diff = r.diff;
+    }
+    const memory = foldFacts(index, activeFacts(await loadStore(now)), now);
+    const sessions = foldSessions(index, await loadRecentSessions());
+    const skills = foldSkills(index, (await loadSkills()).map((s) => ({
+        id: `skill:${s.name}`,
+        name: s.name,
+        text: `${s.description} ${s.whenToUse ?? ''}`.trim(),
+    })));
+    await saveIndex(index, nextManifest);
+    return { ...diff, memory, sessions, skills, vaultPath: brain ?? null };
+}