sensemaking 0.15.4 → 0.16.0

package/dist/esm/commands/related.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedEnabled, resolveSearch } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { embedPending, hasEmbedding, modelPresent, similarNotes } from '../features/embed.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings } from './scope.ts';\n\n// Notes most similar by cosine, excluding self and everything already linked either way.\n// Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.\nexport async function relatedNotes(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  const paths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const path = resolveNote(paths, pathArg);\n\n  const outbound = (db.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src').all(path) as Array<{ dst: string }>).map((r) => r.dst);\n  const backlinks = (db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst').all(path) as Array<{ src: string }>).map((r) => r.src);\n  const exclude = new Set([path, ...outbound, ...backlinks]);\n\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause. An empty table then means one thing: nothing near in meaning that this\n  // note does not already link to, which is a real answer.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json, then run `sense download` (search works without it, on words and links)');\n  }\n  // search gates on the same flag (see wantsVectors above); reading it here too keeps\n  // `semantic: false` meaning one thing. Without this, an overlapping semantic-on preset's\n  // vectors would answer for a scope that declined them.\n  if (!effective.semantic) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" sets \"semantic\": false, so this scope has no vectors and related has no other signal; search it instead (words and links), or set semantic back on for that preset`);\n  }\n  if (!modelPresent(cfg)) {\n    throw new SenseError('EMBED_MODEL_MISSING', 'related ranks notes by meaning, so it needs the embedding model, which is not downloaded; run `sense download` (search still works without it, on words and links)');\n  }\n  const allowed = scopedPaths(db, cfg, overrides);\n  // Top up pending rows before the seed check, or a fresh index reports every note as\n  // having no indexed text until some search has run.\n  await embedPending(db, cfg, cfg.baseDir);\n  if (!hasEmbedding(db, path)) {\n    throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n  }\n  if (!scopeHasEmbeddings(db, cfg, allowed)) return [];\n  return similarNotes(db, cfg, path, { exclude, allowed, k });\n}\n"],"names":["embedEnabled","resolveSearch","SenseError","embedPending","hasEmbedding","modelPresent","similarNotes","resolveNote","scopedPaths","scopeHasEmbeddings","relatedNotes","db","cfg","pathArg","overrides","k","paths","prepare","all","map","r","path","outbound","dst","backlinks","src","exclude","Set","effective","semantic","presetName","allowed","baseDir"],"mappings":"AAEA,SAASA,YAAY,EAAEC,aAAa,QAAQ,qBAAqB;AACjE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,YAAY,EAAEC,YAAY,EAAEC,YAAY,QAAQ,uBAAuB;AAC9F,SAASC,WAAW,QAAQ,YAAY;AACxC,SAASC,WAAW,EAAEC,kBAAkB,QAAQ,aAAa;AAE7D,yFAAyF;AACzF,sFAAsF;AACtF,OAAO,eAAeC,aAAaC,EAAgB,EAAEC,GAAmB,EAAEC,OAAe,EAAEC,SAA0B,EAAEC,CAAS;IAC9H,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,kCAAkCC,GAAG,GAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAC/G,MAAMA,OAAOd,YAAYS,OAAOH;IAEhC,MAAMS,WAAW,AAACX,GAAGM,OAAO,CAAC,mFAAmFC,GAAG,CAACG,MAAiCF,GAAG,CAAC,CAACC,IAAMA,EAAEG,GAAG;IACrK,MAAMC,YAAY,AAACb,GAAGM,OAAO,CAAC,+DAA+DC,GAAG,CAACG,MAAiCF,GAAG,CAAC,CAACC,IAAMA,EAAEK,GAAG;IAClJ,MAAMC,UAAU,IAAIC,IAAI;QAACN;WAASC;WAAaE;KAAU;IAEzD,yFAAyF;IACzF,2FAA2F;IAC3F,yDAAyD;IACzD,MAAMI,YAAY3B,cAAcW,KAAKE;IACrC,IAAI,CAACd,aAAaY,MAAM;QACtB,MAAM,IAAIV,WAAW,kBAAkB;IACzC;IACA,oFAAoF;IACpF,yFAAyF;IACzF,uDAAuD;IACvD,IAAI,CAAC0B,UAAUC,QAAQ,EAAE;QACvB,MAAM,IAAI3B,WAAW,uBAAuB,CAAC,QAAQ,EAAE0B,UAAUE,UAAU,CAAC,oKAAoK,CAAC;IACnP;IACA,IAAI,CAACzB,aAAaO,MAAM;QACtB,MAAM,IAAIV,WAAW,uBAAuB;IAC9C;IACA,MAAM6B,UAAUvB,YAAYG,IAAIC,KAAKE;IACrC,oFAAoF;IACpF,oDAAoD;IACpD,MAAMX,aAAaQ,IAAIC,KAAKA,IAAIoB,OAAO;IACvC,IAAI,CAAC5B,aAAaO,IAAIU,OAAO;QAC3B,MAAM,IAAInB,WAAW,qBAAqB,GAAGmB,KAAK,4GAA4G,CAAC;IACjK;IACA,IAAI,CAACZ,mBAAmBE,IAAIC,KAAKmB,UAAU,OAAO,EAAE;IACpD,OAAOzB,aAAaK,IAAIC,KAAKS,MAAM;QAAEK;QAASK;QAAShB;IAAE;AAC3D"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedConfig, embedEnabled, resolveSearch } from '../config/index.ts';\nimport { embedPending, hasEmbedding, similarNotes } from '../embed/query.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings } from './scope.ts';\n\n// Notes most similar by cosine, excluding self and everything already linked either way.\n// Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.\nexport async function relatedNotes(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  const paths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const path = resolveNote(paths, pathArg);\n\n  const outbound = (db.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src').all(path) as Array<{ dst: string }>).map((r) => r.dst);\n  const backlinks = (db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst').all(path) as Array<{ src: string }>).map((r) => r.src);\n  const exclude = new Set([path, ...outbound, ...backlinks]);\n\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause. An empty table then means one thing: nothing near in meaning that this\n  // note does not already link to, which is a real answer.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');\n  }\n  // search gates on the same signal (see wantsVectors above); reading it here too keeps a\n  // preset's declared signals meaning one thing. Without this, an overlapping vectors-on\n  // preset's vectors would answer for a scope that declined them.\n  if (effective.signals.vectors === undefined) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n  }\n  // A downloadable HF id proceeds -- embedPending's getProvider call fetches it lazily on\n  // consent. Only a local path with missing files can never fetch itself.\n  const e = embedConfig(cfg); // embedEnabled(cfg) above guarantees this is set\n  if (localModelMissing(e)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);\n  }\n  const allowed = scopedPaths(db, cfg, overrides);\n  // Top up pending rows before the seed check, or a fresh index reports every note as\n  // having no indexed text until some search has run.\n  await embedPending(db, cfg, cfg.baseDir);\n  if (!hasEmbedding(db, path)) {\n    throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n  }\n  if (!scopeHasEmbeddings(db, cfg, allowed)) return [];\n  return similarNotes(db, cfg, path, { exclude, allowed, k });\n}\n"],"names":["embedConfig","embedEnabled","resolveSearch","embedPending","hasEmbedding","similarNotes","localModelMissing","MODEL_FILENAMES","SenseError","resolveNote","scopedPaths","scopeHasEmbeddings","relatedNotes","db","cfg","pathArg","overrides","k","paths","prepare","all","map","r","path","outbound","dst","backlinks","src","exclude","Set","effective","signals","vectors","undefined","presetName","e","model","allowed","baseDir"],"mappings":"AAEA,SAASA,WAAW,EAAEC,YAAY,EAAEC,aAAa,QAAQ,qBAAqB;AAC9E,SAASC,YAAY,EAAEC,YAAY,EAAEC,YAAY,QAAQ,oBAAoB;AAC7E,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,WAAW,QAAQ,YAAY;AACxC,SAASC,WAAW,EAAEC,kBAAkB,QAAQ,aAAa;AAE7D,yFAAyF;AACzF,sFAAsF;AACtF,OAAO,eAAeC,aAAaC,EAAgB,EAAEC,GAAmB,EAAEC,OAAe,EAAEC,SAA0B,EAAEC,CAAS;IAC9H,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,kCAAkCC,GAAG,GAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAC/G,MAAMA,OAAOd,YAAYS,OAAOH;IAEhC,MAAMS,WAAW,AAACX,GAAGM,OAAO,CAAC,mFAAmFC,GAAG,CAACG,MAAiCF,GAAG,CAAC,CAACC,IAAMA,EAAEG,GAAG;IACrK,MAAMC,YAAY,AAACb,GAAGM,OAAO,CAAC,+DAA+DC,GAAG,CAACG,MAAiCF,GAAG,CAAC,CAACC,IAAMA,EAAEK,GAAG;IAClJ,MAAMC,UAAU,IAAIC,IAAI;QAACN;WAASC;WAAaE;KAAU;IAEzD,yFAAyF;IACzF,2FAA2F;IAC3F,yDAAyD;IACzD,MAAMI,YAAY5B,cAAcY,KAAKE;IACrC,IAAI,CAACf,aAAaa,MAAM;QACtB,MAAM,IAAIN,WAAW,kBAAkB;IACzC;IACA,wFAAwF;IACxF,uFAAuF;IACvF,gEAAgE;IAChE,IAAIsB,UAAUC,OAAO,CAACC,OAAO,KAAKC,WAAW;QAC3C,MAAM,IAAIzB,WAAW,uBAAuB,CAAC,QAAQ,EAAEsB,UAAUI,UAAU,CAAC,uKAAuK,CAAC;IACtP;IACA,wFAAwF;IACxF,wEAAwE;IACxE,MAAMC,IAAInC,YAAYc,MAAM,iDAAiD;IAC7E,IAAIR,kBAAkB6B,IAAI;QACxB,MAAM,IAAI3B,WAAW,uBAAuB,CAAC,2FAA2F,EAAE2B,EAAEC,KAAK,CAAC,aAAa,EAAE7B,gBAAgB,sGAAsG,CAAC;IAC1R;IACA,MAAM8B,UAAU3B,YAAYG,IAAIC,KAAKE;IACrC,oFAAoF;IACpF,oDAAoD;IACpD,MAAMb,aAAaU,IAAIC,KAAKA,IAAIwB,OAAO;IACvC,IAAI,CAAClC,aAAaS,IAAIU,OAAO;QAC3B,MAAM,IAAIf,WAAW,qBAAqB,GAAGe,KAAK,4GAA4G,CAAC;IACjK;IACA,IAAI,CAACZ,mBAAmBE,IAAIC,KAAKuB,UAAU,OAAO,EAAE;IACpD,OAAOhC,aAAaQ,IAAIC,KAAKS,MAAM;QAAEK;QAASS;QAASpB;IAAE;AAC3D"}

package/dist/esm/commands/search.js

@@ -1,20 +1,14 @@
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { anyPresetEmbeds, contentTokenize, featureEnabled, resolveSearch } from '../config/index.js';
+import { contentTokenize, embedConfig, featureEnabled, resolveSearch } from '../config/index.js';
+import { localModelMissing, MODEL_FILENAMES } from '../embed/store.js';
 import { SenseError } from '../errors.js';
-import { modelPresent, semanticCandidates } from '../features/embed.js';
-import { linkEdges } from '../features/index.js';
-import { personalizedRank } from '../graph.js';
 import { searchError } from '../search-error.js';
 import { segmentMatch } from '../segment.js';
 import { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.js';
-// Mirrors the main columns onto the sidecars, so a title match found through title_seg ranks
-// like a title match found through title, not like a body match.
-const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
-const RRF_K = 60;
+import { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.js';
 // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
 // (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.
-const SNIPPET_BOUND = 16384;
 const EXCERPT_WINDOW = 160;
 // Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt
 // scan matches the same words the query matched on, not FTS5 syntax.
@@ -107,12 +101,12 @@ function lineRangeFor(db, path, line) {
     const row = db.prepare('SELECT start_line, end_line FROM sections WHERE "path" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line);
     return row ? `L${row.start_line}-${row.end_line}` : null;
 }
-// BM25 + link expansion + vectors, fused by reciprocal rank; `via` names the signal per row.
+// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.
 // `opts` arrives already resolved (config.ts:resolveSearch).
 export async function search(db, cfg, terms, opts = {}) {
     var _hits_get, _chunkLines_get, _chunkSimilarity_get;
     const effective = resolveSearch(cfg, opts);
-    const { k } = effective;
+    const { k, signals } = effective;
     const allPaths = db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path);
     const scopePaths = rawScope(db, cfg, opts, allPaths);
     const scopeActive = scopePaths.size < allPaths.length;
@@ -126,11 +120,14 @@ export async function search(db, cfg, terms, opts = {}) {
         throw searchError(err, terms, effective.where);
     }
     const fetch = Math.max(k * 3, 30);
-    // Asking for vectors without a model is a misconfiguration, not a mode: degrading would make
-    // the same search answer differently before and after a download. semantic:false never asks.
-    const wantsVectors = effective.semantic && anyPresetEmbeds(cfg);
-    if (wantsVectors && !modelPresent(cfg)) {
-        throw new SenseError('EMBED_MODEL_MISSING', `preset "${effective.presetName}" searches with vectors, but the embedding model is not available; run \`sense download\`, or set "semantic": false on that preset to search on words and links`);
+    // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a
+    // local path with missing files errors here, since nothing will ever fetch it for itself.
+    const wantsVectors = signals.vectors !== undefined;
+    if (wantsVectors) {
+        const e = embedConfig(cfg); // validate.ts guarantees this is set whenever "vectors" is declared
+        if (localModelMissing(e)) {
+            throw new SenseError('EMBED_MODEL_MISSING', `preset "${effective.presetName}" searches with vectors, but the local model path "${e.model}" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop "vectors" from that preset's signals to search without them`);
+        }
     }
     const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);
     // Terms pass to MATCH as written, with one transform: a run of text whose language marks no
@@ -152,18 +149,17 @@ export async function search(db, cfg, terms, opts = {}) {
     // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.
     if (scopeActive) materializeScope(db, '_search_scope', scopePaths);
     const scopeCond = scopeActive ? `AND content.path IN (SELECT "path" FROM _search_scope)` : '';
-    // Docs past SNIPPET_BOUND skip snippet() entirely -- SQLite
-    // short-circuits the untaken CASE branch, so it's never invoked on them.
-    // Column 2 (text), not -1 (best column): -1 could surface a _seg sidecar as the excerpt,
-    // which is machine-spaced and not what the author wrote. A row that matches only in a
-    // sidecar gets an unhighlighted text excerpt below instead -- a stated cost.
-    const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;
     const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;
-    let matchRows;
-    try {
-        matchRows = db.prepare(matchSql).all(query);
-    } catch (err) {
-        throw searchError(err, terms, scope);
+    const candidates = new Map();
+    let matchRows = [];
+    // A query that rewrites to nothing (e.g. only unspaced-script punctuation) has no
+    // searchable words: zero lexical rows, and the other signals still compose.
+    if (signals.words !== undefined && query.trim() !== '') {
+        try {
+            matchRows = wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, signals.words);
+        } catch (err) {
+            throw searchError(err, terms, scope);
+        }
     }
     const hits = new Map(matchRows.map((r)=>[
             r.path,
@@ -172,58 +168,11 @@ export async function search(db, cfg, terms, opts = {}) {
     // hit === null here means the bound suppressed snippet(), not "no match" -- distinguish
     // from via='link' rows (never in matchRows, so absent from this set) below.
     const oversized = new Set(matchRows.filter((r)=>r.hit === null).map((r)=>r.path));
-    const candidates = new Map();
-    matchRows.forEach((r, i)=>{
-        candidates.set(r.path, {
-            score: 1 / (RRF_K + i),
-            via: 'match'
-        });
-    });
-    const edges = featureEnabled(cfg, 'links') && matchRows.length > 0 ? linkEdges(db) : [];
-    if (edges.length > 0) {
-        // Gates the label only, not the score: restart mass ranks every seed without an incident
-        // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).
-        const linked = new Set(edges.flat());
-        const seeds = new Map(matchRows.map((r, i)=>[
-                r.path,
-                1 / (i + 1)
-            ]));
-        const ranked = [
-            ...personalizedRank(allPaths, edges, seeds)
-        ].filter(([path, score])=>score > 1e-9 && allowedPaths.has(path)).sort((a, b)=>b[1] - a[1]).slice(0, fetch);
-        ranked.forEach(([path], i)=>{
-            const existing = candidates.get(path);
-            if (existing) {
-                existing.score += 1 / (RRF_K + i);
-                if (linked.has(path)) existing.via = 'match+link';
-            } else {
-                candidates.set(path, {
-                    score: 1 / (RRF_K + i),
-                    via: 'link'
-                });
-            }
-        });
-    }
-    // Vector expansion, invoked only: a third RRF list at the swept flat-region constants
-    // (weight 1, pool = fetch). Each row carries its best chunk's line range.
-    const chunkLines = new Map();
-    const chunkSimilarity = new Map();
+    if (signals.links !== undefined) linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);
+    let chunkLines = new Map();
+    let chunkSimilarity = new Map();
     if (semanticEnabled) {
-        const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);
-        vec.forEach(({ path, lines, similarity }, i)=>{
-            chunkLines.set(path, lines);
-            chunkSimilarity.set(path, similarity);
-            const existing = candidates.get(path);
-            if (existing) {
-                existing.score += 1 / (RRF_K + i);
-                existing.via = `${existing.via}+vector`;
-            } else {
-                candidates.set(path, {
-                    score: 1 / (RRF_K + i),
-                    via: 'vector'
-                });
-            }
-        });
+        ({ chunkLines, chunkSimilarity } = await vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, signals.vectors));
     }
     db.exec('DROP TABLE IF EXISTS _search');
     db.exec('CREATE TEMP TABLE _search ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');

package/dist/esm/commands/search.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, contentTokenize, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { modelPresent, semanticCandidates } from '../features/embed.ts';\nimport { linkEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph.ts';\nimport type { Row } from '../output.ts';\nimport { searchError } from '../search-error.ts';\nimport { segmentMatch } from '../segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\n\n// Mirrors the main columns onto the sidecars, so a title match found through title_seg ranks\n// like a title match found through title, not like a body match.\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\nconst RRF_K = 60;\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.\nconst SNIPPET_BOUND = 16_384;\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nfunction lineRangeFor(db: DatabaseSync, path: string, line: number): string | null {\n  const row = db.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// BM25 + link expansion + vectors, fused by reciprocal rank; `via` names the signal per row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k } = effective;\n\n  const allPaths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = rawScope(db, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = narrowByWhere(db, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // Asking for vectors without a model is a misconfiguration, not a mode: degrading would make\n  // the same search answer differently before and after a download. semantic:false never asks.\n  const wantsVectors = effective.semantic && anyPresetEmbeds(cfg);\n  if (wantsVectors && !modelPresent(cfg)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the embedding model is not available; run \\`sense download\\`, or set \"semantic\": false on that preset to search on words and links`);\n  }\n  const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);\n\n  // Terms pass to MATCH as written, with one transform: a run of text whose language marks no\n  // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier\n  // ahead of it retargets its _seg column), matching how it was indexed. Text that already\n  // carries boundaries comes through untouched, so this is a no-op for the languages that\n  // never needed it. Gated on the same predicate that decided whether the sidecars were\n  // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid\n  // syntax still errors rather than being rewritten.\n  // --where\n  // applies inside the candidate query (a post-filter would drop matches ranked past the pool)\n  // and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) materializeScope(db, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n  // Docs past SNIPPET_BOUND skip snippet() entirely -- SQLite\n  // short-circuits the untaken CASE branch, so it's never invoked on them.\n  // Column 2 (text), not -1 (best column): -1 could surface a _seg sidecar as the excerpt,\n  // which is machine-spaced and not what the author wrote. A row that matches only in a\n  // sidecar gets an unhighlighted text excerpt below instead -- a stated cost.\n  const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;\n  let matchRows: Array<{ path: string; hit: string | null }>;\n  try {\n    matchRows = db.prepare(matchSql).all(query) as Array<{ path: string; hit: string | null }>;\n  } catch (err) {\n    throw searchError(err as Error, terms, scope);\n  }\n\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n  const candidates = new Map<string, { score: number; via: string }>();\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: 1 / (RRF_K + i), via: 'match' });\n  });\n\n  const edges = featureEnabled(cfg, 'links') && matchRows.length > 0 ? linkEdges(db) : [];\n  if (edges.length > 0) {\n    // Gates the label only, not the score: restart mass ranks every seed without an incident\n    // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n    const linked = new Set(edges.flat());\n    const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n    const ranked = [...personalizedRank(allPaths, edges, seeds)]\n      .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n      .sort((a, b) => b[1] - a[1])\n      .slice(0, fetch);\n    ranked.forEach(([path], i) => {\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        if (linked.has(path)) existing.via = 'match+link';\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'link' });\n      }\n    });\n  }\n\n  // Vector expansion, invoked only: a third RRF list at the swept flat-region constants\n  // (weight 1, pool = fetch). Each row carries its best chunk's line range.\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);\n    vec.forEach(({ path, lines, similarity }, i) => {\n      chunkLines.set(path, lines);\n      chunkSimilarity.set(path, similarity);\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        existing.via = `${existing.via}+vector`;\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'vector' });\n      }\n    });\n  }\n\n  db.exec('DROP TABLE IF EXISTS _search');\n  db.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const rows = db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","anyPresetEmbeds","contentTokenize","featureEnabled","resolveSearch","SenseError","modelPresent","semanticCandidates","linkEdges","personalizedRank","searchError","segmentMatch","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","WEIGHTED_BM25","RRF_K","SNIPPET_BOUND","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","db","path","row","prepare","get","start_line","end_line","search","cfg","opts","hits","chunkLines","chunkSimilarity","effective","k","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","semantic","presetName","semanticEnabled","scope","whereJoin","whereCond","scopeCond","matchSql","undefined","matchRows","Map","hit","oversized","candidates","forEach","set","score","via","edges","linked","flat","seeds","ranked","has","existing","vec","lines","similarity","exec","insert","c","run","similarityCol","rows","bareTerms","baseDir"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,eAAe,EAAEC,eAAe,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AACrG,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,kBAAkB,QAAQ,uBAAuB;AACxE,SAASC,SAAS,QAAQ,uBAAuB;AACjD,SAASC,gBAAgB,QAAQ,cAAc;AAE/C,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAE3F,6FAA6F;AAC7F,iEAAiE;AACjE,MAAMC,gBAAgB;AACtB,MAAMC,QAAQ;AAEd,iFAAiF;AACjF,8EAA8E;AAC9E,MAAMC,gBAAgB;AACtB,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAASE,aAAaC,EAAgB,EAAEC,IAAY,EAAEJ,IAAY;IAChE,MAAMK,MAAMF,GAAGG,OAAO,CAAC,qIAAqIC,GAAG,CAACH,MAAMJ,MAAMA;IAC5K,OAAOK,MAAM,CAAC,CAAC,EAAEA,IAAIG,UAAU,CAAC,CAAC,EAAEH,IAAII,QAAQ,EAAE,GAAG;AACtD;AAWA,6FAA6F;AAC7F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOP,EAAgB,EAAEQ,GAAmB,EAAEjD,KAAa,EAAEkD,OAAsB,CAAC,CAAC;QA+GpCC,WAAwBC,iBAA8BC;IA9G3H,MAAMC,YAAYnF,cAAc8E,KAAKC;IACrC,MAAM,EAAEK,CAAC,EAAE,GAAGD;IAEd,MAAME,WAAW,AAACf,GAAGG,OAAO,CAAC,kCAAkCa,GAAG,GAA+BjE,GAAG,CAAC,CAACkE,IAAMA,EAAEhB,IAAI;IAClH,MAAMiB,aAAa9E,SAAS4D,IAAIQ,KAAKC,MAAMM;IAC3C,MAAMI,cAAcD,WAAWnC,IAAI,GAAGgC,SAAS3D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAIgE;IACJ,IAAI;QACFA,eAAejF,cAAc6D,IAAIkB,YAAYL,UAAUQ,KAAK;IAC9D,EAAE,OAAOC,KAAK;QACZ,MAAMtF,YAAYsF,KAAc/D,OAAOsD,UAAUQ,KAAK;IACxD;IACA,MAAME,QAAQrC,KAAKsC,GAAG,CAACV,IAAI,GAAG;IAE9B,6FAA6F;IAC7F,6FAA6F;IAC7F,MAAMW,eAAeZ,UAAUa,QAAQ,IAAInG,gBAAgBiF;IAC3D,IAAIiB,gBAAgB,CAAC7F,aAAa4E,MAAM;QACtC,MAAM,IAAI7E,WAAW,uBAAuB,CAAC,QAAQ,EAAEkF,UAAUc,UAAU,CAAC,+JAA+J,CAAC;IAC9O;IACA,MAAMC,kBAAkBH,gBAAgBpF,mBAAmB2D,IAAIQ,KAAKY;IAEpE,4FAA4F;IAC5F,0FAA0F;IAC1F,yFAAyF;IACzF,wFAAwF;IACxF,sFAAsF;IACtF,6FAA6F;IAC7F,mDAAmD;IACnD,UAAU;IACV,6FAA6F;IAC7F,wDAAwD;IACxD,MAAMS,QAAQhB,UAAUQ,KAAK;IAC7B,MAAMS,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIV,aAAajF,iBAAiB8D,IAAI,iBAAiBkB;IACvD,MAAMc,YAAYb,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAC3F,4DAA4D;IAC5D,yEAAyE;IACzE,yFAAyF;IACzF,sFAAsF;IACtF,6EAA6E;IAC7E,MAAMc,WAAW,CAAC,+DAA+D,EAAEzF,cAAc,+EAA+E,EAAEsF,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAE1F,cAAc,OAAO,EAAEiF,OAAO;IACtR,MAAM5E,QAAQnB,gBAAgBgF,SAAS0B,YAAYjG,aAAasB,SAASA;IACzE,IAAI4E;IACJ,IAAI;QACFA,YAAYnC,GAAGG,OAAO,CAAC8B,UAAUjB,GAAG,CAACrE;IACvC,EAAE,OAAO2E,KAAK;QACZ,MAAMtF,YAAYsF,KAAc/D,OAAOsE;IACzC;IAEA,MAAMnB,OAAO,IAAI0B,IAAID,UAAUpF,GAAG,CAAC,CAACkE,IAAM;YAACA,EAAEhB,IAAI;YAAEgB,EAAEoB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI3D,IAAIwD,UAAUhF,MAAM,CAAC,CAAC8D,IAAMA,EAAEoB,GAAG,KAAK,MAAMtF,GAAG,CAAC,CAACkE,IAAMA,EAAEhB,IAAI;IACnF,MAAMsC,aAAa,IAAIH;IACvBD,UAAUK,OAAO,CAAC,CAACvB,GAAGzC;QACpB+D,WAAWE,GAAG,CAACxB,EAAEhB,IAAI,EAAE;YAAEyC,OAAO,IAAKnG,CAAAA,QAAQiC,CAAAA;YAAImE,KAAK;QAAQ;IAChE;IAEA,MAAMC,QAAQnH,eAAe+E,KAAK,YAAY2B,UAAU/E,MAAM,GAAG,IAAItB,UAAUkE,MAAM,EAAE;IACvF,IAAI4C,MAAMxF,MAAM,GAAG,GAAG;QACpB,yFAAyF;QACzF,uFAAuF;QACvF,MAAMyF,SAAS,IAAIlE,IAAIiE,MAAME,IAAI;QACjC,MAAMC,QAAQ,IAAIX,IAAID,UAAUpF,GAAG,CAAC,CAACkE,GAAGzC,IAAM;gBAACyC,EAAEhB,IAAI;gBAAE,IAAKzB,CAAAA,IAAI,CAAA;aAAG;QACnE,MAAMwE,SAAS;eAAIjH,iBAAiBgF,UAAU6B,OAAOG;SAAO,CACzD5F,MAAM,CAAC,CAAC,CAAC8C,MAAMyC,MAAM,GAAKA,QAAQ,QAAQtB,aAAa6B,GAAG,CAAChD,OAC3DjC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BoB,KAAK,CAAC,GAAGkC;QACZyB,OAAOR,OAAO,CAAC,CAAC,CAACvC,KAAK,EAAEzB;YACtB,MAAM0E,WAAWX,WAAWnC,GAAG,CAACH;YAChC,IAAIiD,UAAU;gBACZA,SAASR,KAAK,IAAI,IAAKnG,CAAAA,QAAQiC,CAAAA;gBAC/B,IAAIqE,OAAOI,GAAG,CAAChD,OAAOiD,SAASP,GAAG,GAAG;YACvC,OAAO;gBACLJ,WAAWE,GAAG,CAACxC,MAAM;oBAAEyC,OAAO,IAAKnG,CAAAA,QAAQiC,CAAAA;oBAAImE,KAAK;gBAAO;YAC7D;QACF;IACF;IAEA,sFAAsF;IACtF,0EAA0E;IAC1E,MAAMhC,aAAa,IAAIyB;IACvB,MAAMxB,kBAAkB,IAAIwB;IAC5B,IAAIR,iBAAiB;QACnB,MAAMuB,MAAM,MAAMtH,mBAAmBmE,IAAIQ,KAAKjD,OAAOgE,OAAOH;QAC5D+B,IAAIX,OAAO,CAAC,CAAC,EAAEvC,IAAI,EAAEmD,KAAK,EAAEC,UAAU,EAAE,EAAE7E;YACxCmC,WAAW8B,GAAG,CAACxC,MAAMmD;YACrBxC,gBAAgB6B,GAAG,CAACxC,MAAMoD;YAC1B,MAAMH,WAAWX,WAAWnC,GAAG,CAACH;YAChC,IAAIiD,UAAU;gBACZA,SAASR,KAAK,IAAI,IAAKnG,CAAAA,QAAQiC,CAAAA;gBAC/B0E,SAASP,GAAG,GAAG,GAAGO,SAASP,GAAG,CAAC,OAAO,CAAC;YACzC,OAAO;gBACLJ,WAAWE,GAAG,CAACxC,MAAM;oBAAEyC,OAAO,IAAKnG,CAAAA,QAAQiC,CAAAA;oBAAImE,KAAK;gBAAS;YAC/D;QACF;IACF;IAEA3C,GAAGsD,IAAI,CAAC;IACRtD,GAAGsD,IAAI,CAAC;IACR,MAAMC,SAASvD,GAAGG,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACF,MAAMuD,EAAE,IAAIjB,WAAYgB,OAAOE,GAAG,CAACxD,MAAMuD,EAAEd,KAAK,EAAEc,EAAEb,GAAG,GAAEjC,YAAAA,KAAKN,GAAG,CAACH,mBAATS,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWP,GAAG,CAACH,mBAAfU,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBR,GAAG,CAACH,mBAApBW,kCAAAA,uBAA6B;IAExJ,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMS,QAAQQ,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAM6B,gBAAgB9B,kBAAkB,yBAAyB;IACjE,MAAM+B,OAAO3D,GACVG,OAAO,CACN,CAAC,kIAAkI,EAAEuD,cAAc;;OAElJ,EAAErC,MAAM,oCAAoC,CAAC,EAE/CL,GAAG,CAACF;IAEP,IAAIwB,UAAUvD,IAAI,GAAG,GAAG;QACtB,MAAM6E,YAAYlH,iBAAiBa;QACnC,KAAK,MAAM2C,OAAOyD,KAAM;YACtB,IAAIzD,IAAImC,GAAG,KAAK,QAAQ,CAACC,UAAUW,GAAG,CAAC/C,IAAID,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAO5D,aAAaC,KAAKkF,IAAIqD,OAAO,EAAE3D,IAAID,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAM2E;YACjD1D,IAAImC,GAAG,GAAGjD;YACV,IAAIc,IAAIkD,KAAK,IAAI,MAAMlD,IAAIkD,KAAK,GAAG3H,eAAe+E,KAAK,cAAcT,aAAaC,IAAIE,IAAID,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QAC1I;IACF;IAEA,OAAOqE;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { contentTokenize, embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output.ts';\nimport { searchError } from '../search-error.ts';\nimport { segmentMatch } from '../segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nfunction lineRangeFor(db: DatabaseSync, path: string, line: number): string | null {\n  const row = db.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = rawScope(db, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = narrowByWhere(db, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);\n\n  // Terms pass to MATCH as written, with one transform: a run of text whose language marks no\n  // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier\n  // ahead of it retargets its _seg column), matching how it was indexed. Text that already\n  // carries boundaries comes through untouched, so this is a no-op for the languages that\n  // never needed it. Gated on the same predicate that decided whether the sidecars were\n  // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid\n  // syntax still errors rather than being rewritten.\n  // --where\n  // applies inside the candidate query (a post-filter would drop matches ranked past the pool)\n  // and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) materializeScope(db, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n  const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: Array<{ path: string; hit: string | null }> = [];\n  // A query that rewrites to nothing (e.g. only unspaced-script punctuation) has no\n  // searchable words: zero lexical rows, and the other signals still compose.\n  if (signals.words !== undefined && query.trim() !== '') {\n    try {\n      matchRows = wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  db.exec('DROP TABLE IF EXISTS _search');\n  db.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const rows = db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","contentTokenize","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","segmentMatch","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","db","path","row","prepare","get","start_line","end_line","search","cfg","opts","hits","chunkLines","chunkSimilarity","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hit","oversized","links","exec","insert","c","run","score","via","similarityCol","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,eAAe,EAAEC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AACjG,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,8EAA8E;AAC9E,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAASE,aAAaC,EAAgB,EAAEC,IAAY,EAAEJ,IAAY;IAChE,MAAMK,MAAMF,GAAGG,OAAO,CAAC,qIAAqIC,GAAG,CAACH,MAAMJ,MAAMA;IAC5K,OAAOK,MAAM,CAAC,CAAC,EAAEA,IAAIG,UAAU,CAAC,CAAC,EAAEH,IAAII,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOP,EAAgB,EAAEQ,GAAmB,EAAEjD,KAAa,EAAEkD,OAAsB,CAAC,CAAC;QA6EpCC,WAAwBC,iBAA8BC;IA5E3H,MAAMC,YAAYjF,cAAc4E,KAAKC;IACrC,MAAM,EAAEK,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAChB,GAAGG,OAAO,CAAC,kCAAkCc,GAAG,GAA+BlE,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAClH,MAAMkB,aAAa/E,SAAS4D,IAAIQ,KAAKC,MAAMO;IAC3C,MAAMI,cAAcD,WAAWpC,IAAI,GAAGiC,SAAS5D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAIiE;IACJ,IAAI;QACFA,eAAelF,cAAc6D,IAAImB,YAAYN,UAAUS,KAAK;IAC9D,EAAE,OAAOC,KAAK;QACZ,MAAMvF,YAAYuF,KAAchE,OAAOsD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQtC,KAAKuC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAInG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkBgG,IAAI;YACxB,MAAM,IAAI9F,WAAW,uBAAuB,CAAC,QAAQ,EAAE8E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAEjG,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAMkG,kBAAkBN,gBAAgBrF,mBAAmB2D,IAAIQ,KAAKa;IAEpE,4FAA4F;IAC5F,0FAA0F;IAC1F,yFAAyF;IACzF,wFAAwF;IACxF,sFAAsF;IACtF,6FAA6F;IAC7F,mDAAmD;IACnD,UAAU;IACV,6FAA6F;IAC7F,wDAAwD;IACxD,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIb,aAAalF,iBAAiB8D,IAAI,iBAAiBmB;IACvD,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAC3F,MAAMzE,QAAQlB,gBAAgB+E,SAASoB,YAAY3F,aAAasB,SAASA;IAEzE,MAAM8E,aAAa,IAAIC;IACvB,IAAIC,YAAyD,EAAE;IAC/D,kFAAkF;IAClF,4EAA4E;IAC5E,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAajF,MAAMO,IAAI,OAAO,IAAI;QACtD,IAAI;YACFqF,YAAY/F,gBAAgBwD,IAAIqC,YAAY1F,OAAOuF,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QAC1G,EAAE,OAAOjB,KAAK;YACZ,MAAMvF,YAAYuF,KAAchE,OAAO0E;QACzC;IACF;IACA,MAAMvB,OAAO,IAAI4B,IAAIC,UAAUxF,GAAG,CAAC,CAACmE,IAAM;YAACA,EAAEjB,IAAI;YAAEiB,EAAEuB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI/D,IAAI4D,UAAUpF,MAAM,CAAC,CAAC+D,IAAMA,EAAEuB,GAAG,KAAK,MAAM1F,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAEnF,IAAIc,QAAQ4B,KAAK,KAAKf,WAAWtF,gBAAgB0D,IAAIqC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ4B,KAAK;IAExH,IAAIhC,aAAa,IAAI2B;IACrB,IAAI1B,kBAAkB,IAAI0B;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAErB,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrE,kBAAkByD,IAAIQ,KAAK6B,YAAY9E,OAAOiE,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IACvI;IAEA3B,GAAG4C,IAAI,CAAC;IACR5C,GAAG4C,IAAI,CAAC;IACR,MAAMC,SAAS7C,GAAGG,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACF,MAAM6C,EAAE,IAAIT,WAAYQ,OAAOE,GAAG,CAAC9C,MAAM6C,EAAEE,KAAK,EAAEF,EAAEG,GAAG,GAAEvC,YAAAA,KAAKN,GAAG,CAACH,mBAATS,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWP,GAAG,CAACH,mBAAfU,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBR,GAAG,CAACH,mBAApBW,kCAAAA,uBAA6B;IAExJ,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMU,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAMiB,gBAAgBlB,kBAAkB,yBAAyB;IACjE,MAAMmB,OAAOnD,GACVG,OAAO,CACN,CAAC,kIAAkI,EAAE+C,cAAc;;OAElJ,EAAE5B,MAAM,oCAAoC,CAAC,EAE/CL,GAAG,CAACH;IAEP,IAAI4B,UAAU3D,IAAI,GAAG,GAAG;QACtB,MAAMqE,YAAY1G,iBAAiBa;QACnC,KAAK,MAAM2C,OAAOiD,KAAM;YACtB,IAAIjD,IAAIuC,GAAG,KAAK,QAAQ,CAACC,UAAUW,GAAG,CAACnD,IAAID,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAO1D,aAAaC,KAAKgF,IAAI8C,OAAO,EAAEpD,IAAID,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMmE;YACjDlD,IAAIuC,GAAG,GAAGrD;YACV,IAAIc,IAAIqD,KAAK,IAAI,MAAMrD,IAAIqD,KAAK,GAAG5H,eAAe6E,KAAK,cAAcT,aAAaC,IAAIE,IAAID,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QAC1I;IACF;IAEA,OAAO6D;AACT"}

package/dist/esm/commands/signals.d.ts

@@ -0,0 +1,18 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { ResolvedConfig } from '../config/index.js';
+export type Candidates = Map<string, {
+    score: number;
+    via: string;
+}>;
+export declare function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{
+    path: string;
+    hit: string | null;
+}>;
+export declare function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{
+    path: string;
+    hit: string | null;
+}>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void;
+export declare function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{
+    chunkLines: Map<string, string>;
+    chunkSimilarity: Map<string, number>;
+}>;

package/dist/esm/commands/signals.js

@@ -0,0 +1,78 @@
+import { semanticCandidates } from '../embed/query.js';
+import { linkEdges } from '../features/index.js';
+import { personalizedRank } from '../graph.js';
+// RRF composes whatever the preset declares; the constant lives here since it belongs to
+// the fusion these three functions share, not to search() itself.
+const RRF_K = 60;
+// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared
+// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal
+// is seeded from its output, so search() runs this one first when "words" is declared.
+export function wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, weight) {
+    const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
+    const SNIPPET_BOUND = 16384;
+    const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;
+    const matchRows = db.prepare(matchSql).all(query);
+    matchRows.forEach((r, i)=>{
+        candidates.set(r.path, {
+            score: weight / (RRF_K + i),
+            via: 'match'
+        });
+    });
+    return matchRows;
+}
+// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this
+// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before
+// this was a signal.
+export function linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, weight) {
+    if (matchRows.length === 0) return;
+    const edges = linkEdges(db);
+    if (edges.length === 0) return;
+    // Gates the label only, not the score: restart mass ranks every seed without an incident
+    // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).
+    const linked = new Set(edges.flat());
+    const seeds = new Map(matchRows.map((r, i)=>[
+            r.path,
+            1 / (i + 1)
+        ]));
+    const ranked = [
+        ...personalizedRank(allPaths, edges, seeds)
+    ].filter(([path, score])=>score > 1e-9 && allowedPaths.has(path)).sort((a, b)=>b[1] - a[1]).slice(0, fetch);
+    ranked.forEach(([path], i)=>{
+        const existing = candidates.get(path);
+        if (existing) {
+            existing.score += weight / (RRF_K + i);
+            if (linked.has(path)) existing.via = 'match+link';
+        } else {
+            candidates.set(path, {
+                score: weight / (RRF_K + i),
+                via: 'link'
+            });
+        }
+    });
+}
+// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`
+// at this signal's weight. Each row also carries its best chunk's line range and similarity,
+// for the caller.
+export async function vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, weight) {
+    const chunkLines = new Map();
+    const chunkSimilarity = new Map();
+    const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);
+    vec.forEach(({ path, lines, similarity }, i)=>{
+        chunkLines.set(path, lines);
+        chunkSimilarity.set(path, similarity);
+        const existing = candidates.get(path);
+        if (existing) {
+            existing.score += weight / (RRF_K + i);
+            existing.via = `${existing.via}+vector`;
+        } else {
+            candidates.set(path, {
+                score: weight / (RRF_K + i),
+                via: 'vector'
+            });
+        }
+    });
+    return {
+        chunkLines,
+        chunkSimilarity
+    };
+}

package/dist/esm/commands/signals.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/signals.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { semanticCandidates } from '../embed/query.ts';\nimport { linkEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph.ts';\n\n// RRF composes whatever the preset declares; the constant lives here since it belongs to\n// the fusion these three functions share, not to search() itself.\nconst RRF_K = 60;\n\nexport type Candidates = Map<string, { score: number; via: string }>;\n\n// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared\n// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal\n// is seeded from its output, so search() runs this one first when \"words\" is declared.\nexport function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{ path: string; hit: string | null }> {\n  const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n  const SNIPPET_BOUND = 16_384;\n  const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const matchRows = db.prepare(matchSql).all(query) as Array<{ path: string; hit: string | null }>;\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: weight / (RRF_K + i), via: 'match' });\n  });\n  return matchRows;\n}\n\n// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this\n// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before\n// this was a signal.\nexport function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{ path: string; hit: string | null }>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void {\n  if (matchRows.length === 0) return;\n  const edges = linkEdges(db);\n  if (edges.length === 0) return;\n  // Gates the label only, not the score: restart mass ranks every seed without an incident\n  // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n  const linked = new Set(edges.flat());\n  const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n  const ranked = [...personalizedRank(allPaths, edges, seeds)]\n    .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, fetch);\n  ranked.forEach(([path], i) => {\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      if (linked.has(path)) existing.via = 'match+link';\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'link' });\n    }\n  });\n}\n\n// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`\n// at this signal's weight. Each row also carries its best chunk's line range and similarity,\n// for the caller.\nexport async function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{ chunkLines: Map<string, string>; chunkSimilarity: Map<string, number> }> {\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);\n  vec.forEach(({ path, lines, similarity }, i) => {\n    chunkLines.set(path, lines);\n    chunkSimilarity.set(path, similarity);\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      existing.via = `${existing.via}+vector`;\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'vector' });\n    }\n  });\n  return { chunkLines, chunkSimilarity };\n}\n"],"names":["semanticCandidates","linkEdges","personalizedRank","RRF_K","wordsCandidates","db","candidates","query","whereJoin","whereCond","scopeCond","fetch","weight","WEIGHTED_BM25","SNIPPET_BOUND","matchSql","matchRows","prepare","all","forEach","r","i","set","path","score","via","linksCandidates","allPaths","allowedPaths","length","edges","linked","Set","flat","seeds","Map","map","ranked","filter","has","sort","a","b","slice","existing","get","vectorsCandidates","cfg","terms","chunkLines","chunkSimilarity","vec","lines","similarity"],"mappings":"AAEA,SAASA,kBAAkB,QAAQ,oBAAoB;AACvD,SAASC,SAAS,QAAQ,uBAAuB;AACjD,SAASC,gBAAgB,QAAQ,cAAc;AAE/C,yFAAyF;AACzF,kEAAkE;AAClE,MAAMC,QAAQ;AAId,+FAA+F;AAC/F,gGAAgG;AAChG,uFAAuF;AACvF,OAAO,SAASC,gBAAgBC,EAAgB,EAAEC,UAAsB,EAAEC,KAAa,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,MAAc;IAC7K,MAAMC,gBAAgB;IACtB,MAAMC,gBAAgB;IACtB,MAAMC,WAAW,CAAC,+DAA+D,EAAED,cAAc,+EAA+E,EAAEN,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEG,cAAc,OAAO,EAAEF,OAAO;IACtR,MAAMK,YAAYX,GAAGY,OAAO,CAACF,UAAUG,GAAG,CAACX;IAC3CS,UAAUG,OAAO,CAAC,CAACC,GAAGC;QACpBf,WAAWgB,GAAG,CAACF,EAAEG,IAAI,EAAE;YAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;YAAII,KAAK;QAAQ;IACrE;IACA,OAAOT;AACT;AAEA,8FAA8F;AAC9F,gGAAgG;AAChG,qBAAqB;AACrB,OAAO,SAASU,gBAAgBrB,EAAgB,EAAEC,UAAsB,EAAEU,SAAsD,EAAEW,QAAkB,EAAEC,YAAyB,EAAEjB,KAAa,EAAEC,MAAc;IAC5M,IAAII,UAAUa,MAAM,KAAK,GAAG;IAC5B,MAAMC,QAAQ7B,UAAUI;IACxB,IAAIyB,MAAMD,MAAM,KAAK,GAAG;IACxB,yFAAyF;IACzF,uFAAuF;IACvF,MAAME,SAAS,IAAIC,IAAIF,MAAMG,IAAI;IACjC,MAAMC,QAAQ,IAAIC,IAAInB,UAAUoB,GAAG,CAAC,CAAChB,GAAGC,IAAM;YAACD,EAAEG,IAAI;YAAE,IAAKF,CAAAA,IAAI,CAAA;SAAG;IACnE,MAAMgB,SAAS;WAAInC,iBAAiByB,UAAUG,OAAOI;KAAO,CACzDI,MAAM,CAAC,CAAC,CAACf,MAAMC,MAAM,GAAKA,QAAQ,QAAQI,aAAaW,GAAG,CAAChB,OAC3DiB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGhC;IACZ0B,OAAOlB,OAAO,CAAC,CAAC,CAACI,KAAK,EAAEF;QACtB,MAAMuB,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpC,IAAIU,OAAOQ,GAAG,CAAChB,OAAOqB,SAASnB,GAAG,GAAG;QACvC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAO;QAClE;IACF;AACF;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,kBAAkB;AAClB,OAAO,eAAeqB,kBAAkBzC,EAAgB,EAAE0C,GAAmB,EAAEzC,UAAsB,EAAE0C,KAAa,EAAErC,KAAa,EAAEiB,YAAyB,EAAEhB,MAAc;IAC5K,MAAMqC,aAAa,IAAId;IACvB,MAAMe,kBAAkB,IAAIf;IAC5B,MAAMgB,MAAM,MAAMnD,mBAAmBK,IAAI0C,KAAKC,OAAOrC,OAAOiB;IAC5DuB,IAAIhC,OAAO,CAAC,CAAC,EAAEI,IAAI,EAAE6B,KAAK,EAAEC,UAAU,EAAE,EAAEhC;QACxC4B,WAAW3B,GAAG,CAACC,MAAM6B;QACrBF,gBAAgB5B,GAAG,CAACC,MAAM8B;QAC1B,MAAMT,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpCuB,SAASnB,GAAG,GAAG,GAAGmB,SAASnB,GAAG,CAAC,OAAO,CAAC;QACzC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAS;QACpE;IACF;IACA,OAAO;QAAEwB;QAAYC;IAAgB;AACvC"}

package/dist/esm/commands/status.d.ts

@@ -1,9 +1,9 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { ResolvedConfig } from '../config/index.js';
+import type { ResolvedConfig, SignalWeights } from '../config/index.js';
 export interface PresetCoverage {
     name: string;
     files: number;
     embedded: number;
-    semantic: boolean;
+    signals: SignalWeights;
 }
 export declare function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[];

package/dist/esm/commands/status.js

@@ -1,4 +1,4 @@
-import { anyPresetEmbeds, presetNames, presetSemanticEnabled } from '../config/index.js';
+import { anyPresetEmbeds, presetNames, presetSignals } from '../config/index.js';
 // Indexing derives from presets, so the derivation stays visible. Read from preset_files, not
 // recomputed from globs, so it reflects the cache rather than the config.
 export function presetCoverage(db, cfg) {
@@ -10,7 +10,7 @@ export function presetCoverage(db, cfg) {
             name,
             files,
             embedded,
-            semantic: presetSemanticEnabled(cfg, name)
+            signals: presetSignals(cfg, name)
         };
     });
 }

package/dist/esm/commands/status.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, presetNames, presetSemanticEnabled } from '../config/index.ts';\n\nexport interface PresetCoverage {\n  name: string;\n  files: number;\n  embedded: number;\n  // Reported so 0 embedded reads as \"this scope declined vectors\" rather than \"not yet built\".\n  semantic: boolean;\n}\n\n// Indexing derives from presets, so the derivation stays visible. Read from preset_files, not\n// recomputed from globs, so it reflects the cache rather than the config.\nexport function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[] {\n  const embedActive = anyPresetEmbeds(cfg);\n  return presetNames(cfg).map((name) => {\n    const files = (db.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?').get(name) as { n: number }).n;\n    const embedded = embedActive ? (db.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e.\"path\" = pf.\"path\" AND e.vector IS NOT NULL)').get(name) as { n: number }).n : 0;\n    return { name, files, embedded, semantic: presetSemanticEnabled(cfg, name) };\n  });\n}\n"],"names":["anyPresetEmbeds","presetNames","presetSemanticEnabled","presetCoverage","db","cfg","embedActive","map","name","files","prepare","get","n","embedded","semantic"],"mappings":"AAEA,SAASA,eAAe,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,qBAAqB;AAUzF,8FAA8F;AAC9F,0EAA0E;AAC1E,OAAO,SAASC,eAAeC,EAAgB,EAAEC,GAAmB;IAClE,MAAMC,cAAcN,gBAAgBK;IACpC,OAAOJ,YAAYI,KAAKE,GAAG,CAAC,CAACC;QAC3B,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,2DAA2DC,GAAG,CAACH,MAAwBI,CAAC;QAClH,MAAMC,WAAWP,cAAc,AAACF,GAAGM,OAAO,CAAC,6JAA6JC,GAAG,CAACH,MAAwBI,CAAC,GAAG;QACxO,OAAO;YAAEJ;YAAMC;YAAOI;YAAUC,UAAUZ,sBAAsBG,KAAKG;QAAM;IAC7E;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig, SignalWeights } from '../config/index.ts';\nimport { anyPresetEmbeds, presetNames, presetSignals } from '../config/index.ts';\n\nexport interface PresetCoverage {\n  name: string;\n  files: number;\n  embedded: number;\n  // Reported so 0 embedded reads as \"this scope declined vectors\" rather than \"not yet built\".\n  signals: SignalWeights;\n}\n\n// Indexing derives from presets, so the derivation stays visible. Read from preset_files, not\n// recomputed from globs, so it reflects the cache rather than the config.\nexport function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[] {\n  const embedActive = anyPresetEmbeds(cfg);\n  return presetNames(cfg).map((name) => {\n    const files = (db.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?').get(name) as { n: number }).n;\n    const embedded = embedActive ? (db.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e.\"path\" = pf.\"path\" AND e.vector IS NOT NULL)').get(name) as { n: number }).n : 0;\n    return { name, files, embedded, signals: presetSignals(cfg, name) };\n  });\n}\n"],"names":["anyPresetEmbeds","presetNames","presetSignals","presetCoverage","db","cfg","embedActive","map","name","files","prepare","get","n","embedded","signals"],"mappings":"AAEA,SAASA,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,qBAAqB;AAUjF,8FAA8F;AAC9F,0EAA0E;AAC1E,OAAO,SAASC,eAAeC,EAAgB,EAAEC,GAAmB;IAClE,MAAMC,cAAcN,gBAAgBK;IACpC,OAAOJ,YAAYI,KAAKE,GAAG,CAAC,CAACC;QAC3B,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,2DAA2DC,GAAG,CAACH,MAAwBI,CAAC;QAClH,MAAMC,WAAWP,cAAc,AAACF,GAAGM,OAAO,CAAC,6JAA6JC,GAAG,CAACH,MAAwBI,CAAC,GAAG;QACxO,OAAO;YAAEJ;YAAMC;YAAOI;YAAUC,SAASZ,cAAcG,KAAKG;QAAM;IACpE;AACF"}

package/dist/esm/config/access.d.ts

@@ -1,7 +1,9 @@
+import type { SignalName, SignalWeights } from './signals.js';
 import { type Config, type FeatureName } from './types.js';
 export declare function presetNames(cfg: Config): string[];
 export declare function embedEnabled(cfg: Config): boolean;
-export declare function presetSemanticEnabled(cfg: Config, name: string): boolean;
+export declare function presetSignals(cfg: Config, name: string): SignalWeights;
+export declare function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean;
 export declare function anyPresetEmbeds(cfg: Config): boolean;
 export declare function featureEnabled(cfg: Config, name: FeatureName): boolean;
 export declare function enabledFeatures(cfg: Config): FeatureName[];
@@ -11,9 +13,10 @@ export declare function featureStates(cfg: Config): {
 };
 export declare function embedConfig(cfg: Config): {
     model: string;
-    type: 'static' | 'api';
+    provider: 'static' | 'openai' | 'cohere';
     url?: string;
     key?: string;
+    languages?: string[];
 } | null;
 export declare function contentTokenize(cfg: Config): string | undefined;
 export declare function featureSignature(cfg: Config): string;

package/dist/esm/config/access.js

@@ -1,3 +1,4 @@
+import { modelIdentity } from '../embed/identity.js';
 import { FEATURE_NAMES } from './types.js';
 export function presetNames(cfg) {
     return Object.keys(cfg.presets);
@@ -7,17 +8,31 @@ export function embedEnabled(cfg) {
     var _cfg_embed;
     return typeof ((_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.model) === 'string' && cfg.embed.model.length > 0;
 }
-// A preset's own vector participation: absent or true means on, false means off.
-export function presetSemanticEnabled(cfg, name) {
-    var _cfg_presets_name;
-    return ((_cfg_presets_name = cfg.presets[name]) === null || _cfg_presets_name === void 0 ? void 0 : _cfg_presets_name.semantic) !== false;
+// A preset's effective signal weights: declared exhaustively when `signals` is present;
+// otherwise every signal whose prerequisite holds, each at weight 1 -- words always, links when
+// the feature is on, vectors when the tree names a model. Single source search() and status/map
+// read.
+export function presetSignals(cfg, name) {
+    var _cfg_presets_name, _cfg_features;
+    const declared = (_cfg_presets_name = cfg.presets[name]) === null || _cfg_presets_name === void 0 ? void 0 : _cfg_presets_name.signals;
+    if (declared) return declared;
+    const signals = {
+        words: 1
+    };
+    if (((_cfg_features = cfg.features) === null || _cfg_features === void 0 ? void 0 : _cfg_features.links) !== false) signals.links = 1;
+    if (embedEnabled(cfg)) signals.vectors = 1;
+    return signals;
+}
+export function presetHasSignal(cfg, name, signal) {
+    return presetSignals(cfg, name)[signal] !== undefined;
 }
-// Whether embedding needs to run: a model is named AND some preset wants vectors.
+// Whether embedding needs to run: a model is named AND some preset's effective signals include vectors.
 export function anyPresetEmbeds(cfg) {
-    return embedEnabled(cfg) && presetNames(cfg).some((name)=>presetSemanticEnabled(cfg, name));
+    return presetNames(cfg).some((name)=>presetHasSignal(cfg, name, 'vectors'));
 }
 // Opt-out features (default on): absent block or key means enabled. `rank` additionally
-// requires `links`. `embed` is derived, not a features-block member: on iff `embed` names a model.
+// requires `links`. `embed` is derived, not a features-block member: on iff some preset's
+// effective signals include vectors.
 export function featureEnabled(cfg, name) {
     var _cfg_features;
     if (name === 'embed') return anyPresetEmbeds(cfg);
@@ -41,14 +56,15 @@ export function featureStates(cfg) {
 // fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration
 // write into the file, never an implicit runtime default.
 export function embedConfig(cfg) {
-    var _e_type;
+    var _e_provider;
     if (!embedEnabled(cfg)) return null;
     const e = cfg.embed;
     return {
         model: e.model,
-        type: (_e_type = e.type) !== null && _e_type !== void 0 ? _e_type : 'static',
+        provider: (_e_provider = e.provider) !== null && _e_provider !== void 0 ? _e_provider : 'static',
         url: e.url,
-        key: e.key
+        key: e.key,
+        languages: e.languages
     };
 }
 // The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than
@@ -63,7 +79,10 @@ export function contentTokenize(cfg) {
 export function featureSignature(cfg) {
     const globalPart = enabledFeatures(cfg).filter((name)=>name !== 'embed').join(',');
     const e = embedConfig(cfg);
-    const embedPart = e ? `embed:${e.type}:${e.model}` : 'embed:off';
+    // Weight identity from identity.ts, no network: a static model's resolved sha or local
+    // size+mtime, appended once known so changed weights re-embed.
+    const identity = e && e.provider === 'static' ? modelIdentity(e.model) : undefined;
+    const embedPart = e ? `embed:${e.provider}:${e.model}${identity !== undefined ? `@${identity}` : ''}` : 'embed:off';
     // One keyed segment per preset so a rebuild notice can name exactly which preset moved.
     const presetsPart = [
         ...presetNames(cfg)
@@ -76,7 +95,7 @@ export function featureSignature(cfg) {
         const exclude = [
             ...(_p_exclude = p.exclude) !== null && _p_exclude !== void 0 ? _p_exclude : []
         ].sort().join('+');
-        return `preset:${name}:${include}:${exclude}:${p.semantic === false ? 'off' : 'on'}`;
+        return `preset:${name}:${include}:${exclude}:${presetHasSignal(cfg, name, 'vectors') ? 'on' : 'off'}`;
     }).join('|');
     const tokenize = contentTokenize(cfg);
     const parts = [