sensemaking 0.17.1 → 0.18.0

package/dist/esm/commands/search.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { contentTokenize, embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport { segmentMatch } from '../text/segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, rows get the JS excerpt.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nfunction lineRangeFor(db: DatabaseSync, path: string, line: number): string | null {\n  const row = db.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = rawScope(db, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = narrowByWhere(db, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);\n\n  // Terms pass to MATCH as written, with one transform: a run of text whose language marks no\n  // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier\n  // ahead of it retargets its _seg column), matching how it was indexed. Text that already\n  // carries boundaries comes through untouched, so this is a no-op for the languages that\n  // never needed it. Gated on the same predicate that decided whether the sidecars were\n  // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid\n  // syntax still errors rather than being rewritten.\n  // --where\n  // applies inside the candidate query (a post-filter would drop matches ranked past the pool)\n  // and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) materializeScope(db, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n  const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: Array<{ path: string; hit: string | null }> = [];\n  // A query that rewrites to nothing (e.g. only unspaced-script punctuation) has no\n  // searchable words: zero lexical rows, and the other signals still compose.\n  if (signals.words !== undefined && query.trim() !== '') {\n    try {\n      matchRows = wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  db.exec('DROP TABLE IF EXISTS _search');\n  db.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const rows = db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","contentTokenize","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","segmentMatch","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","db","path","row","prepare","get","start_line","end_line","search","cfg","opts","hits","chunkLines","chunkSimilarity","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hit","oversized","links","exec","insert","c","run","score","via","similarityCol","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,eAAe,EAAEC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AACjG,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,4BAA4B;AACxD,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,mGAAmG;AACnG,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAASE,aAAaC,EAAgB,EAAEC,IAAY,EAAEJ,IAAY;IAChE,MAAMK,MAAMF,GAAGG,OAAO,CAAC,qIAAqIC,GAAG,CAACH,MAAMJ,MAAMA;IAC5K,OAAOK,MAAM,CAAC,CAAC,EAAEA,IAAIG,UAAU,CAAC,CAAC,EAAEH,IAAII,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOP,EAAgB,EAAEQ,GAAmB,EAAEjD,KAAa,EAAEkD,OAAsB,CAAC,CAAC;QA6EpCC,WAAwBC,iBAA8BC;IA5E3H,MAAMC,YAAYjF,cAAc4E,KAAKC;IACrC,MAAM,EAAEK,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAChB,GAAGG,OAAO,CAAC,kCAAkCc,GAAG,GAA+BlE,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAClH,MAAMkB,aAAa/E,SAAS4D,IAAIQ,KAAKC,MAAMO;IAC3C,MAAMI,cAAcD,WAAWpC,IAAI,GAAGiC,SAAS5D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAIiE;IACJ,IAAI;QACFA,eAAelF,cAAc6D,IAAImB,YAAYN,UAAUS,KAAK;IAC9D,EAAE,OAAOC,KAAK;QACZ,MAAMvF,YAAYuF,KAAchE,OAAOsD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQtC,KAAKuC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAInG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkBgG,IAAI;YACxB,MAAM,IAAI9F,WAAW,uBAAuB,CAAC,QAAQ,EAAE8E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAEjG,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAMkG,kBAAkBN,gBAAgBrF,mBAAmB2D,IAAIQ,KAAKa;IAEpE,4FAA4F;IAC5F,0FAA0F;IAC1F,yFAAyF;IACzF,wFAAwF;IACxF,sFAAsF;IACtF,6FAA6F;IAC7F,mDAAmD;IACnD,UAAU;IACV,6FAA6F;IAC7F,wDAAwD;IACxD,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIb,aAAalF,iBAAiB8D,IAAI,iBAAiBmB;IACvD,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAC3F,MAAMzE,QAAQlB,gBAAgB+E,SAASoB,YAAY3F,aAAasB,SAASA;IAEzE,MAAM8E,aAAa,IAAIC;IACvB,IAAIC,YAAyD,EAAE;IAC/D,kFAAkF;IAClF,4EAA4E;IAC5E,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAajF,MAAMO,IAAI,OAAO,IAAI;QACtD,IAAI;YACFqF,YAAY/F,gBAAgBwD,IAAIqC,YAAY1F,OAAOuF,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QAC1G,EAAE,OAAOjB,KAAK;YACZ,MAAMvF,YAAYuF,KAAchE,OAAO0E;QACzC;IACF;IACA,MAAMvB,OAAO,IAAI4B,IAAIC,UAAUxF,GAAG,CAAC,CAACmE,IAAM;YAACA,EAAEjB,IAAI;YAAEiB,EAAEuB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI/D,IAAI4D,UAAUpF,MAAM,CAAC,CAAC+D,IAAMA,EAAEuB,GAAG,KAAK,MAAM1F,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAEnF,IAAIc,QAAQ4B,KAAK,KAAKf,WAAWtF,gBAAgB0D,IAAIqC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ4B,KAAK;IAExH,IAAIhC,aAAa,IAAI2B;IACrB,IAAI1B,kBAAkB,IAAI0B;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAErB,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrE,kBAAkByD,IAAIQ,KAAK6B,YAAY9E,OAAOiE,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IACvI;IAEA3B,GAAG4C,IAAI,CAAC;IACR5C,GAAG4C,IAAI,CAAC;IACR,MAAMC,SAAS7C,GAAGG,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACF,MAAM6C,EAAE,IAAIT,WAAYQ,OAAOE,GAAG,CAAC9C,MAAM6C,EAAEE,KAAK,EAAEF,EAAEG,GAAG,GAAEvC,YAAAA,KAAKN,GAAG,CAACH,mBAATS,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWP,GAAG,CAACH,mBAAfU,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBR,GAAG,CAACH,mBAApBW,kCAAAA,uBAA6B;IAExJ,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMU,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAMiB,gBAAgBlB,kBAAkB,yBAAyB;IACjE,MAAMmB,OAAOnD,GACVG,OAAO,CACN,CAAC,kIAAkI,EAAE+C,cAAc;;OAElJ,EAAE5B,MAAM,oCAAoC,CAAC,EAE/CL,GAAG,CAACH;IAEP,IAAI4B,UAAU3D,IAAI,GAAG,GAAG;QACtB,MAAMqE,YAAY1G,iBAAiBa;QACnC,KAAK,MAAM2C,OAAOiD,KAAM;YACtB,IAAIjD,IAAIuC,GAAG,KAAK,QAAQ,CAACC,UAAUW,GAAG,CAACnD,IAAID,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAO1D,aAAaC,KAAKgF,IAAI8C,OAAO,EAAEpD,IAAID,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMmE;YACjDlD,IAAIuC,GAAG,GAAGrD;YACV,IAAIc,IAAIqD,KAAK,IAAI,MAAMrD,IAAIqD,KAAK,GAAG5H,eAAe6E,KAAK,cAAcT,aAAaC,IAAIE,IAAID,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QAC1I;IACF;IAEA,OAAO6D;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, rows get the JS excerpt.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nasync function lineRangeFor(store: Store, path: string, line: number): Promise<string | null> {\n  const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');\n  const row = (await stmt.get(path, line, line)) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(store: Store, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = await rawScope(store, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = await narrowByWhere(store, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && (await scopeHasEmbeddings(store, cfg, allowedPaths));\n\n  // --where applies inside the candidate query (a post-filter would drop matches ranked past\n  // the pool) and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: LexicalHit[] = [];\n  // A query that is only whitespace has no searchable words: zero lexical rows, and the other\n  // signals still compose. Any store-specific rewrite of `terms` (sqlite's grapheme phrase for\n  // unspaced scripts, duckdb's word/substring split) happens inside store.lexical.query itself.\n  if (signals.words !== undefined && terms.trim() !== '') {\n    try {\n      matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  await store.exec('DROP TABLE IF EXISTS _search');\n  // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the\n  // rounded score/similarity are printed at full precision in rows.\n  await store.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score DOUBLE, via TEXT, hit TEXT, lines TEXT, similarity DOUBLE)');\n  if (candidates.size > 0) {\n    await store.runBatch(\n      'INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)',\n      [...candidates].map(([path, c]) => [path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null])\n    );\n  }\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const selectStmt = await store.prepare(\n    `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n  );\n  const rows = (await selectStmt.all(k)) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","store","path","stmt","prepare","row","get","start_line","end_line","search","cfg","opts","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hits","hit","oversized","links","chunkLines","chunkSimilarity","exec","runBatch","c","score","via","similarityCol","selectStmt","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AAChF,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,4BAA4B;AAExD,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,mGAAmG;AACnG,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,eAAeE,aAAaC,KAAY,EAAEC,IAAY,EAAEJ,IAAY;IAClE,MAAMK,OAAO,MAAMF,MAAMG,OAAO,CAAC;IACjC,MAAMC,MAAO,MAAMF,KAAKG,GAAG,CAACJ,MAAMJ,MAAMA;IACxC,OAAOO,MAAM,CAAC,CAAC,EAAEA,IAAIE,UAAU,CAAC,CAAC,EAAEF,IAAIG,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOR,KAAY,EAAES,GAAmB,EAAElD,KAAa,EAAEmD,OAAsB,CAAC,CAAC;IACrG,MAAMC,YAAY9E,cAAc4E,KAAKC;IACrC,MAAM,EAAEE,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMd,MAAMG,OAAO,CAAC,iCAAgC,EAAGY,GAAG,EAAC,EAA+BhE,GAAG,CAAC,CAACiE,IAAMA,EAAEf,IAAI;IACrI,MAAMgB,aAAa,MAAM7E,SAAS4D,OAAOS,KAAKC,MAAMI;IACpD,MAAMI,cAAcD,WAAWlC,IAAI,GAAG+B,SAAS1D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAI+D;IACJ,IAAI;QACFA,eAAe,MAAMhF,cAAc6D,OAAOiB,YAAYN,UAAUS,KAAK;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAMpF,YAAYoF,KAAc9D,OAAOoD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQpC,KAAKqC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAIhG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkB6F,IAAI;YACxB,MAAM,IAAI3F,WAAW,uBAAuB,CAAC,QAAQ,EAAE2E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAE9F,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAM+F,kBAAkBN,gBAAiB,MAAMnF,mBAAmB2D,OAAOS,KAAKU;IAE9E,2FAA2F;IAC3F,kEAAkE;IAClE,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIb,aAAa,MAAMhF,iBAAiB8D,OAAO,iBAAiBiB;IAChE,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAE3F,MAAMiB,aAAa,IAAIC;IACvB,IAAIC,YAA0B,EAAE;IAChC,4FAA4F;IAC5F,6FAA6F;IAC7F,8FAA8F;IAC9F,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAanE,MAAML,IAAI,OAAO,IAAI;QACtD,IAAI;YACFmF,YAAY,MAAM7F,gBAAgBwD,OAAOmC,YAAY5E,OAAOyE,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QACnH,EAAE,OAAOjB,KAAK;YACZ,MAAMpF,YAAYoF,KAAc9D,OAAOwE;QACzC;IACF;IACA,MAAMQ,OAAO,IAAIH,IAAIC,UAAUtF,GAAG,CAAC,CAACiE,IAAM;YAACA,EAAEf,IAAI;YAAEe,EAAEwB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI9D,IAAI0D,UAAUlF,MAAM,CAAC,CAAC6D,IAAMA,EAAEwB,GAAG,KAAK,MAAMzF,GAAG,CAAC,CAACiE,IAAMA,EAAEf,IAAI;IAEnF,IAAIY,QAAQ6B,KAAK,KAAKhB,WAAW,MAAMpF,gBAAgB0D,OAAOmC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ6B,KAAK;IAEjI,IAAIC,aAAa,IAAIP;IACrB,IAAIQ,kBAAkB,IAAIR;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAEa,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrG,kBAAkByD,OAAOS,KAAK0B,YAAY5E,OAAO+D,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IAC1I;IAEA,MAAMzB,MAAM6C,IAAI,CAAC;IACjB,8FAA8F;IAC9F,kEAAkE;IAClE,MAAM7C,MAAM6C,IAAI,CAAC;IACjB,IAAIV,WAAWpD,IAAI,GAAG,GAAG;QACvB,MAAMiB,MAAM8C,QAAQ,CAClB,8FACA;eAAIX;SAAW,CAACpF,GAAG,CAAC,CAAC,CAACkD,MAAM8C,EAAE;gBAA4BR,WAAwBI,iBAA8BC;mBAA7E;gBAAC3C;gBAAM8C,EAAEC,KAAK;gBAAED,EAAEE,GAAG;iBAAEV,YAAAA,KAAKlC,GAAG,CAACJ,mBAATsC,uBAAAA,YAAkB;iBAAMI,kBAAAA,WAAWtC,GAAG,CAACJ,mBAAf0C,6BAAAA,kBAAwB;iBAAMC,uBAAAA,gBAAgBvC,GAAG,CAACJ,mBAApB2C,kCAAAA,uBAA6B;aAAK;;IAEtJ;IAEA,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMxB,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAMmB,gBAAgBpB,kBAAkB,yBAAyB;IACjE,MAAMqB,aAAa,MAAMnD,MAAMG,OAAO,CACpC,CAAC,kIAAkI,EAAE+C,cAAc;;OAEhJ,EAAE9B,MAAM,oCAAoC,CAAC;IAElD,MAAMgC,OAAQ,MAAMD,WAAWpC,GAAG,CAACH;IAEnC,IAAI6B,UAAU1D,IAAI,GAAG,GAAG;QACtB,MAAMsE,YAAY3G,iBAAiBa;QACnC,KAAK,MAAM6C,OAAOgD,KAAM;YACtB,IAAIhD,IAAIoC,GAAG,KAAK,QAAQ,CAACC,UAAUa,GAAG,CAAClD,IAAIH,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAOxD,aAAaC,KAAK+E,IAAI8C,OAAO,EAAEnD,IAAIH,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMoE;YACjDjD,IAAIoC,GAAG,GAAGpD;YACV,IAAIgB,IAAIoD,KAAK,IAAI,MAAMpD,IAAIoD,KAAK,GAAG5H,eAAe6E,KAAK,cAAc,MAAMV,aAAaC,OAAOI,IAAIH,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QACnJ;IACF;IAEA,OAAO8D;AACT"}

package/dist/esm/commands/signals.d.ts

@@ -1,18 +1,12 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig } from '../config/index.js';
+import type { LexicalHit, Store } from '../store/types.js';
 export type Candidates = Map<string, {
     score: number;
     via: string;
 }>;
-export declare function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{
-    path: string;
-    hit: string | null;
-}>;
-export declare function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{
-    path: string;
-    hit: string | null;
-}>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void;
-export declare function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{
+export declare function wordsCandidates(store: Store, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Promise<LexicalHit[]>;
+export declare function linksCandidates(store: Store, candidates: Candidates, matchRows: LexicalHit[], allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): Promise<void>;
+export declare function vectorsCandidates(store: Store, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{
     chunkLines: Map<string, string>;
     chunkSimilarity: Map<string, number>;
 }>;

package/dist/esm/commands/signals.js

@@ -1,17 +1,20 @@
 import { semanticCandidates } from '../embed/query.js';
-import { linkEdges } from '../features/index.js';
+import { LINK_EDGES_SQL, toEdges } from '../features/index.js';
 import { personalizedRank } from '../graph/graph.js';
 // RRF composes whatever the preset declares; the constant lives here since it belongs to
 // the fusion these three functions share, not to search() itself.
 const RRF_K = 60;
-// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared
-// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal
-// is seeded from its output, so search() runs this one first when "words" is declared.
-export function wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, weight) {
-    const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
-    const SNIPPET_BOUND = 16384;
-    const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;
-    const matchRows = db.prepare(matchSql).all(query);
+// BM25 match rows (store.lexical.query), folded into `candidates` by reciprocal rank scaled by
+// the preset's declared weight for this signal (default 1, bit-identical to the unweighted
+// score). Every other signal is seeded from its output, so search() runs this one first when
+// "words" is declared.
+export async function wordsCandidates(store, candidates, query, whereJoin, whereCond, scopeCond, fetch, weight) {
+    const matchRows = await store.lexical.query(query, {
+        whereJoin,
+        whereCond,
+        scopeCond,
+        limit: fetch
+    });
     matchRows.forEach((r, i)=>{
         candidates.set(r.path, {
             score: weight / (RRF_K + i),
@@ -23,9 +26,10 @@ export function wordsCandidates(db, candidates, query, whereJoin, whereCond, sco
 // Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this
 // signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before
 // this was a signal.
-export function linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, weight) {
+export async function linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, weight) {
     if (matchRows.length === 0) return;
-    const edges = linkEdges(db);
+    const stmt = await store.prepare(LINK_EDGES_SQL);
+    const edges = toEdges(await stmt.all());
     if (edges.length === 0) return;
     // Gates the label only, not the score: restart mass ranks every seed without an incident
     // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).
@@ -53,10 +57,10 @@ export function linksCandidates(db, candidates, matchRows, allPaths, allowedPath
 // Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`
 // at this signal's weight. Each row also carries its best chunk's line range and similarity,
 // for the caller.
-export async function vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, weight) {
+export async function vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, weight) {
     const chunkLines = new Map();
     const chunkSimilarity = new Map();
-    const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);
+    const vec = await semanticCandidates(store, cfg, terms, fetch, allowedPaths);
     vec.forEach(({ path, lines, similarity }, i)=>{
         chunkLines.set(path, lines);
         chunkSimilarity.set(path, similarity);

package/dist/esm/commands/signals.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/signals.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { semanticCandidates } from '../embed/query.ts';\nimport { linkEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph/graph.ts';\n\n// RRF composes whatever the preset declares; the constant lives here since it belongs to\n// the fusion these three functions share, not to search() itself.\nconst RRF_K = 60;\n\nexport type Candidates = Map<string, { score: number; via: string }>;\n\n// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared\n// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal\n// is seeded from its output, so search() runs this one first when \"words\" is declared.\nexport function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{ path: string; hit: string | null }> {\n  const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n  const SNIPPET_BOUND = 16_384;\n  const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const matchRows = db.prepare(matchSql).all(query) as Array<{ path: string; hit: string | null }>;\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: weight / (RRF_K + i), via: 'match' });\n  });\n  return matchRows;\n}\n\n// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this\n// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before\n// this was a signal.\nexport function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{ path: string; hit: string | null }>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void {\n  if (matchRows.length === 0) return;\n  const edges = linkEdges(db);\n  if (edges.length === 0) return;\n  // Gates the label only, not the score: restart mass ranks every seed without an incident\n  // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n  const linked = new Set(edges.flat());\n  const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n  const ranked = [...personalizedRank(allPaths, edges, seeds)]\n    .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, fetch);\n  ranked.forEach(([path], i) => {\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      if (linked.has(path)) existing.via = 'match+link';\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'link' });\n    }\n  });\n}\n\n// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`\n// at this signal's weight. Each row also carries its best chunk's line range and similarity,\n// for the caller.\nexport async function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{ chunkLines: Map<string, string>; chunkSimilarity: Map<string, number> }> {\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);\n  vec.forEach(({ path, lines, similarity }, i) => {\n    chunkLines.set(path, lines);\n    chunkSimilarity.set(path, similarity);\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      existing.via = `${existing.via}+vector`;\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'vector' });\n    }\n  });\n  return { chunkLines, chunkSimilarity };\n}\n"],"names":["semanticCandidates","linkEdges","personalizedRank","RRF_K","wordsCandidates","db","candidates","query","whereJoin","whereCond","scopeCond","fetch","weight","WEIGHTED_BM25","SNIPPET_BOUND","matchSql","matchRows","prepare","all","forEach","r","i","set","path","score","via","linksCandidates","allPaths","allowedPaths","length","edges","linked","Set","flat","seeds","Map","map","ranked","filter","has","sort","a","b","slice","existing","get","vectorsCandidates","cfg","terms","chunkLines","chunkSimilarity","vec","lines","similarity"],"mappings":"AAEA,SAASA,kBAAkB,QAAQ,oBAAoB;AACvD,SAASC,SAAS,QAAQ,uBAAuB;AACjD,SAASC,gBAAgB,QAAQ,oBAAoB;AAErD,yFAAyF;AACzF,kEAAkE;AAClE,MAAMC,QAAQ;AAId,+FAA+F;AAC/F,gGAAgG;AAChG,uFAAuF;AACvF,OAAO,SAASC,gBAAgBC,EAAgB,EAAEC,UAAsB,EAAEC,KAAa,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,MAAc;IAC7K,MAAMC,gBAAgB;IACtB,MAAMC,gBAAgB;IACtB,MAAMC,WAAW,CAAC,+DAA+D,EAAED,cAAc,+EAA+E,EAAEN,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEG,cAAc,OAAO,EAAEF,OAAO;IACtR,MAAMK,YAAYX,GAAGY,OAAO,CAACF,UAAUG,GAAG,CAACX;IAC3CS,UAAUG,OAAO,CAAC,CAACC,GAAGC;QACpBf,WAAWgB,GAAG,CAACF,EAAEG,IAAI,EAAE;YAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;YAAII,KAAK;QAAQ;IACrE;IACA,OAAOT;AACT;AAEA,8FAA8F;AAC9F,gGAAgG;AAChG,qBAAqB;AACrB,OAAO,SAASU,gBAAgBrB,EAAgB,EAAEC,UAAsB,EAAEU,SAAsD,EAAEW,QAAkB,EAAEC,YAAyB,EAAEjB,KAAa,EAAEC,MAAc;IAC5M,IAAII,UAAUa,MAAM,KAAK,GAAG;IAC5B,MAAMC,QAAQ7B,UAAUI;IACxB,IAAIyB,MAAMD,MAAM,KAAK,GAAG;IACxB,yFAAyF;IACzF,uFAAuF;IACvF,MAAME,SAAS,IAAIC,IAAIF,MAAMG,IAAI;IACjC,MAAMC,QAAQ,IAAIC,IAAInB,UAAUoB,GAAG,CAAC,CAAChB,GAAGC,IAAM;YAACD,EAAEG,IAAI;YAAE,IAAKF,CAAAA,IAAI,CAAA;SAAG;IACnE,MAAMgB,SAAS;WAAInC,iBAAiByB,UAAUG,OAAOI;KAAO,CACzDI,MAAM,CAAC,CAAC,CAACf,MAAMC,MAAM,GAAKA,QAAQ,QAAQI,aAAaW,GAAG,CAAChB,OAC3DiB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGhC;IACZ0B,OAAOlB,OAAO,CAAC,CAAC,CAACI,KAAK,EAAEF;QACtB,MAAMuB,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpC,IAAIU,OAAOQ,GAAG,CAAChB,OAAOqB,SAASnB,GAAG,GAAG;QACvC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAO;QAClE;IACF;AACF;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,kBAAkB;AAClB,OAAO,eAAeqB,kBAAkBzC,EAAgB,EAAE0C,GAAmB,EAAEzC,UAAsB,EAAE0C,KAAa,EAAErC,KAAa,EAAEiB,YAAyB,EAAEhB,MAAc;IAC5K,MAAMqC,aAAa,IAAId;IACvB,MAAMe,kBAAkB,IAAIf;IAC5B,MAAMgB,MAAM,MAAMnD,mBAAmBK,IAAI0C,KAAKC,OAAOrC,OAAOiB;IAC5DuB,IAAIhC,OAAO,CAAC,CAAC,EAAEI,IAAI,EAAE6B,KAAK,EAAEC,UAAU,EAAE,EAAEhC;QACxC4B,WAAW3B,GAAG,CAACC,MAAM6B;QACrBF,gBAAgB5B,GAAG,CAACC,MAAM8B;QAC1B,MAAMT,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpCuB,SAASnB,GAAG,GAAG,GAAGmB,SAASnB,GAAG,CAAC,OAAO,CAAC;QACzC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAS;QACpE;IACF;IACA,OAAO;QAAEwB;QAAYC;IAAgB;AACvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/signals.ts"],"sourcesContent":["import type { ResolvedConfig } from '../config/index.ts';\nimport { semanticCandidates } from '../embed/query.ts';\nimport { LINK_EDGES_SQL, toEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph/graph.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\n\n// RRF composes whatever the preset declares; the constant lives here since it belongs to\n// the fusion these three functions share, not to search() itself.\nconst RRF_K = 60;\n\nexport type Candidates = Map<string, { score: number; via: string }>;\n\n// BM25 match rows (store.lexical.query), folded into `candidates` by reciprocal rank scaled by\n// the preset's declared weight for this signal (default 1, bit-identical to the unweighted\n// score). Every other signal is seeded from its output, so search() runs this one first when\n// \"words\" is declared.\nexport async function wordsCandidates(store: Store, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Promise<LexicalHit[]> {\n  const matchRows = await store.lexical.query(query, { whereJoin, whereCond, scopeCond, limit: fetch });\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: weight / (RRF_K + i), via: 'match' });\n  });\n  return matchRows;\n}\n\n// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this\n// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before\n// this was a signal.\nexport async function linksCandidates(store: Store, candidates: Candidates, matchRows: LexicalHit[], allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): Promise<void> {\n  if (matchRows.length === 0) return;\n  const stmt = await store.prepare(LINK_EDGES_SQL);\n  const edges = toEdges((await stmt.all()) as Array<{ src: string; dst: string }>);\n  if (edges.length === 0) return;\n  // Gates the label only, not the score: restart mass ranks every seed without an incident\n  // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n  const linked = new Set(edges.flat());\n  const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n  const ranked = [...personalizedRank(allPaths, edges, seeds)]\n    .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, fetch);\n  ranked.forEach(([path], i) => {\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      if (linked.has(path)) existing.via = 'match+link';\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'link' });\n    }\n  });\n}\n\n// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`\n// at this signal's weight. Each row also carries its best chunk's line range and similarity,\n// for the caller.\nexport async function vectorsCandidates(store: Store, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{ chunkLines: Map<string, string>; chunkSimilarity: Map<string, number> }> {\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  const vec = await semanticCandidates(store, cfg, terms, fetch, allowedPaths);\n  vec.forEach(({ path, lines, similarity }, i) => {\n    chunkLines.set(path, lines);\n    chunkSimilarity.set(path, similarity);\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      existing.via = `${existing.via}+vector`;\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'vector' });\n    }\n  });\n  return { chunkLines, chunkSimilarity };\n}\n"],"names":["semanticCandidates","LINK_EDGES_SQL","toEdges","personalizedRank","RRF_K","wordsCandidates","store","candidates","query","whereJoin","whereCond","scopeCond","fetch","weight","matchRows","lexical","limit","forEach","r","i","set","path","score","via","linksCandidates","allPaths","allowedPaths","length","stmt","prepare","edges","all","linked","Set","flat","seeds","Map","map","ranked","filter","has","sort","a","b","slice","existing","get","vectorsCandidates","cfg","terms","chunkLines","chunkSimilarity","vec","lines","similarity"],"mappings":"AACA,SAASA,kBAAkB,QAAQ,oBAAoB;AACvD,SAASC,cAAc,EAAEC,OAAO,QAAQ,uBAAuB;AAC/D,SAASC,gBAAgB,QAAQ,oBAAoB;AAGrD,yFAAyF;AACzF,kEAAkE;AAClE,MAAMC,QAAQ;AAId,+FAA+F;AAC/F,2FAA2F;AAC3F,6FAA6F;AAC7F,uBAAuB;AACvB,OAAO,eAAeC,gBAAgBC,KAAY,EAAEC,UAAsB,EAAEC,KAAa,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,MAAc;IAC/K,MAAMC,YAAY,MAAMR,MAAMS,OAAO,CAACP,KAAK,CAACA,OAAO;QAAEC;QAAWC;QAAWC;QAAWK,OAAOJ;IAAM;IACnGE,UAAUG,OAAO,CAAC,CAACC,GAAGC;QACpBZ,WAAWa,GAAG,CAACF,EAAEG,IAAI,EAAE;YAAEC,OAAOT,SAAUT,CAAAA,QAAQe,CAAAA;YAAII,KAAK;QAAQ;IACrE;IACA,OAAOT;AACT;AAEA,8FAA8F;AAC9F,gGAAgG;AAChG,qBAAqB;AACrB,OAAO,eAAeU,gBAAgBlB,KAAY,EAAEC,UAAsB,EAAEO,SAAuB,EAAEW,QAAkB,EAAEC,YAAyB,EAAEd,KAAa,EAAEC,MAAc;IAC/K,IAAIC,UAAUa,MAAM,KAAK,GAAG;IAC5B,MAAMC,OAAO,MAAMtB,MAAMuB,OAAO,CAAC5B;IACjC,MAAM6B,QAAQ5B,QAAS,MAAM0B,KAAKG,GAAG;IACrC,IAAID,MAAMH,MAAM,KAAK,GAAG;IACxB,yFAAyF;IACzF,uFAAuF;IACvF,MAAMK,SAAS,IAAIC,IAAIH,MAAMI,IAAI;IACjC,MAAMC,QAAQ,IAAIC,IAAItB,UAAUuB,GAAG,CAAC,CAACnB,GAAGC,IAAM;YAACD,EAAEG,IAAI;YAAE,IAAKF,CAAAA,IAAI,CAAA;SAAG;IACnE,MAAMmB,SAAS;WAAInC,iBAAiBsB,UAAUK,OAAOK;KAAO,CACzDI,MAAM,CAAC,CAAC,CAAClB,MAAMC,MAAM,GAAKA,QAAQ,QAAQI,aAAac,GAAG,CAACnB,OAC3DoB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGhC;IACZ0B,OAAOrB,OAAO,CAAC,CAAC,CAACI,KAAK,EAAEF;QACtB,MAAM0B,WAAWtC,WAAWuC,GAAG,CAACzB;QAChC,IAAIwB,UAAU;YACZA,SAASvB,KAAK,IAAIT,SAAUT,CAAAA,QAAQe,CAAAA;YACpC,IAAIa,OAAOQ,GAAG,CAACnB,OAAOwB,SAAStB,GAAG,GAAG;QACvC,OAAO;YACLhB,WAAWa,GAAG,CAACC,MAAM;gBAAEC,OAAOT,SAAUT,CAAAA,QAAQe,CAAAA;gBAAII,KAAK;YAAO;QAClE;IACF;AACF;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,kBAAkB;AAClB,OAAO,eAAewB,kBAAkBzC,KAAY,EAAE0C,GAAmB,EAAEzC,UAAsB,EAAE0C,KAAa,EAAErC,KAAa,EAAEc,YAAyB,EAAEb,MAAc;IACxK,MAAMqC,aAAa,IAAId;IACvB,MAAMe,kBAAkB,IAAIf;IAC5B,MAAMgB,MAAM,MAAMpD,mBAAmBM,OAAO0C,KAAKC,OAAOrC,OAAOc;IAC/D0B,IAAInC,OAAO,CAAC,CAAC,EAAEI,IAAI,EAAEgC,KAAK,EAAEC,UAAU,EAAE,EAAEnC;QACxC+B,WAAW9B,GAAG,CAACC,MAAMgC;QACrBF,gBAAgB/B,GAAG,CAACC,MAAMiC;QAC1B,MAAMT,WAAWtC,WAAWuC,GAAG,CAACzB;QAChC,IAAIwB,UAAU;YACZA,SAASvB,KAAK,IAAIT,SAAUT,CAAAA,QAAQe,CAAAA;YACpC0B,SAAStB,GAAG,GAAG,GAAGsB,SAAStB,GAAG,CAAC,OAAO,CAAC;QACzC,OAAO;YACLhB,WAAWa,GAAG,CAACC,MAAM;gBAAEC,OAAOT,SAAUT,CAAAA,QAAQe,CAAAA;gBAAII,KAAK;YAAS;QACpE;IACF;IACA,OAAO;QAAE2B;QAAYC;IAAgB;AACvC"}

package/dist/esm/commands/status.d.ts

@@ -1,9 +1,9 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig, SignalWeights } from '../config/index.js';
+import type { Store } from '../store/types.js';
 export interface PresetCoverage {
     name: string;
     files: number;
     embedded: number;
     signals: SignalWeights;
 }
-export declare function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[];
+export declare function presetCoverage(store: Store, cfg: ResolvedConfig): Promise<PresetCoverage[]>;

package/dist/esm/commands/status.js

@@ -1,16 +1,22 @@
 import { anyPresetEmbeds, presetNames, presetSignals } from '../config/index.js';
 // Indexing derives from presets, so the derivation stays visible. Read from preset_files, not
 // recomputed from globs, so it reflects the cache rather than the config.
-export function presetCoverage(db, cfg) {
+export async function presetCoverage(store, cfg) {
     const embedActive = anyPresetEmbeds(cfg);
-    return presetNames(cfg).map((name)=>{
-        const files = db.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?').get(name).n;
-        const embedded = embedActive ? db.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e."path" = pf."path" AND e.vector IS NOT NULL)').get(name).n : 0;
-        return {
+    const filesStmt = await store.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?');
+    const embeddedStmt = embedActive ? await store.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e."path" = pf."path" AND e.vector IS NOT NULL)') : null;
+    // Sequential, not Promise.all: both statements are reused across presets, and a store's
+    // prepared statement is not safe to bind/run concurrently with itself.
+    const rows = [];
+    for (const name of presetNames(cfg)){
+        const files = (await filesStmt.get(name)).n;
+        const embedded = embeddedStmt ? (await embeddedStmt.get(name)).n : 0;
+        rows.push({
             name,
             files,
             embedded,
             signals: presetSignals(cfg, name)
-        };
-    });
+        });
+    }
+    return rows;
 }

package/dist/esm/commands/status.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig, SignalWeights } from '../config/index.ts';\nimport { anyPresetEmbeds, presetNames, presetSignals } from '../config/index.ts';\n\nexport interface PresetCoverage {\n  name: string;\n  files: number;\n  embedded: number;\n  // Reported so 0 embedded reads as \"this scope declined vectors\" rather than \"not yet built\".\n  signals: SignalWeights;\n}\n\n// Indexing derives from presets, so the derivation stays visible. Read from preset_files, not\n// recomputed from globs, so it reflects the cache rather than the config.\nexport function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[] {\n  const embedActive = anyPresetEmbeds(cfg);\n  return presetNames(cfg).map((name) => {\n    const files = (db.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?').get(name) as { n: number }).n;\n    const embedded = embedActive ? (db.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e.\"path\" = pf.\"path\" AND e.vector IS NOT NULL)').get(name) as { n: number }).n : 0;\n    return { name, files, embedded, signals: presetSignals(cfg, name) };\n  });\n}\n"],"names":["anyPresetEmbeds","presetNames","presetSignals","presetCoverage","db","cfg","embedActive","map","name","files","prepare","get","n","embedded","signals"],"mappings":"AAEA,SAASA,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,qBAAqB;AAUjF,8FAA8F;AAC9F,0EAA0E;AAC1E,OAAO,SAASC,eAAeC,EAAgB,EAAEC,GAAmB;IAClE,MAAMC,cAAcN,gBAAgBK;IACpC,OAAOJ,YAAYI,KAAKE,GAAG,CAAC,CAACC;QAC3B,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,2DAA2DC,GAAG,CAACH,MAAwBI,CAAC;QAClH,MAAMC,WAAWP,cAAc,AAACF,GAAGM,OAAO,CAAC,6JAA6JC,GAAG,CAACH,MAAwBI,CAAC,GAAG;QACxO,OAAO;YAAEJ;YAAMC;YAAOI;YAAUC,SAASZ,cAAcG,KAAKG;QAAM;IACpE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import type { ResolvedConfig, SignalWeights } from '../config/index.ts';\nimport { anyPresetEmbeds, presetNames, presetSignals } from '../config/index.ts';\nimport type { Store } from '../store/types.ts';\n\nexport interface PresetCoverage {\n  name: string;\n  files: number;\n  embedded: number;\n  // Reported so 0 embedded reads as \"this scope declined vectors\" rather than \"not yet built\".\n  signals: SignalWeights;\n}\n\n// Indexing derives from presets, so the derivation stays visible. Read from preset_files, not\n// recomputed from globs, so it reflects the cache rather than the config.\nexport async function presetCoverage(store: Store, cfg: ResolvedConfig): Promise<PresetCoverage[]> {\n  const embedActive = anyPresetEmbeds(cfg);\n  const filesStmt = await store.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?');\n  const embeddedStmt = embedActive ? await store.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e.\"path\" = pf.\"path\" AND e.vector IS NOT NULL)') : null;\n  // Sequential, not Promise.all: both statements are reused across presets, and a store's\n  // prepared statement is not safe to bind/run concurrently with itself.\n  const rows: PresetCoverage[] = [];\n  for (const name of presetNames(cfg)) {\n    const files = ((await filesStmt.get(name)) as { n: number }).n;\n    const embedded = embeddedStmt ? ((await embeddedStmt.get(name)) as { n: number }).n : 0;\n    rows.push({ name, files, embedded, signals: presetSignals(cfg, name) });\n  }\n  return rows;\n}\n"],"names":["anyPresetEmbeds","presetNames","presetSignals","presetCoverage","store","cfg","embedActive","filesStmt","prepare","embeddedStmt","rows","name","files","get","n","embedded","push","signals"],"mappings":"AACA,SAASA,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,qBAAqB;AAWjF,8FAA8F;AAC9F,0EAA0E;AAC1E,OAAO,eAAeC,eAAeC,KAAY,EAAEC,GAAmB;IACpE,MAAMC,cAAcN,gBAAgBK;IACpC,MAAME,YAAY,MAAMH,MAAMI,OAAO,CAAC;IACtC,MAAMC,eAAeH,cAAc,MAAMF,MAAMI,OAAO,CAAC,+JAA+J;IACtN,wFAAwF;IACxF,uEAAuE;IACvE,MAAME,OAAyB,EAAE;IACjC,KAAK,MAAMC,QAAQV,YAAYI,KAAM;QACnC,MAAMO,QAAQ,AAAE,CAAA,MAAML,UAAUM,GAAG,CAACF,KAAI,EAAqBG,CAAC;QAC9D,MAAMC,WAAWN,eAAe,AAAE,CAAA,MAAMA,aAAaI,GAAG,CAACF,KAAI,EAAqBG,CAAC,GAAG;QACtFJ,KAAKM,IAAI,CAAC;YAAEL;YAAMC;YAAOG;YAAUE,SAASf,cAAcG,KAAKM;QAAM;IACvE;IACA,OAAOD;AACT"}

package/dist/esm/config/access.d.ts

@@ -2,6 +2,7 @@ import type { Feature } from '../features/types.js';
 import type { SignalName, SignalWeights } from './signals.js';
 import { type Config, type FeatureName } from './types.js';
 export declare function presetNames(cfg: Config): string[];
+export declare function storeName(cfg: Config): 'sqlite' | 'duckdb';
 export declare function embedEnabled(cfg: Config): boolean;
 export declare function presetSignals(cfg: Config, name: string): SignalWeights;
 export declare function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean;

package/dist/esm/config/access.js

@@ -2,6 +2,11 @@ import { FEATURE_NAMES } from './types.js';
 export function presetNames(cfg) {
     return Object.keys(cfg.presets);
 }
+// The backing store engine this tree uses; absent means the default.
+export function storeName(cfg) {
+    var _cfg_store;
+    return (_cfg_store = cfg.store) !== null && _cfg_store !== void 0 ? _cfg_store : 'sqlite';
+}
 // Whether the tree names a model at all.
 export function embedEnabled(cfg) {
     var _cfg_embed;

package/dist/esm/config/access.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/access.ts"],"sourcesContent":["import type { Feature } from '../features/types.ts';\nimport type { SignalName, SignalWeights } from './signals.ts';\nimport { type Config, type EmbedConfig, FEATURE_NAMES, type FeatureName } from './types.ts';\n\nexport function presetNames(cfg: Config): string[] {\n  return Object.keys(cfg.presets);\n}\n\n// Whether the tree names a model at all.\nexport function embedEnabled(cfg: Config): boolean {\n  return typeof cfg.embed?.model === 'string' && cfg.embed.model.length > 0;\n}\n\n// A preset's effective signal weights: declared exhaustively when `signals` is present;\n// otherwise every signal whose prerequisite holds, each at weight 1 -- words always, links when\n// the feature is on, vectors when the tree names a model. Single source search() and status/map\n// read.\nexport function presetSignals(cfg: Config, name: string): SignalWeights {\n  const declared = cfg.presets[name]?.signals;\n  if (declared) return declared;\n  const signals: SignalWeights = { words: 1 };\n  if (cfg.features?.links !== false) signals.links = 1;\n  if (embedEnabled(cfg)) signals.vectors = 1;\n  return signals;\n}\n\nexport function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean {\n  return presetSignals(cfg, name)[signal] !== undefined;\n}\n\n// Whether embedding needs to run: a model is named AND some preset's effective signals include vectors.\nexport function anyPresetEmbeds(cfg: Config): boolean {\n  return presetNames(cfg).some((name) => presetHasSignal(cfg, name, 'vectors'));\n}\n\n// Opt-out features (default on): absent block or key means enabled. `rank` additionally\n// requires `links`. `embed` is derived, not a features-block member: on iff some preset's\n// effective signals include vectors.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  if (name === 'embed') return anyPresetEmbeds(cfg);\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Feature states, so \"off\" and \"empty\" stay distinguishable in output. `embed` is excluded:\n// it is not a features-block toggle, and `status` reports it on its own line.\nexport function featureStates(cfg: Config): { on: FeatureName[]; off: FeatureName[] } {\n  const reported = FEATURE_NAMES.filter((name) => name !== 'embed');\n  return {\n    on: reported.filter((name) => featureEnabled(cfg, name)),\n    off: reported.filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n\n// Resolved embed provider settings, or null when the config names no model. There is no\n// fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration\n// write into the file, never an implicit runtime default.\nexport function embedConfig(cfg: Config): { model: string; provider: 'static' | 'openai' | 'cohere'; url?: string; key?: string; languages?: string[]; chunkTokens?: number } | null {\n  if (!embedEnabled(cfg)) return null;\n  const e = cfg.embed as EmbedConfig;\n  return { model: e.model as string, provider: e.provider ?? 'static', url: e.url, key: e.key, languages: e.languages, chunkTokens: e.chunkTokens };\n}\n\n// The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than\n// the default string so featureSignature can leave the segment off entirely for trees that\n// never set it, which is what keeps existing indexes from rebuilding on upgrade.\nexport function contentTokenize(cfg: Config): string | undefined {\n  const tokenize = cfg.content?.tokenize;\n  return tokenize === undefined || tokenize.trim() === '' ? undefined : tokenize.trim();\n}\n\n// Cache key over everything indexing derives from, so any change to those inputs rebuilds.\n// Segment order is fixed (features list, then `features` in registry order, then tokenize, then presets) since reordering alone would fire a spurious rebuild; tokenize/presets stay here because no Feature module owns either.\nexport function featureSignature(cfg: Config, features: Feature[]): string {\n  const globalPart = enabledFeatures(cfg)\n    .filter((name) => name !== 'embed')\n    .join(',');\n  const featureParts = features.map((feature) => feature.signature?.(cfg)).filter((part): part is string => part !== undefined);\n  // One keyed segment per preset so a rebuild notice can name exactly which preset moved.\n  const presetsPart = [...presetNames(cfg)]\n    .sort()\n    .map((name) => {\n      const p = cfg.presets[name];\n      const include = [...p.include].sort().join('+');\n      const exclude = [...(p.exclude ?? [])].sort().join('+');\n      return `preset:${name}:${include}:${exclude}:${presetHasSignal(cfg, name, 'vectors') ? 'on' : 'off'}`;\n    })\n    .join('|');\n  const tokenize = contentTokenize(cfg);\n  const parts = [`features:${globalPart}`, ...featureParts];\n  if (tokenize !== undefined) parts.push(`tokenize:${tokenize}`);\n  parts.push(presetsPart);\n  return parts.join('|');\n}\n"],"names":["FEATURE_NAMES","presetNames","cfg","Object","keys","presets","embedEnabled","embed","model","length","presetSignals","name","declared","signals","words","features","links","vectors","presetHasSignal","signal","undefined","anyPresetEmbeds","some","featureEnabled","enabled","enabledFeatures","filter","featureStates","reported","on","off","embedConfig","e","provider","url","key","languages","chunkTokens","contentTokenize","tokenize","content","trim","featureSignature","globalPart","join","featureParts","map","feature","signature","part","presetsPart","sort","p","include","exclude","parts","push"],"mappings":"AAEA,SAAwCA,aAAa,QAA0B,aAAa;AAE5F,OAAO,SAASC,YAAYC,GAAW;IACrC,OAAOC,OAAOC,IAAI,CAACF,IAAIG,OAAO;AAChC;AAEA,yCAAyC;AACzC,OAAO,SAASC,aAAaJ,GAAW;QACxBA;IAAd,OAAO,SAAOA,aAAAA,IAAIK,KAAK,cAATL,iCAAAA,WAAWM,KAAK,MAAK,YAAYN,IAAIK,KAAK,CAACC,KAAK,CAACC,MAAM,GAAG;AAC1E;AAEA,wFAAwF;AACxF,gGAAgG;AAChG,gGAAgG;AAChG,QAAQ;AACR,OAAO,SAASC,cAAcR,GAAW,EAAES,IAAY;QACpCT,mBAGbA;IAHJ,MAAMU,YAAWV,oBAAAA,IAAIG,OAAO,CAACM,KAAK,cAAjBT,wCAAAA,kBAAmBW,OAAO;IAC3C,IAAID,UAAU,OAAOA;IACrB,MAAMC,UAAyB;QAAEC,OAAO;IAAE;IAC1C,IAAIZ,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,cAAcc,KAAK,MAAK,OAAOH,QAAQG,KAAK,GAAG;IACnD,IAAIV,aAAaJ,MAAMW,QAAQI,OAAO,GAAG;IACzC,OAAOJ;AACT;AAEA,OAAO,SAASK,gBAAgBhB,GAAW,EAAES,IAAY,EAAEQ,MAAkB;IAC3E,OAAOT,cAAcR,KAAKS,KAAK,CAACQ,OAAO,KAAKC;AAC9C;AAEA,wGAAwG;AACxG,OAAO,SAASC,gBAAgBnB,GAAW;IACzC,OAAOD,YAAYC,KAAKoB,IAAI,CAAC,CAACX,OAASO,gBAAgBhB,KAAKS,MAAM;AACpE;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,qCAAqC;AACrC,OAAO,SAASY,eAAerB,GAAW,EAAES,IAAiB;QAE3CT;IADhB,IAAIS,SAAS,SAAS,OAAOU,gBAAgBnB;IAC7C,MAAMsB,UAAUtB,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,aAAc,CAACS,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOa,WAAWD,eAAerB,KAAK;IAC3D,OAAOsB;AACT;AAEA,OAAO,SAASC,gBAAgBvB,GAAW;IACzC,OAAOF,cAAc0B,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;AAC5D;AAEA,4FAA4F;AAC5F,8EAA8E;AAC9E,OAAO,SAASgB,cAAczB,GAAW;IACvC,MAAM0B,WAAW5B,cAAc0B,MAAM,CAAC,CAACf,OAASA,SAAS;IACzD,OAAO;QACLkB,IAAID,SAASF,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;QAClDmB,KAAKF,SAASF,MAAM,CAAC,CAACf,OAAS,CAACY,eAAerB,KAAKS;IACtD;AACF;AAEA,wFAAwF;AACxF,2FAA2F;AAC3F,0DAA0D;AAC1D,OAAO,SAASoB,YAAY7B,GAAW;QAGQ8B;IAF7C,IAAI,CAAC1B,aAAaJ,MAAM,OAAO;IAC/B,MAAM8B,IAAI9B,IAAIK,KAAK;IACnB,OAAO;QAAEC,OAAOwB,EAAExB,KAAK;QAAYyB,QAAQ,GAAED,cAAAA,EAAEC,QAAQ,cAAVD,yBAAAA,cAAc;QAAUE,KAAKF,EAAEE,GAAG;QAAEC,KAAKH,EAAEG,GAAG;QAAEC,WAAWJ,EAAEI,SAAS;QAAEC,aAAaL,EAAEK,WAAW;IAAC;AAClJ;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,SAASC,gBAAgBpC,GAAW;QACxBA;IAAjB,MAAMqC,YAAWrC,eAAAA,IAAIsC,OAAO,cAAXtC,mCAAAA,aAAaqC,QAAQ;IACtC,OAAOA,aAAanB,aAAamB,SAASE,IAAI,OAAO,KAAKrB,YAAYmB,SAASE,IAAI;AACrF;AAEA,2FAA2F;AAC3F,iOAAiO;AACjO,OAAO,SAASC,iBAAiBxC,GAAW,EAAEa,QAAmB;IAC/D,MAAM4B,aAAalB,gBAAgBvB,KAChCwB,MAAM,CAAC,CAACf,OAASA,SAAS,SAC1BiC,IAAI,CAAC;IACR,MAAMC,eAAe9B,SAAS+B,GAAG,CAAC,CAACC;YAAYA;gBAAAA,qBAAAA,QAAQC,SAAS,cAAjBD,yCAAAA,wBAAAA,SAAoB7C;OAAMwB,MAAM,CAAC,CAACuB,OAAyBA,SAAS7B;IACnH,wFAAwF;IACxF,MAAM8B,cAAc;WAAIjD,YAAYC;KAAK,CACtCiD,IAAI,GACJL,GAAG,CAAC,CAACnC;YAGiByC;QAFrB,MAAMA,IAAIlD,IAAIG,OAAO,CAACM,KAAK;QAC3B,MAAM0C,UAAU;eAAID,EAAEC,OAAO;SAAC,CAACF,IAAI,GAAGP,IAAI,CAAC;QAC3C,MAAMU,UAAU;gBAAKF,aAAAA,EAAEE,OAAO,cAATF,wBAAAA,aAAa,EAAE;SAAE,CAACD,IAAI,GAAGP,IAAI,CAAC;QACnD,OAAO,CAAC,OAAO,EAAEjC,KAAK,CAAC,EAAE0C,QAAQ,CAAC,EAAEC,QAAQ,CAAC,EAAEpC,gBAAgBhB,KAAKS,MAAM,aAAa,OAAO,OAAO;IACvG,GACCiC,IAAI,CAAC;IACR,MAAML,WAAWD,gBAAgBpC;IACjC,MAAMqD,QAAQ;QAAC,CAAC,SAAS,EAAEZ,YAAY;WAAKE;KAAa;IACzD,IAAIN,aAAanB,WAAWmC,MAAMC,IAAI,CAAC,CAAC,SAAS,EAAEjB,UAAU;IAC7DgB,MAAMC,IAAI,CAACN;IACX,OAAOK,MAAMX,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/access.ts"],"sourcesContent":["import type { Feature } from '../features/types.ts';\nimport type { SignalName, SignalWeights } from './signals.ts';\nimport { type Config, type EmbedConfig, FEATURE_NAMES, type FeatureName } from './types.ts';\n\nexport function presetNames(cfg: Config): string[] {\n  return Object.keys(cfg.presets);\n}\n\n// The backing store engine this tree uses; absent means the default.\nexport function storeName(cfg: Config): 'sqlite' | 'duckdb' {\n  return cfg.store ?? 'sqlite';\n}\n\n// Whether the tree names a model at all.\nexport function embedEnabled(cfg: Config): boolean {\n  return typeof cfg.embed?.model === 'string' && cfg.embed.model.length > 0;\n}\n\n// A preset's effective signal weights: declared exhaustively when `signals` is present;\n// otherwise every signal whose prerequisite holds, each at weight 1 -- words always, links when\n// the feature is on, vectors when the tree names a model. Single source search() and status/map\n// read.\nexport function presetSignals(cfg: Config, name: string): SignalWeights {\n  const declared = cfg.presets[name]?.signals;\n  if (declared) return declared;\n  const signals: SignalWeights = { words: 1 };\n  if (cfg.features?.links !== false) signals.links = 1;\n  if (embedEnabled(cfg)) signals.vectors = 1;\n  return signals;\n}\n\nexport function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean {\n  return presetSignals(cfg, name)[signal] !== undefined;\n}\n\n// Whether embedding needs to run: a model is named AND some preset's effective signals include vectors.\nexport function anyPresetEmbeds(cfg: Config): boolean {\n  return presetNames(cfg).some((name) => presetHasSignal(cfg, name, 'vectors'));\n}\n\n// Opt-out features (default on): absent block or key means enabled. `rank` additionally\n// requires `links`. `embed` is derived, not a features-block member: on iff some preset's\n// effective signals include vectors.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  if (name === 'embed') return anyPresetEmbeds(cfg);\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Feature states, so \"off\" and \"empty\" stay distinguishable in output. `embed` is excluded:\n// it is not a features-block toggle, and `status` reports it on its own line.\nexport function featureStates(cfg: Config): { on: FeatureName[]; off: FeatureName[] } {\n  const reported = FEATURE_NAMES.filter((name) => name !== 'embed');\n  return {\n    on: reported.filter((name) => featureEnabled(cfg, name)),\n    off: reported.filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n\n// Resolved embed provider settings, or null when the config names no model. There is no\n// fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration\n// write into the file, never an implicit runtime default.\nexport function embedConfig(cfg: Config): { model: string; provider: 'static' | 'openai' | 'cohere'; url?: string; key?: string; languages?: string[]; chunkTokens?: number } | null {\n  if (!embedEnabled(cfg)) return null;\n  const e = cfg.embed as EmbedConfig;\n  return { model: e.model as string, provider: e.provider ?? 'static', url: e.url, key: e.key, languages: e.languages, chunkTokens: e.chunkTokens };\n}\n\n// The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than\n// the default string so featureSignature can leave the segment off entirely for trees that\n// never set it, which is what keeps existing indexes from rebuilding on upgrade.\nexport function contentTokenize(cfg: Config): string | undefined {\n  const tokenize = cfg.content?.tokenize;\n  return tokenize === undefined || tokenize.trim() === '' ? undefined : tokenize.trim();\n}\n\n// Cache key over everything indexing derives from, so any change to those inputs rebuilds.\n// Segment order is fixed (features list, then `features` in registry order, then tokenize, then presets) since reordering alone would fire a spurious rebuild; tokenize/presets stay here because no Feature module owns either.\nexport function featureSignature(cfg: Config, features: Feature[]): string {\n  const globalPart = enabledFeatures(cfg)\n    .filter((name) => name !== 'embed')\n    .join(',');\n  const featureParts = features.map((feature) => feature.signature?.(cfg)).filter((part): part is string => part !== undefined);\n  // One keyed segment per preset so a rebuild notice can name exactly which preset moved.\n  const presetsPart = [...presetNames(cfg)]\n    .sort()\n    .map((name) => {\n      const p = cfg.presets[name];\n      const include = [...p.include].sort().join('+');\n      const exclude = [...(p.exclude ?? [])].sort().join('+');\n      return `preset:${name}:${include}:${exclude}:${presetHasSignal(cfg, name, 'vectors') ? 'on' : 'off'}`;\n    })\n    .join('|');\n  const tokenize = contentTokenize(cfg);\n  const parts = [`features:${globalPart}`, ...featureParts];\n  if (tokenize !== undefined) parts.push(`tokenize:${tokenize}`);\n  parts.push(presetsPart);\n  return parts.join('|');\n}\n"],"names":["FEATURE_NAMES","presetNames","cfg","Object","keys","presets","storeName","store","embedEnabled","embed","model","length","presetSignals","name","declared","signals","words","features","links","vectors","presetHasSignal","signal","undefined","anyPresetEmbeds","some","featureEnabled","enabled","enabledFeatures","filter","featureStates","reported","on","off","embedConfig","e","provider","url","key","languages","chunkTokens","contentTokenize","tokenize","content","trim","featureSignature","globalPart","join","featureParts","map","feature","signature","part","presetsPart","sort","p","include","exclude","parts","push"],"mappings":"AAEA,SAAwCA,aAAa,QAA0B,aAAa;AAE5F,OAAO,SAASC,YAAYC,GAAW;IACrC,OAAOC,OAAOC,IAAI,CAACF,IAAIG,OAAO;AAChC;AAEA,qEAAqE;AACrE,OAAO,SAASC,UAAUJ,GAAW;QAC5BA;IAAP,QAAOA,aAAAA,IAAIK,KAAK,cAATL,wBAAAA,aAAa;AACtB;AAEA,yCAAyC;AACzC,OAAO,SAASM,aAAaN,GAAW;QACxBA;IAAd,OAAO,SAAOA,aAAAA,IAAIO,KAAK,cAATP,iCAAAA,WAAWQ,KAAK,MAAK,YAAYR,IAAIO,KAAK,CAACC,KAAK,CAACC,MAAM,GAAG;AAC1E;AAEA,wFAAwF;AACxF,gGAAgG;AAChG,gGAAgG;AAChG,QAAQ;AACR,OAAO,SAASC,cAAcV,GAAW,EAAEW,IAAY;QACpCX,mBAGbA;IAHJ,MAAMY,YAAWZ,oBAAAA,IAAIG,OAAO,CAACQ,KAAK,cAAjBX,wCAAAA,kBAAmBa,OAAO;IAC3C,IAAID,UAAU,OAAOA;IACrB,MAAMC,UAAyB;QAAEC,OAAO;IAAE;IAC1C,IAAId,EAAAA,gBAAAA,IAAIe,QAAQ,cAAZf,oCAAAA,cAAcgB,KAAK,MAAK,OAAOH,QAAQG,KAAK,GAAG;IACnD,IAAIV,aAAaN,MAAMa,QAAQI,OAAO,GAAG;IACzC,OAAOJ;AACT;AAEA,OAAO,SAASK,gBAAgBlB,GAAW,EAAEW,IAAY,EAAEQ,MAAkB;IAC3E,OAAOT,cAAcV,KAAKW,KAAK,CAACQ,OAAO,KAAKC;AAC9C;AAEA,wGAAwG;AACxG,OAAO,SAASC,gBAAgBrB,GAAW;IACzC,OAAOD,YAAYC,KAAKsB,IAAI,CAAC,CAACX,OAASO,gBAAgBlB,KAAKW,MAAM;AACpE;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,qCAAqC;AACrC,OAAO,SAASY,eAAevB,GAAW,EAAEW,IAAiB;QAE3CX;IADhB,IAAIW,SAAS,SAAS,OAAOU,gBAAgBrB;IAC7C,MAAMwB,UAAUxB,EAAAA,gBAAAA,IAAIe,QAAQ,cAAZf,oCAAAA,aAAc,CAACW,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOa,WAAWD,eAAevB,KAAK;IAC3D,OAAOwB;AACT;AAEA,OAAO,SAASC,gBAAgBzB,GAAW;IACzC,OAAOF,cAAc4B,MAAM,CAAC,CAACf,OAASY,eAAevB,KAAKW;AAC5D;AAEA,4FAA4F;AAC5F,8EAA8E;AAC9E,OAAO,SAASgB,cAAc3B,GAAW;IACvC,MAAM4B,WAAW9B,cAAc4B,MAAM,CAAC,CAACf,OAASA,SAAS;IACzD,OAAO;QACLkB,IAAID,SAASF,MAAM,CAAC,CAACf,OAASY,eAAevB,KAAKW;QAClDmB,KAAKF,SAASF,MAAM,CAAC,CAACf,OAAS,CAACY,eAAevB,KAAKW;IACtD;AACF;AAEA,wFAAwF;AACxF,2FAA2F;AAC3F,0DAA0D;AAC1D,OAAO,SAASoB,YAAY/B,GAAW;QAGQgC;IAF7C,IAAI,CAAC1B,aAAaN,MAAM,OAAO;IAC/B,MAAMgC,IAAIhC,IAAIO,KAAK;IACnB,OAAO;QAAEC,OAAOwB,EAAExB,KAAK;QAAYyB,QAAQ,GAAED,cAAAA,EAAEC,QAAQ,cAAVD,yBAAAA,cAAc;QAAUE,KAAKF,EAAEE,GAAG;QAAEC,KAAKH,EAAEG,GAAG;QAAEC,WAAWJ,EAAEI,SAAS;QAAEC,aAAaL,EAAEK,WAAW;IAAC;AAClJ;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,SAASC,gBAAgBtC,GAAW;QACxBA;IAAjB,MAAMuC,YAAWvC,eAAAA,IAAIwC,OAAO,cAAXxC,mCAAAA,aAAauC,QAAQ;IACtC,OAAOA,aAAanB,aAAamB,SAASE,IAAI,OAAO,KAAKrB,YAAYmB,SAASE,IAAI;AACrF;AAEA,2FAA2F;AAC3F,iOAAiO;AACjO,OAAO,SAASC,iBAAiB1C,GAAW,EAAEe,QAAmB;IAC/D,MAAM4B,aAAalB,gBAAgBzB,KAChC0B,MAAM,CAAC,CAACf,OAASA,SAAS,SAC1BiC,IAAI,CAAC;IACR,MAAMC,eAAe9B,SAAS+B,GAAG,CAAC,CAACC;YAAYA;gBAAAA,qBAAAA,QAAQC,SAAS,cAAjBD,yCAAAA,wBAAAA,SAAoB/C;OAAM0B,MAAM,CAAC,CAACuB,OAAyBA,SAAS7B;IACnH,wFAAwF;IACxF,MAAM8B,cAAc;WAAInD,YAAYC;KAAK,CACtCmD,IAAI,GACJL,GAAG,CAAC,CAACnC;YAGiByC;QAFrB,MAAMA,IAAIpD,IAAIG,OAAO,CAACQ,KAAK;QAC3B,MAAM0C,UAAU;eAAID,EAAEC,OAAO;SAAC,CAACF,IAAI,GAAGP,IAAI,CAAC;QAC3C,MAAMU,UAAU;gBAAKF,aAAAA,EAAEE,OAAO,cAATF,wBAAAA,aAAa,EAAE;SAAE,CAACD,IAAI,GAAGP,IAAI,CAAC;QACnD,OAAO,CAAC,OAAO,EAAEjC,KAAK,CAAC,EAAE0C,QAAQ,CAAC,EAAEC,QAAQ,CAAC,EAAEpC,gBAAgBlB,KAAKW,MAAM,aAAa,OAAO,OAAO;IACvG,GACCiC,IAAI,CAAC;IACR,MAAML,WAAWD,gBAAgBtC;IACjC,MAAMuD,QAAQ;QAAC,CAAC,SAAS,EAAEZ,YAAY;WAAKE;KAAa;IACzD,IAAIN,aAAanB,WAAWmC,MAAMC,IAAI,CAAC,CAAC,SAAS,EAAEjB,UAAU;IAC7DgB,MAAMC,IAAI,CAACN;IACX,OAAOK,MAAMX,IAAI,CAAC;AACpB"}

package/dist/esm/config/index.d.ts

@@ -1,4 +1,4 @@
-export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.js';
+export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals, storeName } from './access.js';
 export type { InitOverrides } from './load.js';
 export { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.js';
 export { resolvePreset, resolveSearch } from './resolve.js';

package/dist/esm/config/index.js

@@ -1,4 +1,4 @@
-export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.js';
+export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals, storeName } from './access.js';
 export { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.js';
 export { resolvePreset, resolveSearch } from './resolve.js';
 export { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.js';

package/dist/esm/config/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/index.ts"],"sourcesContent":["export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.ts';\nexport type { InitOverrides } from './load.ts';\nexport { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.ts';\nexport { resolvePreset, resolveSearch } from './resolve.ts';\nexport type { SignalName, SignalWeights } from './signals.ts';\nexport { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.ts';\nexport type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.ts';\nexport { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.ts';\nexport { KNOWN_EMBED_KEYS } from './validate.ts';\n"],"names":["anyPresetEmbeds","contentTokenize","embedConfig","embedEnabled","enabledFeatures","featureEnabled","featureSignature","featureStates","presetHasSignal","presetNames","presetSignals","findConfigPath","initConfig","loadConfig","migrateConfig","resolvePreset","resolveSearch","SIGNAL_NAMES","SIGNAL_PREREQUISITES","CONFIG_FILENAME","DEFAULT_EMBED_MODEL","STATE_DIR","SUPPORTED_CONFIG_VERSION","KNOWN_EMBED_KEYS"],"mappings":"AAAA,SAASA,eAAe,EAAEC,eAAe,EAAEC,WAAW,EAAEC,YAAY,EAAEC,eAAe,EAAEC,cAAc,EAAEC,gBAAgB,EAAEC,aAAa,EAAEC,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,cAAc;AAEzM,SAASC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,QAAQ,YAAY;AAClF,SAASC,aAAa,EAAEC,aAAa,QAAQ,eAAe;AAE5D,SAASC,YAAY,EAAEC,oBAAoB,QAAQ,eAAe;AAElE,SAASC,eAAe,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,aAAa;AACvG,SAASC,gBAAgB,QAAQ,gBAAgB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/index.ts"],"sourcesContent":["export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals, storeName } from './access.ts';\nexport type { InitOverrides } from './load.ts';\nexport { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.ts';\nexport { resolvePreset, resolveSearch } from './resolve.ts';\nexport type { SignalName, SignalWeights } from './signals.ts';\nexport { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.ts';\nexport type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.ts';\nexport { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.ts';\nexport { KNOWN_EMBED_KEYS } from './validate.ts';\n"],"names":["anyPresetEmbeds","contentTokenize","embedConfig","embedEnabled","enabledFeatures","featureEnabled","featureSignature","featureStates","presetHasSignal","presetNames","presetSignals","storeName","findConfigPath","initConfig","loadConfig","migrateConfig","resolvePreset","resolveSearch","SIGNAL_NAMES","SIGNAL_PREREQUISITES","CONFIG_FILENAME","DEFAULT_EMBED_MODEL","STATE_DIR","SUPPORTED_CONFIG_VERSION","KNOWN_EMBED_KEYS"],"mappings":"AAAA,SAASA,eAAe,EAAEC,eAAe,EAAEC,WAAW,EAAEC,YAAY,EAAEC,eAAe,EAAEC,cAAc,EAAEC,gBAAgB,EAAEC,aAAa,EAAEC,eAAe,EAAEC,WAAW,EAAEC,aAAa,EAAEC,SAAS,QAAQ,cAAc;AAEpN,SAASC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,QAAQ,YAAY;AAClF,SAASC,aAAa,EAAEC,aAAa,QAAQ,eAAe;AAE5D,SAASC,YAAY,EAAEC,oBAAoB,QAAQ,eAAe;AAElE,SAASC,eAAe,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,aAAa;AACvG,SAASC,gBAAgB,QAAQ,gBAAgB"}

package/dist/esm/config/types.d.ts

@@ -45,6 +45,7 @@ export interface Config {
     content?: {
         tokenize?: string;
     };
+    store?: 'sqlite' | 'duckdb';
     queries: Record<string, SavedQuery>;
 }
 export interface ResolvedConfig extends Config {

package/dist/esm/config/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/types.ts"],"sourcesContent":["import type { SignalWeights } from './signals.ts';\n\nexport const CONFIG_FILENAME = 'sense.config.json';\nexport const STATE_DIR = '.sense';\n\n// Highest sense.config.json `version` this build understands. Older versions auto-migrate on load.\nexport const SUPPORTED_CONFIG_VERSION = 5;\n\n// Each feature owns its tables, parse-time extraction, and reconcile step; commands degrade when one is off.\n// links/sections/tags/rank are opt-out toggles in the top-level `features` block; embed is not a\n// member of that block -- it is on exactly when an `embed` block names a model.\nexport const FEATURE_NAMES = ['links', 'sections', 'tags', 'rank', 'embed'] as const;\nexport type FeatureName = (typeof FEATURE_NAMES)[number];\n\n// `embed` names the model and gives the tree vectors at all; a preset's `signals` says which\n// engines that scope's searches use.\nexport interface EmbedConfig {\n  model?: string;\n  provider?: 'static' | 'openai' | 'cohere';\n  url?: string; // openai provider: OpenAI-compatible base URL, e.g. http://localhost:11434/v1\n  key?: string; // openai/cohere provider: name of the env var holding the bearer token, if any\n  languages?: string[]; // owner-declared model languages; enables the language-fit check where no card exists\n  chunkTokens?: number; // chunk size ceiling in estimated tokens for small-context models; default 500\n}\n\nexport const DEFAULT_EMBED_MODEL = 'minishlab/potion-retrieval-32M';\n\n// A named, self-contained file-selection scope. include/exclude are globby patterns resolved\n// relative to the config file. No inheritance between presets -- every field a preset wants\n// it declares itself. Presets may overlap freely; they are views, not partitions.\nexport interface Preset {\n  include: string[];\n  exclude?: string[];\n  k?: number; // result count for `search` scoped to this preset; default 10\n  // Exhaustive when present: every signal this preset's searches compose, each mapped to\n  // its RRF weight (a finite number > 0; presence is enablement). Absent means every signal\n  // whose prerequisite holds, each at weight 1 -- words always, links when the feature is on,\n  // vectors when the tree names a model. A layer searched for exact wording (ingested sources,\n  // archives, generated output) declares {\"words\":1,\"links\":1} and costs no embedding at all.\n  signals?: SignalWeights;\n  where?: string; // standing SQL condition against frontmatter alias `f`\n}\n\n// A saved `search` invocation: `sense <name>` runs like\n// `sense search <search> [--preset] [--include] [--exclude] [--where] [--k]`.\nexport interface SavedSearch {\n  search: string;\n  preset?: string;\n  include?: string[];\n  exclude?: string[];\n  where?: string;\n  k?: number;\n}\n\n// An entry names the verb it runs, one to one with the commands: { sql } runs like\n// `sense sql`, { search } like `sense search`.\nexport type SavedQuery = { sql: string } | SavedSearch;\n\nexport interface Config {\n  // Editor-only pointer to schema.json; never read by sense.\n  $schema?: string;\n  version?: number;\n  // File selection, index-time, and per-preset search defaults. A file is indexed iff any\n  // preset's include/exclude covers it (union). `default` is used when a command names no preset.\n  presets: Record<string, Preset>;\n  // Global feature defaults; absent block or key means enabled. embed is not a member here --\n  // see the `embed` block.\n  features?: { links?: boolean; sections?: boolean; tags?: boolean; rank?: boolean };\n  // Names the embedding model. Present means every indexed file gets vectors; absent means none do.\n  embed?: EmbedConfig;\n  // Settings for the `content` FTS5 table. `tokenize` decides which languages the tree can be\n  // word-searched in at all; changing it rebuilds the index.\n  content?: { tokenize?: string };\n  queries: Record<string, SavedQuery>;\n}\n\nexport interface ResolvedConfig extends Config {\n  baseDir: string;\n  configPath: string | null;\n  // Keys the file declares that this build does not read; cli reports them.\n  unknownKeys?: string[];\n  // Set when loadConfig auto-migrated the file on disk; cli reports it.\n  migratedFrom?: number;\n}\n\nexport interface SearchOverrides {\n  preset?: string;\n  k?: number;\n  where?: string;\n  include?: string[];\n  exclude?: string[];\n  noExclude?: boolean; // drop the preset's exclude for this command; --exclude still applies\n}\n\nexport interface EffectiveSearch {\n  presetName: string;\n  k: number;\n  where?: string;\n  include: string[];\n  exclude?: string[];\n  signals: SignalWeights; // this preset's effective signal weights\n}\n"],"names":["CONFIG_FILENAME","STATE_DIR","SUPPORTED_CONFIG_VERSION","FEATURE_NAMES","DEFAULT_EMBED_MODEL"],"mappings":"AAEA,OAAO,MAAMA,kBAAkB,oBAAoB;AACnD,OAAO,MAAMC,YAAY,SAAS;AAElC,mGAAmG;AACnG,OAAO,MAAMC,2BAA2B,EAAE;AAE1C,6GAA6G;AAC7G,iGAAiG;AACjG,gFAAgF;AAChF,OAAO,MAAMC,gBAAgB;IAAC;IAAS;IAAY;IAAQ;IAAQ;CAAQ,CAAU;AAcrF,OAAO,MAAMC,sBAAsB,iCAAiC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/types.ts"],"sourcesContent":["import type { SignalWeights } from './signals.ts';\n\nexport const CONFIG_FILENAME = 'sense.config.json';\nexport const STATE_DIR = '.sense';\n\n// Highest sense.config.json `version` this build understands. Older versions auto-migrate on load.\nexport const SUPPORTED_CONFIG_VERSION = 5;\n\n// Each feature owns its tables, parse-time extraction, and reconcile step; commands degrade when one is off.\n// links/sections/tags/rank are opt-out toggles in the top-level `features` block; embed is not a\n// member of that block -- it is on exactly when an `embed` block names a model.\nexport const FEATURE_NAMES = ['links', 'sections', 'tags', 'rank', 'embed'] as const;\nexport type FeatureName = (typeof FEATURE_NAMES)[number];\n\n// `embed` names the model and gives the tree vectors at all; a preset's `signals` says which\n// engines that scope's searches use.\nexport interface EmbedConfig {\n  model?: string;\n  provider?: 'static' | 'openai' | 'cohere';\n  url?: string; // openai provider: OpenAI-compatible base URL, e.g. http://localhost:11434/v1\n  key?: string; // openai/cohere provider: name of the env var holding the bearer token, if any\n  languages?: string[]; // owner-declared model languages; enables the language-fit check where no card exists\n  chunkTokens?: number; // chunk size ceiling in estimated tokens for small-context models; default 500\n}\n\nexport const DEFAULT_EMBED_MODEL = 'minishlab/potion-retrieval-32M';\n\n// A named, self-contained file-selection scope. include/exclude are globby patterns resolved\n// relative to the config file. No inheritance between presets -- every field a preset wants\n// it declares itself. Presets may overlap freely; they are views, not partitions.\nexport interface Preset {\n  include: string[];\n  exclude?: string[];\n  k?: number; // result count for `search` scoped to this preset; default 10\n  // Exhaustive when present: every signal this preset's searches compose, each mapped to\n  // its RRF weight (a finite number > 0; presence is enablement). Absent means every signal\n  // whose prerequisite holds, each at weight 1 -- words always, links when the feature is on,\n  // vectors when the tree names a model. A layer searched for exact wording (ingested sources,\n  // archives, generated output) declares {\"words\":1,\"links\":1} and costs no embedding at all.\n  signals?: SignalWeights;\n  where?: string; // standing SQL condition against frontmatter alias `f`\n}\n\n// A saved `search` invocation: `sense <name>` runs like\n// `sense search <search> [--preset] [--include] [--exclude] [--where] [--k]`.\nexport interface SavedSearch {\n  search: string;\n  preset?: string;\n  include?: string[];\n  exclude?: string[];\n  where?: string;\n  k?: number;\n}\n\n// An entry names the verb it runs, one to one with the commands: { sql } runs like\n// `sense sql`, { search } like `sense search`.\nexport type SavedQuery = { sql: string } | SavedSearch;\n\nexport interface Config {\n  // Editor-only pointer to schema.json; never read by sense.\n  $schema?: string;\n  version?: number;\n  // File selection, index-time, and per-preset search defaults. A file is indexed iff any\n  // preset's include/exclude covers it (union). `default` is used when a command names no preset.\n  presets: Record<string, Preset>;\n  // Global feature defaults; absent block or key means enabled. embed is not a member here --\n  // see the `embed` block.\n  features?: { links?: boolean; sections?: boolean; tags?: boolean; rank?: boolean };\n  // Names the embedding model. Present means every indexed file gets vectors; absent means none do.\n  embed?: EmbedConfig;\n  // Settings for the `content` FTS5 table. `tokenize` decides which languages the tree can be\n  // word-searched in at all; changing it rebuilds the index.\n  content?: { tokenize?: string };\n  // Backing store engine for this tree; defaults to \"sqlite\" (zero-dependency). \"duckdb\" is an\n  // optional dependency and, in this build, implements only the portable surface (frontmatter,\n  // links, sections, tags, rank, raw sql) -- a config needing lexical or vector search under it fails at open.\n  store?: 'sqlite' | 'duckdb';\n  queries: Record<string, SavedQuery>;\n}\n\nexport interface ResolvedConfig extends Config {\n  baseDir: string;\n  configPath: string | null;\n  // Keys the file declares that this build does not read; cli reports them.\n  unknownKeys?: string[];\n  // Set when loadConfig auto-migrated the file on disk; cli reports it.\n  migratedFrom?: number;\n}\n\nexport interface SearchOverrides {\n  preset?: string;\n  k?: number;\n  where?: string;\n  include?: string[];\n  exclude?: string[];\n  noExclude?: boolean; // drop the preset's exclude for this command; --exclude still applies\n}\n\nexport interface EffectiveSearch {\n  presetName: string;\n  k: number;\n  where?: string;\n  include: string[];\n  exclude?: string[];\n  signals: SignalWeights; // this preset's effective signal weights\n}\n"],"names":["CONFIG_FILENAME","STATE_DIR","SUPPORTED_CONFIG_VERSION","FEATURE_NAMES","DEFAULT_EMBED_MODEL"],"mappings":"AAEA,OAAO,MAAMA,kBAAkB,oBAAoB;AACnD,OAAO,MAAMC,YAAY,SAAS;AAElC,mGAAmG;AACnG,OAAO,MAAMC,2BAA2B,EAAE;AAE1C,6GAA6G;AAC7G,iGAAiG;AACjG,gFAAgF;AAChF,OAAO,MAAMC,gBAAgB;IAAC;IAAS;IAAY;IAAQ;IAAQ;CAAQ,CAAU;AAcrF,OAAO,MAAMC,sBAAsB,iCAAiC"}

package/dist/esm/config/validate.js

@@ -9,6 +9,7 @@ const KNOWN_KEYS = new Set([
     'features',
     'embed',
     'content',
+    'store',
     'queries'
 ]);
 const KNOWN_PRESET_KEYS = new Set([
@@ -61,7 +62,7 @@ export function validateLegacyScan(parsed, configPath) {
     }
 }
 // Only the shape is checked here. Whether the linked SQLite accepts the tokenizer is settled
-// by probing it in db.ts, so this never has to carry a table of which version added what.
+// by probing it in store/sqlite/open.ts, so this never has to carry a table of which version added what.
 function validateContentBlock(value, configPath) {
     if (typeof value !== 'object' || value === null || Array.isArray(value)) {
         throw new SenseError('CONFIG_INVALID', `${configPath}: content must be an object of { tokenize?: string }`);
@@ -75,6 +76,11 @@ function validateContentBlock(value, configPath) {
         throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize must be a non-empty string, e.g. "trigram" or "unicode61 tokenchars '-_'"`);
     }
 }
+function validateStoreKey(value, configPath) {
+    if (value !== 'sqlite' && value !== 'duckdb') {
+        throw new SenseError('CONFIG_INVALID', `${configPath}: store must be "sqlite" or "duckdb"`);
+    }
+}
 function validateFeaturesBlock(value, configPath) {
     if (typeof value !== 'object' || value === null || Array.isArray(value)) {
         throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);
@@ -266,6 +272,9 @@ export function validateConfig(parsed, configPath) {
     if (cfg.content !== undefined) {
         validateContentBlock(cfg.content, configPath);
     }
+    if (cfg.store !== undefined) {
+        validateStoreKey(cfg.store, configPath);
+    }
     // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).
     if (cfg.embed === undefined) {
         for (const [name, value] of Object.entries(presets)){