sensemaking 0.15.3 → 0.16.0

package/dist/esm/commands/search.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, contentTokenize, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { modelPresent, semanticCandidates } from '../features/embed.ts';\nimport { linkEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph.ts';\nimport type { Row } from '../output.ts';\nimport { searchError } from '../search-error.ts';\nimport { segmentMatch } from '../segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\n\n// Mirrors the main columns onto the sidecars, so a title match found through title_seg ranks\n// like a title match found through title, not like a body match.\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\nconst RRF_K = 60;\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.\nconst SNIPPET_BOUND = 16_384;\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nfunction lineRangeFor(db: DatabaseSync, path: string, line: number): string | null {\n  const row = db.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// BM25 + link expansion + vectors, fused by reciprocal rank; `via` names the signal per row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k } = effective;\n\n  const allPaths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = rawScope(db, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = narrowByWhere(db, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // Asking for vectors without a model is a misconfiguration, not a mode: degrading would make\n  // the same search answer differently before and after a download. semantic:false never asks.\n  const wantsVectors = effective.semantic && anyPresetEmbeds(cfg);\n  if (wantsVectors && !modelPresent(cfg)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the embedding model is not available; run \\`sense download\\`, or set \"semantic\": false on that preset to search on words and links`);\n  }\n  const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);\n\n  // Terms pass to MATCH as written, with one transform: a run of text whose language marks no\n  // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier\n  // ahead of it retargets its _seg column), matching how it was indexed. Text that already\n  // carries boundaries comes through untouched, so this is a no-op for the languages that\n  // never needed it. Gated on the same predicate that decided whether the sidecars were\n  // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid\n  // syntax still errors rather than being rewritten.\n  // --where\n  // applies inside the candidate query (a post-filter would drop matches ranked past the pool)\n  // and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) materializeScope(db, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n  // Docs past SNIPPET_BOUND skip snippet() entirely -- SQLite\n  // short-circuits the untaken CASE branch, so it's never invoked on them.\n  // Column 2 (text), not -1 (best column): -1 could surface a _seg sidecar as the excerpt,\n  // which is machine-spaced and not what the author wrote. A row that matches only in a\n  // sidecar gets an unhighlighted text excerpt below instead -- a stated cost.\n  const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;\n  let matchRows: Array<{ path: string; hit: string | null }>;\n  try {\n    matchRows = db.prepare(matchSql).all(query) as Array<{ path: string; hit: string | null }>;\n  } catch (err) {\n    throw searchError(err as Error, terms, scope);\n  }\n\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n  const candidates = new Map<string, { score: number; via: string }>();\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: 1 / (RRF_K + i), via: 'match' });\n  });\n\n  const edges = featureEnabled(cfg, 'links') && matchRows.length > 0 ? linkEdges(db) : [];\n  if (edges.length > 0) {\n    // Gates the label only, not the score: restart mass ranks every seed without an incident\n    // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n    const linked = new Set(edges.flat());\n    const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n    const ranked = [...personalizedRank(allPaths, edges, seeds)]\n      .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n      .sort((a, b) => b[1] - a[1])\n      .slice(0, fetch);\n    ranked.forEach(([path], i) => {\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        if (linked.has(path)) existing.via = 'match+link';\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'link' });\n      }\n    });\n  }\n\n  // Vector expansion, invoked only: a third RRF list at the swept flat-region constants\n  // (weight 1, pool = fetch). Each row carries its best chunk's line range.\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);\n    vec.forEach(({ path, lines, similarity }, i) => {\n      chunkLines.set(path, lines);\n      chunkSimilarity.set(path, similarity);\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        existing.via = `${existing.via}+vector`;\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'vector' });\n      }\n    });\n  }\n\n  db.exec('DROP TABLE IF EXISTS _search');\n  db.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const rows = db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","anyPresetEmbeds","contentTokenize","featureEnabled","resolveSearch","SenseError","modelPresent","semanticCandidates","linkEdges","personalizedRank","searchError","segmentMatch","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","WEIGHTED_BM25","RRF_K","SNIPPET_BOUND","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","db","path","row","prepare","get","start_line","end_line","search","cfg","opts","hits","chunkLines","chunkSimilarity","effective","k","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","semantic","presetName","semanticEnabled","scope","whereJoin","whereCond","scopeCond","matchSql","undefined","matchRows","Map","hit","oversized","candidates","forEach","set","score","via","edges","linked","flat","seeds","ranked","has","existing","vec","lines","similarity","exec","insert","c","run","similarityCol","rows","bareTerms","baseDir"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,eAAe,EAAEC,eAAe,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AACrG,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,kBAAkB,QAAQ,uBAAuB;AACxE,SAASC,SAAS,QAAQ,uBAAuB;AACjD,SAASC,gBAAgB,QAAQ,cAAc;AAE/C,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAE3F,6FAA6F;AAC7F,iEAAiE;AACjE,MAAMC,gBAAgB;AACtB,MAAMC,QAAQ;AAEd,iFAAiF;AACjF,8EAA8E;AAC9E,MAAMC,gBAAgB;AACtB,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAASE,aAAaC,EAAgB,EAAEC,IAAY,EAAEJ,IAAY;IAChE,MAAMK,MAAMF,GAAGG,OAAO,CAAC,qIAAqIC,GAAG,CAACH,MAAMJ,MAAMA;IAC5K,OAAOK,MAAM,CAAC,CAAC,EAAEA,IAAIG,UAAU,CAAC,CAAC,EAAEH,IAAII,QAAQ,EAAE,GAAG;AACtD;AAWA,6FAA6F;AAC7F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOP,EAAgB,EAAEQ,GAAmB,EAAEjD,KAAa,EAAEkD,OAAsB,CAAC,CAAC;QA+GpCC,WAAwBC,iBAA8BC;IA9G3H,MAAMC,YAAYnF,cAAc8E,KAAKC;IACrC,MAAM,EAAEK,CAAC,EAAE,GAAGD;IAEd,MAAME,WAAW,AAACf,GAAGG,OAAO,CAAC,kCAAkCa,GAAG,GAA+BjE,GAAG,CAAC,CAACkE,IAAMA,EAAEhB,IAAI;IAClH,MAAMiB,aAAa9E,SAAS4D,IAAIQ,KAAKC,MAAMM;IAC3C,MAAMI,cAAcD,WAAWnC,IAAI,GAAGgC,SAAS3D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAIgE;IACJ,IAAI;QACFA,eAAejF,cAAc6D,IAAIkB,YAAYL,UAAUQ,KAAK;IAC9D,EAAE,OAAOC,KAAK;QACZ,MAAMtF,YAAYsF,KAAc/D,OAAOsD,UAAUQ,KAAK;IACxD;IACA,MAAME,QAAQrC,KAAKsC,GAAG,CAACV,IAAI,GAAG;IAE9B,6FAA6F;IAC7F,6FAA6F;IAC7F,MAAMW,eAAeZ,UAAUa,QAAQ,IAAInG,gBAAgBiF;IAC3D,IAAIiB,gBAAgB,CAAC7F,aAAa4E,MAAM;QACtC,MAAM,IAAI7E,WAAW,uBAAuB,CAAC,QAAQ,EAAEkF,UAAUc,UAAU,CAAC,+JAA+J,CAAC;IAC9O;IACA,MAAMC,kBAAkBH,gBAAgBpF,mBAAmB2D,IAAIQ,KAAKY;IAEpE,4FAA4F;IAC5F,0FAA0F;IAC1F,yFAAyF;IACzF,wFAAwF;IACxF,sFAAsF;IACtF,6FAA6F;IAC7F,mDAAmD;IACnD,UAAU;IACV,6FAA6F;IAC7F,wDAAwD;IACxD,MAAMS,QAAQhB,UAAUQ,KAAK;IAC7B,MAAMS,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIV,aAAajF,iBAAiB8D,IAAI,iBAAiBkB;IACvD,MAAMc,YAAYb,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAC3F,4DAA4D;IAC5D,yEAAyE;IACzE,yFAAyF;IACzF,sFAAsF;IACtF,6EAA6E;IAC7E,MAAMc,WAAW,CAAC,+DAA+D,EAAEzF,cAAc,+EAA+E,EAAEsF,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAE1F,cAAc,OAAO,EAAEiF,OAAO;IACtR,MAAM5E,QAAQnB,gBAAgBgF,SAAS0B,YAAYjG,aAAasB,SAASA;IACzE,IAAI4E;IACJ,IAAI;QACFA,YAAYnC,GAAGG,OAAO,CAAC8B,UAAUjB,GAAG,CAACrE;IACvC,EAAE,OAAO2E,KAAK;QACZ,MAAMtF,YAAYsF,KAAc/D,OAAOsE;IACzC;IAEA,MAAMnB,OAAO,IAAI0B,IAAID,UAAUpF,GAAG,CAAC,CAACkE,IAAM;YAACA,EAAEhB,IAAI;YAAEgB,EAAEoB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI3D,IAAIwD,UAAUhF,MAAM,CAAC,CAAC8D,IAAMA,EAAEoB,GAAG,KAAK,MAAMtF,GAAG,CAAC,CAACkE,IAAMA,EAAEhB,IAAI;IACnF,MAAMsC,aAAa,IAAIH;IACvBD,UAAUK,OAAO,CAAC,CAACvB,GAAGzC;QACpB+D,WAAWE,GAAG,CAACxB,EAAEhB,IAAI,EAAE;YAAEyC,OAAO,IAAKnG,CAAAA,QAAQiC,CAAAA;YAAImE,KAAK;QAAQ;IAChE;IAEA,MAAMC,QAAQnH,eAAe+E,KAAK,YAAY2B,UAAU/E,MAAM,GAAG,IAAItB,UAAUkE,MAAM,EAAE;IACvF,IAAI4C,MAAMxF,MAAM,GAAG,GAAG;QACpB,yFAAyF;QACzF,uFAAuF;QACvF,MAAMyF,SAAS,IAAIlE,IAAIiE,MAAME,IAAI;QACjC,MAAMC,QAAQ,IAAIX,IAAID,UAAUpF,GAAG,CAAC,CAACkE,GAAGzC,IAAM;gBAACyC,EAAEhB,IAAI;gBAAE,IAAKzB,CAAAA,IAAI,CAAA;aAAG;QACnE,MAAMwE,SAAS;eAAIjH,iBAAiBgF,UAAU6B,OAAOG;SAAO,CACzD5F,MAAM,CAAC,CAAC,CAAC8C,MAAMyC,MAAM,GAAKA,QAAQ,QAAQtB,aAAa6B,GAAG,CAAChD,OAC3DjC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BoB,KAAK,CAAC,GAAGkC;QACZyB,OAAOR,OAAO,CAAC,CAAC,CAACvC,KAAK,EAAEzB;YACtB,MAAM0E,WAAWX,WAAWnC,GAAG,CAACH;YAChC,IAAIiD,UAAU;gBACZA,SAASR,KAAK,IAAI,IAAKnG,CAAAA,QAAQiC,CAAAA;gBAC/B,IAAIqE,OAAOI,GAAG,CAAChD,OAAOiD,SAASP,GAAG,GAAG;YACvC,OAAO;gBACLJ,WAAWE,GAAG,CAACxC,MAAM;oBAAEyC,OAAO,IAAKnG,CAAAA,QAAQiC,CAAAA;oBAAImE,KAAK;gBAAO;YAC7D;QACF;IACF;IAEA,sFAAsF;IACtF,0EAA0E;IAC1E,MAAMhC,aAAa,IAAIyB;IACvB,MAAMxB,kBAAkB,IAAIwB;IAC5B,IAAIR,iBAAiB;QACnB,MAAMuB,MAAM,MAAMtH,mBAAmBmE,IAAIQ,KAAKjD,OAAOgE,OAAOH;QAC5D+B,IAAIX,OAAO,CAAC,CAAC,EAAEvC,IAAI,EAAEmD,KAAK,EAAEC,UAAU,EAAE,EAAE7E;YACxCmC,WAAW8B,GAAG,CAACxC,MAAMmD;YACrBxC,gBAAgB6B,GAAG,CAACxC,MAAMoD;YAC1B,MAAMH,WAAWX,WAAWnC,GAAG,CAACH;YAChC,IAAIiD,UAAU;gBACZA,SAASR,KAAK,IAAI,IAAKnG,CAAAA,QAAQiC,CAAAA;gBAC/B0E,SAASP,GAAG,GAAG,GAAGO,SAASP,GAAG,CAAC,OAAO,CAAC;YACzC,OAAO;gBACLJ,WAAWE,GAAG,CAACxC,MAAM;oBAAEyC,OAAO,IAAKnG,CAAAA,QAAQiC,CAAAA;oBAAImE,KAAK;gBAAS;YAC/D;QACF;IACF;IAEA3C,GAAGsD,IAAI,CAAC;IACRtD,GAAGsD,IAAI,CAAC;IACR,MAAMC,SAASvD,GAAGG,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACF,MAAMuD,EAAE,IAAIjB,WAAYgB,OAAOE,GAAG,CAACxD,MAAMuD,EAAEd,KAAK,EAAEc,EAAEb,GAAG,GAAEjC,YAAAA,KAAKN,GAAG,CAACH,mBAATS,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWP,GAAG,CAACH,mBAAfU,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBR,GAAG,CAACH,mBAApBW,kCAAAA,uBAA6B;IAExJ,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMS,QAAQQ,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAM6B,gBAAgB9B,kBAAkB,yBAAyB;IACjE,MAAM+B,OAAO3D,GACVG,OAAO,CACN,CAAC,kIAAkI,EAAEuD,cAAc;;OAElJ,EAAErC,MAAM,oCAAoC,CAAC,EAE/CL,GAAG,CAACF;IAEP,IAAIwB,UAAUvD,IAAI,GAAG,GAAG;QACtB,MAAM6E,YAAYlH,iBAAiBa;QACnC,KAAK,MAAM2C,OAAOyD,KAAM;YACtB,IAAIzD,IAAImC,GAAG,KAAK,QAAQ,CAACC,UAAUW,GAAG,CAAC/C,IAAID,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAO5D,aAAaC,KAAKkF,IAAIqD,OAAO,EAAE3D,IAAID,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAM2E;YACjD1D,IAAImC,GAAG,GAAGjD;YACV,IAAIc,IAAIkD,KAAK,IAAI,MAAMlD,IAAIkD,KAAK,GAAG3H,eAAe+E,KAAK,cAAcT,aAAaC,IAAIE,IAAID,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QAC1I;IACF;IAEA,OAAOqE;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { contentTokenize, embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output.ts';\nimport { searchError } from '../search-error.ts';\nimport { segmentMatch } from '../segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nfunction lineRangeFor(db: DatabaseSync, path: string, line: number): string | null {\n  const row = db.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = rawScope(db, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = narrowByWhere(db, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);\n\n  // Terms pass to MATCH as written, with one transform: a run of text whose language marks no\n  // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier\n  // ahead of it retargets its _seg column), matching how it was indexed. Text that already\n  // carries boundaries comes through untouched, so this is a no-op for the languages that\n  // never needed it. Gated on the same predicate that decided whether the sidecars were\n  // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid\n  // syntax still errors rather than being rewritten.\n  // --where\n  // applies inside the candidate query (a post-filter would drop matches ranked past the pool)\n  // and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) materializeScope(db, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n  const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: Array<{ path: string; hit: string | null }> = [];\n  // A query that rewrites to nothing (e.g. only unspaced-script punctuation) has no\n  // searchable words: zero lexical rows, and the other signals still compose.\n  if (signals.words !== undefined && query.trim() !== '') {\n    try {\n      matchRows = wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  db.exec('DROP TABLE IF EXISTS _search');\n  db.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const rows = db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","contentTokenize","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","segmentMatch","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","db","path","row","prepare","get","start_line","end_line","search","cfg","opts","hits","chunkLines","chunkSimilarity","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hit","oversized","links","exec","insert","c","run","score","via","similarityCol","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,eAAe,EAAEC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AACjG,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,8EAA8E;AAC9E,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAASE,aAAaC,EAAgB,EAAEC,IAAY,EAAEJ,IAAY;IAChE,MAAMK,MAAMF,GAAGG,OAAO,CAAC,qIAAqIC,GAAG,CAACH,MAAMJ,MAAMA;IAC5K,OAAOK,MAAM,CAAC,CAAC,EAAEA,IAAIG,UAAU,CAAC,CAAC,EAAEH,IAAII,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOP,EAAgB,EAAEQ,GAAmB,EAAEjD,KAAa,EAAEkD,OAAsB,CAAC,CAAC;QA6EpCC,WAAwBC,iBAA8BC;IA5E3H,MAAMC,YAAYjF,cAAc4E,KAAKC;IACrC,MAAM,EAAEK,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAChB,GAAGG,OAAO,CAAC,kCAAkCc,GAAG,GAA+BlE,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAClH,MAAMkB,aAAa/E,SAAS4D,IAAIQ,KAAKC,MAAMO;IAC3C,MAAMI,cAAcD,WAAWpC,IAAI,GAAGiC,SAAS5D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAIiE;IACJ,IAAI;QACFA,eAAelF,cAAc6D,IAAImB,YAAYN,UAAUS,KAAK;IAC9D,EAAE,OAAOC,KAAK;QACZ,MAAMvF,YAAYuF,KAAchE,OAAOsD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQtC,KAAKuC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAInG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkBgG,IAAI;YACxB,MAAM,IAAI9F,WAAW,uBAAuB,CAAC,QAAQ,EAAE8E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAEjG,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAMkG,kBAAkBN,gBAAgBrF,mBAAmB2D,IAAIQ,KAAKa;IAEpE,4FAA4F;IAC5F,0FAA0F;IAC1F,yFAAyF;IACzF,wFAAwF;IACxF,sFAAsF;IACtF,6FAA6F;IAC7F,mDAAmD;IACnD,UAAU;IACV,6FAA6F;IAC7F,wDAAwD;IACxD,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIb,aAAalF,iBAAiB8D,IAAI,iBAAiBmB;IACvD,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAC3F,MAAMzE,QAAQlB,gBAAgB+E,SAASoB,YAAY3F,aAAasB,SAASA;IAEzE,MAAM8E,aAAa,IAAIC;IACvB,IAAIC,YAAyD,EAAE;IAC/D,kFAAkF;IAClF,4EAA4E;IAC5E,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAajF,MAAMO,IAAI,OAAO,IAAI;QACtD,IAAI;YACFqF,YAAY/F,gBAAgBwD,IAAIqC,YAAY1F,OAAOuF,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QAC1G,EAAE,OAAOjB,KAAK;YACZ,MAAMvF,YAAYuF,KAAchE,OAAO0E;QACzC;IACF;IACA,MAAMvB,OAAO,IAAI4B,IAAIC,UAAUxF,GAAG,CAAC,CAACmE,IAAM;YAACA,EAAEjB,IAAI;YAAEiB,EAAEuB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI/D,IAAI4D,UAAUpF,MAAM,CAAC,CAAC+D,IAAMA,EAAEuB,GAAG,KAAK,MAAM1F,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAEnF,IAAIc,QAAQ4B,KAAK,KAAKf,WAAWtF,gBAAgB0D,IAAIqC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ4B,KAAK;IAExH,IAAIhC,aAAa,IAAI2B;IACrB,IAAI1B,kBAAkB,IAAI0B;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAErB,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrE,kBAAkByD,IAAIQ,KAAK6B,YAAY9E,OAAOiE,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IACvI;IAEA3B,GAAG4C,IAAI,CAAC;IACR5C,GAAG4C,IAAI,CAAC;IACR,MAAMC,SAAS7C,GAAGG,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACF,MAAM6C,EAAE,IAAIT,WAAYQ,OAAOE,GAAG,CAAC9C,MAAM6C,EAAEE,KAAK,EAAEF,EAAEG,GAAG,GAAEvC,YAAAA,KAAKN,GAAG,CAACH,mBAATS,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWP,GAAG,CAACH,mBAAfU,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBR,GAAG,CAACH,mBAApBW,kCAAAA,uBAA6B;IAExJ,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMU,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAMiB,gBAAgBlB,kBAAkB,yBAAyB;IACjE,MAAMmB,OAAOnD,GACVG,OAAO,CACN,CAAC,kIAAkI,EAAE+C,cAAc;;OAElJ,EAAE5B,MAAM,oCAAoC,CAAC,EAE/CL,GAAG,CAACH;IAEP,IAAI4B,UAAU3D,IAAI,GAAG,GAAG;QACtB,MAAMqE,YAAY1G,iBAAiBa;QACnC,KAAK,MAAM2C,OAAOiD,KAAM;YACtB,IAAIjD,IAAIuC,GAAG,KAAK,QAAQ,CAACC,UAAUW,GAAG,CAACnD,IAAID,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAO1D,aAAaC,KAAKgF,IAAI8C,OAAO,EAAEpD,IAAID,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMmE;YACjDlD,IAAIuC,GAAG,GAAGrD;YACV,IAAIc,IAAIqD,KAAK,IAAI,MAAMrD,IAAIqD,KAAK,GAAG5H,eAAe6E,KAAK,cAAcT,aAAaC,IAAIE,IAAID,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QAC1I;IACF;IAEA,OAAO6D;AACT"}

package/dist/esm/commands/signals.d.ts

@@ -0,0 +1,18 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { ResolvedConfig } from '../config/index.js';
+export type Candidates = Map<string, {
+    score: number;
+    via: string;
+}>;
+export declare function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{
+    path: string;
+    hit: string | null;
+}>;
+export declare function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{
+    path: string;
+    hit: string | null;
+}>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void;
+export declare function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{
+    chunkLines: Map<string, string>;
+    chunkSimilarity: Map<string, number>;
+}>;

package/dist/esm/commands/signals.js

@@ -0,0 +1,78 @@
+import { semanticCandidates } from '../embed/query.js';
+import { linkEdges } from '../features/index.js';
+import { personalizedRank } from '../graph.js';
+// RRF composes whatever the preset declares; the constant lives here since it belongs to
+// the fusion these three functions share, not to search() itself.
+const RRF_K = 60;
+// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared
+// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal
+// is seeded from its output, so search() runs this one first when "words" is declared.
+export function wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, weight) {
+    const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
+    const SNIPPET_BOUND = 16384;
+    const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;
+    const matchRows = db.prepare(matchSql).all(query);
+    matchRows.forEach((r, i)=>{
+        candidates.set(r.path, {
+            score: weight / (RRF_K + i),
+            via: 'match'
+        });
+    });
+    return matchRows;
+}
+// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this
+// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before
+// this was a signal.
+export function linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, weight) {
+    if (matchRows.length === 0) return;
+    const edges = linkEdges(db);
+    if (edges.length === 0) return;
+    // Gates the label only, not the score: restart mass ranks every seed without an incident
+    // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).
+    const linked = new Set(edges.flat());
+    const seeds = new Map(matchRows.map((r, i)=>[
+            r.path,
+            1 / (i + 1)
+        ]));
+    const ranked = [
+        ...personalizedRank(allPaths, edges, seeds)
+    ].filter(([path, score])=>score > 1e-9 && allowedPaths.has(path)).sort((a, b)=>b[1] - a[1]).slice(0, fetch);
+    ranked.forEach(([path], i)=>{
+        const existing = candidates.get(path);
+        if (existing) {
+            existing.score += weight / (RRF_K + i);
+            if (linked.has(path)) existing.via = 'match+link';
+        } else {
+            candidates.set(path, {
+                score: weight / (RRF_K + i),
+                via: 'link'
+            });
+        }
+    });
+}
+// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`
+// at this signal's weight. Each row also carries its best chunk's line range and similarity,
+// for the caller.
+export async function vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, weight) {
+    const chunkLines = new Map();
+    const chunkSimilarity = new Map();
+    const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);
+    vec.forEach(({ path, lines, similarity }, i)=>{
+        chunkLines.set(path, lines);
+        chunkSimilarity.set(path, similarity);
+        const existing = candidates.get(path);
+        if (existing) {
+            existing.score += weight / (RRF_K + i);
+            existing.via = `${existing.via}+vector`;
+        } else {
+            candidates.set(path, {
+                score: weight / (RRF_K + i),
+                via: 'vector'
+            });
+        }
+    });
+    return {
+        chunkLines,
+        chunkSimilarity
+    };
+}

package/dist/esm/commands/signals.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/signals.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { semanticCandidates } from '../embed/query.ts';\nimport { linkEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph.ts';\n\n// RRF composes whatever the preset declares; the constant lives here since it belongs to\n// the fusion these three functions share, not to search() itself.\nconst RRF_K = 60;\n\nexport type Candidates = Map<string, { score: number; via: string }>;\n\n// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared\n// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal\n// is seeded from its output, so search() runs this one first when \"words\" is declared.\nexport function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{ path: string; hit: string | null }> {\n  const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n  const SNIPPET_BOUND = 16_384;\n  const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const matchRows = db.prepare(matchSql).all(query) as Array<{ path: string; hit: string | null }>;\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: weight / (RRF_K + i), via: 'match' });\n  });\n  return matchRows;\n}\n\n// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this\n// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before\n// this was a signal.\nexport function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{ path: string; hit: string | null }>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void {\n  if (matchRows.length === 0) return;\n  const edges = linkEdges(db);\n  if (edges.length === 0) return;\n  // Gates the label only, not the score: restart mass ranks every seed without an incident\n  // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n  const linked = new Set(edges.flat());\n  const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n  const ranked = [...personalizedRank(allPaths, edges, seeds)]\n    .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, fetch);\n  ranked.forEach(([path], i) => {\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      if (linked.has(path)) existing.via = 'match+link';\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'link' });\n    }\n  });\n}\n\n// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`\n// at this signal's weight. Each row also carries its best chunk's line range and similarity,\n// for the caller.\nexport async function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{ chunkLines: Map<string, string>; chunkSimilarity: Map<string, number> }> {\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);\n  vec.forEach(({ path, lines, similarity }, i) => {\n    chunkLines.set(path, lines);\n    chunkSimilarity.set(path, similarity);\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      existing.via = `${existing.via}+vector`;\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'vector' });\n    }\n  });\n  return { chunkLines, chunkSimilarity };\n}\n"],"names":["semanticCandidates","linkEdges","personalizedRank","RRF_K","wordsCandidates","db","candidates","query","whereJoin","whereCond","scopeCond","fetch","weight","WEIGHTED_BM25","SNIPPET_BOUND","matchSql","matchRows","prepare","all","forEach","r","i","set","path","score","via","linksCandidates","allPaths","allowedPaths","length","edges","linked","Set","flat","seeds","Map","map","ranked","filter","has","sort","a","b","slice","existing","get","vectorsCandidates","cfg","terms","chunkLines","chunkSimilarity","vec","lines","similarity"],"mappings":"AAEA,SAASA,kBAAkB,QAAQ,oBAAoB;AACvD,SAASC,SAAS,QAAQ,uBAAuB;AACjD,SAASC,gBAAgB,QAAQ,cAAc;AAE/C,yFAAyF;AACzF,kEAAkE;AAClE,MAAMC,QAAQ;AAId,+FAA+F;AAC/F,gGAAgG;AAChG,uFAAuF;AACvF,OAAO,SAASC,gBAAgBC,EAAgB,EAAEC,UAAsB,EAAEC,KAAa,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,MAAc;IAC7K,MAAMC,gBAAgB;IACtB,MAAMC,gBAAgB;IACtB,MAAMC,WAAW,CAAC,+DAA+D,EAAED,cAAc,+EAA+E,EAAEN,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEG,cAAc,OAAO,EAAEF,OAAO;IACtR,MAAMK,YAAYX,GAAGY,OAAO,CAACF,UAAUG,GAAG,CAACX;IAC3CS,UAAUG,OAAO,CAAC,CAACC,GAAGC;QACpBf,WAAWgB,GAAG,CAACF,EAAEG,IAAI,EAAE;YAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;YAAII,KAAK;QAAQ;IACrE;IACA,OAAOT;AACT;AAEA,8FAA8F;AAC9F,gGAAgG;AAChG,qBAAqB;AACrB,OAAO,SAASU,gBAAgBrB,EAAgB,EAAEC,UAAsB,EAAEU,SAAsD,EAAEW,QAAkB,EAAEC,YAAyB,EAAEjB,KAAa,EAAEC,MAAc;IAC5M,IAAII,UAAUa,MAAM,KAAK,GAAG;IAC5B,MAAMC,QAAQ7B,UAAUI;IACxB,IAAIyB,MAAMD,MAAM,KAAK,GAAG;IACxB,yFAAyF;IACzF,uFAAuF;IACvF,MAAME,SAAS,IAAIC,IAAIF,MAAMG,IAAI;IACjC,MAAMC,QAAQ,IAAIC,IAAInB,UAAUoB,GAAG,CAAC,CAAChB,GAAGC,IAAM;YAACD,EAAEG,IAAI;YAAE,IAAKF,CAAAA,IAAI,CAAA;SAAG;IACnE,MAAMgB,SAAS;WAAInC,iBAAiByB,UAAUG,OAAOI;KAAO,CACzDI,MAAM,CAAC,CAAC,CAACf,MAAMC,MAAM,GAAKA,QAAQ,QAAQI,aAAaW,GAAG,CAAChB,OAC3DiB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGhC;IACZ0B,OAAOlB,OAAO,CAAC,CAAC,CAACI,KAAK,EAAEF;QACtB,MAAMuB,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpC,IAAIU,OAAOQ,GAAG,CAAChB,OAAOqB,SAASnB,GAAG,GAAG;QACvC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAO;QAClE;IACF;AACF;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,kBAAkB;AAClB,OAAO,eAAeqB,kBAAkBzC,EAAgB,EAAE0C,GAAmB,EAAEzC,UAAsB,EAAE0C,KAAa,EAAErC,KAAa,EAAEiB,YAAyB,EAAEhB,MAAc;IAC5K,MAAMqC,aAAa,IAAId;IACvB,MAAMe,kBAAkB,IAAIf;IAC5B,MAAMgB,MAAM,MAAMnD,mBAAmBK,IAAI0C,KAAKC,OAAOrC,OAAOiB;IAC5DuB,IAAIhC,OAAO,CAAC,CAAC,EAAEI,IAAI,EAAE6B,KAAK,EAAEC,UAAU,EAAE,EAAEhC;QACxC4B,WAAW3B,GAAG,CAACC,MAAM6B;QACrBF,gBAAgB5B,GAAG,CAACC,MAAM8B;QAC1B,MAAMT,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpCuB,SAASnB,GAAG,GAAG,GAAGmB,SAASnB,GAAG,CAAC,OAAO,CAAC;QACzC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAS;QACpE;IACF;IACA,OAAO;QAAEwB;QAAYC;IAAgB;AACvC"}

package/dist/esm/commands/status.d.ts

@@ -1,9 +1,9 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { ResolvedConfig } from '../config/index.js';
+import type { ResolvedConfig, SignalWeights } from '../config/index.js';
 export interface PresetCoverage {
     name: string;
     files: number;
     embedded: number;
-    semantic: boolean;
+    signals: SignalWeights;
 }
 export declare function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[];

package/dist/esm/commands/status.js

@@ -1,4 +1,4 @@
-import { anyPresetEmbeds, presetNames, presetSemanticEnabled } from '../config/index.js';
+import { anyPresetEmbeds, presetNames, presetSignals } from '../config/index.js';
 // Indexing derives from presets, so the derivation stays visible. Read from preset_files, not
 // recomputed from globs, so it reflects the cache rather than the config.
 export function presetCoverage(db, cfg) {
@@ -10,7 +10,7 @@ export function presetCoverage(db, cfg) {
             name,
             files,
             embedded,
-            semantic: presetSemanticEnabled(cfg, name)
+            signals: presetSignals(cfg, name)
         };
     });
 }

package/dist/esm/commands/status.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, presetNames, presetSemanticEnabled } from '../config/index.ts';\n\nexport interface PresetCoverage {\n  name: string;\n  files: number;\n  embedded: number;\n  // Reported so 0 embedded reads as \"this scope declined vectors\" rather than \"not yet built\".\n  semantic: boolean;\n}\n\n// Indexing derives from presets, so the derivation stays visible. Read from preset_files, not\n// recomputed from globs, so it reflects the cache rather than the config.\nexport function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[] {\n  const embedActive = anyPresetEmbeds(cfg);\n  return presetNames(cfg).map((name) => {\n    const files = (db.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?').get(name) as { n: number }).n;\n    const embedded = embedActive ? (db.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e.\"path\" = pf.\"path\" AND e.vector IS NOT NULL)').get(name) as { n: number }).n : 0;\n    return { name, files, embedded, semantic: presetSemanticEnabled(cfg, name) };\n  });\n}\n"],"names":["anyPresetEmbeds","presetNames","presetSemanticEnabled","presetCoverage","db","cfg","embedActive","map","name","files","prepare","get","n","embedded","semantic"],"mappings":"AAEA,SAASA,eAAe,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,qBAAqB;AAUzF,8FAA8F;AAC9F,0EAA0E;AAC1E,OAAO,SAASC,eAAeC,EAAgB,EAAEC,GAAmB;IAClE,MAAMC,cAAcN,gBAAgBK;IACpC,OAAOJ,YAAYI,KAAKE,GAAG,CAAC,CAACC;QAC3B,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,2DAA2DC,GAAG,CAACH,MAAwBI,CAAC;QAClH,MAAMC,WAAWP,cAAc,AAACF,GAAGM,OAAO,CAAC,6JAA6JC,GAAG,CAACH,MAAwBI,CAAC,GAAG;QACxO,OAAO;YAAEJ;YAAMC;YAAOI;YAAUC,UAAUZ,sBAAsBG,KAAKG;QAAM;IAC7E;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig, SignalWeights } from '../config/index.ts';\nimport { anyPresetEmbeds, presetNames, presetSignals } from '../config/index.ts';\n\nexport interface PresetCoverage {\n  name: string;\n  files: number;\n  embedded: number;\n  // Reported so 0 embedded reads as \"this scope declined vectors\" rather than \"not yet built\".\n  signals: SignalWeights;\n}\n\n// Indexing derives from presets, so the derivation stays visible. Read from preset_files, not\n// recomputed from globs, so it reflects the cache rather than the config.\nexport function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[] {\n  const embedActive = anyPresetEmbeds(cfg);\n  return presetNames(cfg).map((name) => {\n    const files = (db.prepare('SELECT COUNT(*) AS n FROM preset_files WHERE preset = ?').get(name) as { n: number }).n;\n    const embedded = embedActive ? (db.prepare('SELECT COUNT(*) AS n FROM preset_files pf WHERE pf.preset = ? AND EXISTS (SELECT 1 FROM embeddings e WHERE e.\"path\" = pf.\"path\" AND e.vector IS NOT NULL)').get(name) as { n: number }).n : 0;\n    return { name, files, embedded, signals: presetSignals(cfg, name) };\n  });\n}\n"],"names":["anyPresetEmbeds","presetNames","presetSignals","presetCoverage","db","cfg","embedActive","map","name","files","prepare","get","n","embedded","signals"],"mappings":"AAEA,SAASA,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,qBAAqB;AAUjF,8FAA8F;AAC9F,0EAA0E;AAC1E,OAAO,SAASC,eAAeC,EAAgB,EAAEC,GAAmB;IAClE,MAAMC,cAAcN,gBAAgBK;IACpC,OAAOJ,YAAYI,KAAKE,GAAG,CAAC,CAACC;QAC3B,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,2DAA2DC,GAAG,CAACH,MAAwBI,CAAC;QAClH,MAAMC,WAAWP,cAAc,AAACF,GAAGM,OAAO,CAAC,6JAA6JC,GAAG,CAACH,MAAwBI,CAAC,GAAG;QACxO,OAAO;YAAEJ;YAAMC;YAAOI;YAAUC,SAASZ,cAAcG,KAAKG;QAAM;IACpE;AACF"}

package/dist/esm/config/access.d.ts

@@ -1,7 +1,9 @@
+import type { SignalName, SignalWeights } from './signals.js';
 import { type Config, type FeatureName } from './types.js';
 export declare function presetNames(cfg: Config): string[];
 export declare function embedEnabled(cfg: Config): boolean;
-export declare function presetSemanticEnabled(cfg: Config, name: string): boolean;
+export declare function presetSignals(cfg: Config, name: string): SignalWeights;
+export declare function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean;
 export declare function anyPresetEmbeds(cfg: Config): boolean;
 export declare function featureEnabled(cfg: Config, name: FeatureName): boolean;
 export declare function enabledFeatures(cfg: Config): FeatureName[];
@@ -11,9 +13,10 @@ export declare function featureStates(cfg: Config): {
 };
 export declare function embedConfig(cfg: Config): {
     model: string;
-    type: 'static' | 'api';
+    provider: 'static' | 'openai' | 'cohere';
     url?: string;
     key?: string;
+    languages?: string[];
 } | null;
 export declare function contentTokenize(cfg: Config): string | undefined;
 export declare function featureSignature(cfg: Config): string;

package/dist/esm/config/access.js

@@ -1,3 +1,4 @@
+import { modelIdentity } from '../embed/identity.js';
 import { FEATURE_NAMES } from './types.js';
 export function presetNames(cfg) {
     return Object.keys(cfg.presets);
@@ -7,17 +8,31 @@ export function embedEnabled(cfg) {
     var _cfg_embed;
     return typeof ((_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.model) === 'string' && cfg.embed.model.length > 0;
 }
-// A preset's own vector participation: absent or true means on, false means off.
-export function presetSemanticEnabled(cfg, name) {
-    var _cfg_presets_name;
-    return ((_cfg_presets_name = cfg.presets[name]) === null || _cfg_presets_name === void 0 ? void 0 : _cfg_presets_name.semantic) !== false;
+// A preset's effective signal weights: declared exhaustively when `signals` is present;
+// otherwise every signal whose prerequisite holds, each at weight 1 -- words always, links when
+// the feature is on, vectors when the tree names a model. Single source search() and status/map
+// read.
+export function presetSignals(cfg, name) {
+    var _cfg_presets_name, _cfg_features;
+    const declared = (_cfg_presets_name = cfg.presets[name]) === null || _cfg_presets_name === void 0 ? void 0 : _cfg_presets_name.signals;
+    if (declared) return declared;
+    const signals = {
+        words: 1
+    };
+    if (((_cfg_features = cfg.features) === null || _cfg_features === void 0 ? void 0 : _cfg_features.links) !== false) signals.links = 1;
+    if (embedEnabled(cfg)) signals.vectors = 1;
+    return signals;
+}
+export function presetHasSignal(cfg, name, signal) {
+    return presetSignals(cfg, name)[signal] !== undefined;
 }
-// Whether embedding needs to run: a model is named AND some preset wants vectors.
+// Whether embedding needs to run: a model is named AND some preset's effective signals include vectors.
 export function anyPresetEmbeds(cfg) {
-    return embedEnabled(cfg) && presetNames(cfg).some((name)=>presetSemanticEnabled(cfg, name));
+    return presetNames(cfg).some((name)=>presetHasSignal(cfg, name, 'vectors'));
 }
 // Opt-out features (default on): absent block or key means enabled. `rank` additionally
-// requires `links`. `embed` is derived, not a features-block member: on iff `embed` names a model.
+// requires `links`. `embed` is derived, not a features-block member: on iff some preset's
+// effective signals include vectors.
 export function featureEnabled(cfg, name) {
     var _cfg_features;
     if (name === 'embed') return anyPresetEmbeds(cfg);
@@ -41,14 +56,15 @@ export function featureStates(cfg) {
 // fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration
 // write into the file, never an implicit runtime default.
 export function embedConfig(cfg) {
-    var _e_type;
+    var _e_provider;
     if (!embedEnabled(cfg)) return null;
     const e = cfg.embed;
     return {
         model: e.model,
-        type: (_e_type = e.type) !== null && _e_type !== void 0 ? _e_type : 'static',
+        provider: (_e_provider = e.provider) !== null && _e_provider !== void 0 ? _e_provider : 'static',
         url: e.url,
-        key: e.key
+        key: e.key,
+        languages: e.languages
     };
 }
 // The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than
@@ -63,7 +79,10 @@ export function contentTokenize(cfg) {
 export function featureSignature(cfg) {
     const globalPart = enabledFeatures(cfg).filter((name)=>name !== 'embed').join(',');
     const e = embedConfig(cfg);
-    const embedPart = e ? `embed:${e.type}:${e.model}` : 'embed:off';
+    // Weight identity from identity.ts, no network: a static model's resolved sha or local
+    // size+mtime, appended once known so changed weights re-embed.
+    const identity = e && e.provider === 'static' ? modelIdentity(e.model) : undefined;
+    const embedPart = e ? `embed:${e.provider}:${e.model}${identity !== undefined ? `@${identity}` : ''}` : 'embed:off';
     // One keyed segment per preset so a rebuild notice can name exactly which preset moved.
     const presetsPart = [
         ...presetNames(cfg)
@@ -76,7 +95,7 @@ export function featureSignature(cfg) {
         const exclude = [
             ...(_p_exclude = p.exclude) !== null && _p_exclude !== void 0 ? _p_exclude : []
         ].sort().join('+');
-        return `preset:${name}:${include}:${exclude}:${p.semantic === false ? 'off' : 'on'}`;
+        return `preset:${name}:${include}:${exclude}:${presetHasSignal(cfg, name, 'vectors') ? 'on' : 'off'}`;
     }).join('|');
     const tokenize = contentTokenize(cfg);
     const parts = [

package/dist/esm/config/access.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/access.ts"],"sourcesContent":["import { type Config, type EmbedConfig, FEATURE_NAMES, type FeatureName } from './types.ts';\n\nexport function presetNames(cfg: Config): string[] {\n  return Object.keys(cfg.presets);\n}\n\n// Whether the tree names a model at all.\nexport function embedEnabled(cfg: Config): boolean {\n  return typeof cfg.embed?.model === 'string' && cfg.embed.model.length > 0;\n}\n\n// A preset's own vector participation: absent or true means on, false means off.\nexport function presetSemanticEnabled(cfg: Config, name: string): boolean {\n  return cfg.presets[name]?.semantic !== false;\n}\n\n// Whether embedding needs to run: a model is named AND some preset wants vectors.\nexport function anyPresetEmbeds(cfg: Config): boolean {\n  return embedEnabled(cfg) && presetNames(cfg).some((name) => presetSemanticEnabled(cfg, name));\n}\n\n// Opt-out features (default on): absent block or key means enabled. `rank` additionally\n// requires `links`. `embed` is derived, not a features-block member: on iff `embed` names a model.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  if (name === 'embed') return anyPresetEmbeds(cfg);\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Feature states, so \"off\" and \"empty\" stay distinguishable in output. `embed` is excluded:\n// it is not a features-block toggle, and `status` reports it on its own line.\nexport function featureStates(cfg: Config): { on: FeatureName[]; off: FeatureName[] } {\n  const reported = FEATURE_NAMES.filter((name) => name !== 'embed');\n  return {\n    on: reported.filter((name) => featureEnabled(cfg, name)),\n    off: reported.filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n\n// Resolved embed provider settings, or null when the config names no model. There is no\n// fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration\n// write into the file, never an implicit runtime default.\nexport function embedConfig(cfg: Config): { model: string; type: 'static' | 'api'; url?: string; key?: string } | null {\n  if (!embedEnabled(cfg)) return null;\n  const e = cfg.embed as EmbedConfig;\n  return { model: e.model as string, type: e.type ?? 'static', url: e.url, key: e.key };\n}\n\n// The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than\n// the default string so featureSignature can leave the segment off entirely for trees that\n// never set it, which is what keeps existing indexes from rebuilding on upgrade.\nexport function contentTokenize(cfg: Config): string | undefined {\n  const tokenize = cfg.content?.tokenize;\n  return tokenize === undefined || tokenize.trim() === '' ? undefined : tokenize.trim();\n}\n\n// Cache key over everything indexing derives from, so any change to those inputs rebuilds.\nexport function featureSignature(cfg: Config): string {\n  const globalPart = enabledFeatures(cfg)\n    .filter((name) => name !== 'embed')\n    .join(',');\n  const e = embedConfig(cfg);\n  const embedPart = e ? `embed:${e.type}:${e.model}` : 'embed:off';\n  // One keyed segment per preset so a rebuild notice can name exactly which preset moved.\n  const presetsPart = [...presetNames(cfg)]\n    .sort()\n    .map((name) => {\n      const p = cfg.presets[name];\n      const include = [...p.include].sort().join('+');\n      const exclude = [...(p.exclude ?? [])].sort().join('+');\n      return `preset:${name}:${include}:${exclude}:${p.semantic === false ? 'off' : 'on'}`;\n    })\n    .join('|');\n  const tokenize = contentTokenize(cfg);\n  const parts = [`features:${globalPart}`, embedPart];\n  if (tokenize !== undefined) parts.push(`tokenize:${tokenize}`);\n  parts.push(presetsPart);\n  return parts.join('|');\n}\n"],"names":["FEATURE_NAMES","presetNames","cfg","Object","keys","presets","embedEnabled","embed","model","length","presetSemanticEnabled","name","semantic","anyPresetEmbeds","some","featureEnabled","enabled","features","enabledFeatures","filter","featureStates","reported","on","off","embedConfig","e","type","url","key","contentTokenize","tokenize","content","undefined","trim","featureSignature","globalPart","join","embedPart","presetsPart","sort","map","p","include","exclude","parts","push"],"mappings":"AAAA,SAAwCA,aAAa,QAA0B,aAAa;AAE5F,OAAO,SAASC,YAAYC,GAAW;IACrC,OAAOC,OAAOC,IAAI,CAACF,IAAIG,OAAO;AAChC;AAEA,yCAAyC;AACzC,OAAO,SAASC,aAAaJ,GAAW;QACxBA;IAAd,OAAO,SAAOA,aAAAA,IAAIK,KAAK,cAATL,iCAAAA,WAAWM,KAAK,MAAK,YAAYN,IAAIK,KAAK,CAACC,KAAK,CAACC,MAAM,GAAG;AAC1E;AAEA,iFAAiF;AACjF,OAAO,SAASC,sBAAsBR,GAAW,EAAES,IAAY;QACtDT;IAAP,OAAOA,EAAAA,oBAAAA,IAAIG,OAAO,CAACM,KAAK,cAAjBT,wCAAAA,kBAAmBU,QAAQ,MAAK;AACzC;AAEA,kFAAkF;AAClF,OAAO,SAASC,gBAAgBX,GAAW;IACzC,OAAOI,aAAaJ,QAAQD,YAAYC,KAAKY,IAAI,CAAC,CAACH,OAASD,sBAAsBR,KAAKS;AACzF;AAEA,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,SAASI,eAAeb,GAAW,EAAES,IAAiB;QAE3CT;IADhB,IAAIS,SAAS,SAAS,OAAOE,gBAAgBX;IAC7C,MAAMc,UAAUd,EAAAA,gBAAAA,IAAIe,QAAQ,cAAZf,oCAAAA,aAAc,CAACS,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOK,WAAWD,eAAeb,KAAK;IAC3D,OAAOc;AACT;AAEA,OAAO,SAASE,gBAAgBhB,GAAW;IACzC,OAAOF,cAAcmB,MAAM,CAAC,CAACR,OAASI,eAAeb,KAAKS;AAC5D;AAEA,4FAA4F;AAC5F,8EAA8E;AAC9E,OAAO,SAASS,cAAclB,GAAW;IACvC,MAAMmB,WAAWrB,cAAcmB,MAAM,CAAC,CAACR,OAASA,SAAS;IACzD,OAAO;QACLW,IAAID,SAASF,MAAM,CAAC,CAACR,OAASI,eAAeb,KAAKS;QAClDY,KAAKF,SAASF,MAAM,CAAC,CAACR,OAAS,CAACI,eAAeb,KAAKS;IACtD;AACF;AAEA,wFAAwF;AACxF,2FAA2F;AAC3F,0DAA0D;AAC1D,OAAO,SAASa,YAAYtB,GAAW;QAGIuB;IAFzC,IAAI,CAACnB,aAAaJ,MAAM,OAAO;IAC/B,MAAMuB,IAAIvB,IAAIK,KAAK;IACnB,OAAO;QAAEC,OAAOiB,EAAEjB,KAAK;QAAYkB,IAAI,GAAED,UAAAA,EAAEC,IAAI,cAAND,qBAAAA,UAAU;QAAUE,KAAKF,EAAEE,GAAG;QAAEC,KAAKH,EAAEG,GAAG;IAAC;AACtF;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,SAASC,gBAAgB3B,GAAW;QACxBA;IAAjB,MAAM4B,YAAW5B,eAAAA,IAAI6B,OAAO,cAAX7B,mCAAAA,aAAa4B,QAAQ;IACtC,OAAOA,aAAaE,aAAaF,SAASG,IAAI,OAAO,KAAKD,YAAYF,SAASG,IAAI;AACrF;AAEA,2FAA2F;AAC3F,OAAO,SAASC,iBAAiBhC,GAAW;IAC1C,MAAMiC,aAAajB,gBAAgBhB,KAChCiB,MAAM,CAAC,CAACR,OAASA,SAAS,SAC1ByB,IAAI,CAAC;IACR,MAAMX,IAAID,YAAYtB;IACtB,MAAMmC,YAAYZ,IAAI,CAAC,MAAM,EAAEA,EAAEC,IAAI,CAAC,CAAC,EAAED,EAAEjB,KAAK,EAAE,GAAG;IACrD,wFAAwF;IACxF,MAAM8B,cAAc;WAAIrC,YAAYC;KAAK,CACtCqC,IAAI,GACJC,GAAG,CAAC,CAAC7B;YAGiB8B;QAFrB,MAAMA,IAAIvC,IAAIG,OAAO,CAACM,KAAK;QAC3B,MAAM+B,UAAU;eAAID,EAAEC,OAAO;SAAC,CAACH,IAAI,GAAGH,IAAI,CAAC;QAC3C,MAAMO,UAAU;gBAAKF,aAAAA,EAAEE,OAAO,cAATF,wBAAAA,aAAa,EAAE;SAAE,CAACF,IAAI,GAAGH,IAAI,CAAC;QACnD,OAAO,CAAC,OAAO,EAAEzB,KAAK,CAAC,EAAE+B,QAAQ,CAAC,EAAEC,QAAQ,CAAC,EAAEF,EAAE7B,QAAQ,KAAK,QAAQ,QAAQ,MAAM;IACtF,GACCwB,IAAI,CAAC;IACR,MAAMN,WAAWD,gBAAgB3B;IACjC,MAAM0C,QAAQ;QAAC,CAAC,SAAS,EAAET,YAAY;QAAEE;KAAU;IACnD,IAAIP,aAAaE,WAAWY,MAAMC,IAAI,CAAC,CAAC,SAAS,EAAEf,UAAU;IAC7Dc,MAAMC,IAAI,CAACP;IACX,OAAOM,MAAMR,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/access.ts"],"sourcesContent":["import { modelIdentity } from '../embed/identity.ts';\nimport type { SignalName, SignalWeights } from './signals.ts';\nimport { type Config, type EmbedConfig, FEATURE_NAMES, type FeatureName } from './types.ts';\n\nexport function presetNames(cfg: Config): string[] {\n  return Object.keys(cfg.presets);\n}\n\n// Whether the tree names a model at all.\nexport function embedEnabled(cfg: Config): boolean {\n  return typeof cfg.embed?.model === 'string' && cfg.embed.model.length > 0;\n}\n\n// A preset's effective signal weights: declared exhaustively when `signals` is present;\n// otherwise every signal whose prerequisite holds, each at weight 1 -- words always, links when\n// the feature is on, vectors when the tree names a model. Single source search() and status/map\n// read.\nexport function presetSignals(cfg: Config, name: string): SignalWeights {\n  const declared = cfg.presets[name]?.signals;\n  if (declared) return declared;\n  const signals: SignalWeights = { words: 1 };\n  if (cfg.features?.links !== false) signals.links = 1;\n  if (embedEnabled(cfg)) signals.vectors = 1;\n  return signals;\n}\n\nexport function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean {\n  return presetSignals(cfg, name)[signal] !== undefined;\n}\n\n// Whether embedding needs to run: a model is named AND some preset's effective signals include vectors.\nexport function anyPresetEmbeds(cfg: Config): boolean {\n  return presetNames(cfg).some((name) => presetHasSignal(cfg, name, 'vectors'));\n}\n\n// Opt-out features (default on): absent block or key means enabled. `rank` additionally\n// requires `links`. `embed` is derived, not a features-block member: on iff some preset's\n// effective signals include vectors.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  if (name === 'embed') return anyPresetEmbeds(cfg);\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Feature states, so \"off\" and \"empty\" stay distinguishable in output. `embed` is excluded:\n// it is not a features-block toggle, and `status` reports it on its own line.\nexport function featureStates(cfg: Config): { on: FeatureName[]; off: FeatureName[] } {\n  const reported = FEATURE_NAMES.filter((name) => name !== 'embed');\n  return {\n    on: reported.filter((name) => featureEnabled(cfg, name)),\n    off: reported.filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n\n// Resolved embed provider settings, or null when the config names no model. There is no\n// fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration\n// write into the file, never an implicit runtime default.\nexport function embedConfig(cfg: Config): { model: string; provider: 'static' | 'openai' | 'cohere'; url?: string; key?: string; languages?: string[] } | null {\n  if (!embedEnabled(cfg)) return null;\n  const e = cfg.embed as EmbedConfig;\n  return { model: e.model as string, provider: e.provider ?? 'static', url: e.url, key: e.key, languages: e.languages };\n}\n\n// The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than\n// the default string so featureSignature can leave the segment off entirely for trees that\n// never set it, which is what keeps existing indexes from rebuilding on upgrade.\nexport function contentTokenize(cfg: Config): string | undefined {\n  const tokenize = cfg.content?.tokenize;\n  return tokenize === undefined || tokenize.trim() === '' ? undefined : tokenize.trim();\n}\n\n// Cache key over everything indexing derives from, so any change to those inputs rebuilds.\nexport function featureSignature(cfg: Config): string {\n  const globalPart = enabledFeatures(cfg)\n    .filter((name) => name !== 'embed')\n    .join(',');\n  const e = embedConfig(cfg);\n  // Weight identity from identity.ts, no network: a static model's resolved sha or local\n  // size+mtime, appended once known so changed weights re-embed.\n  const identity = e && e.provider === 'static' ? modelIdentity(e.model) : undefined;\n  const embedPart = e ? `embed:${e.provider}:${e.model}${identity !== undefined ? `@${identity}` : ''}` : 'embed:off';\n  // One keyed segment per preset so a rebuild notice can name exactly which preset moved.\n  const presetsPart = [...presetNames(cfg)]\n    .sort()\n    .map((name) => {\n      const p = cfg.presets[name];\n      const include = [...p.include].sort().join('+');\n      const exclude = [...(p.exclude ?? [])].sort().join('+');\n      return `preset:${name}:${include}:${exclude}:${presetHasSignal(cfg, name, 'vectors') ? 'on' : 'off'}`;\n    })\n    .join('|');\n  const tokenize = contentTokenize(cfg);\n  const parts = [`features:${globalPart}`, embedPart];\n  if (tokenize !== undefined) parts.push(`tokenize:${tokenize}`);\n  parts.push(presetsPart);\n  return parts.join('|');\n}\n"],"names":["modelIdentity","FEATURE_NAMES","presetNames","cfg","Object","keys","presets","embedEnabled","embed","model","length","presetSignals","name","declared","signals","words","features","links","vectors","presetHasSignal","signal","undefined","anyPresetEmbeds","some","featureEnabled","enabled","enabledFeatures","filter","featureStates","reported","on","off","embedConfig","e","provider","url","key","languages","contentTokenize","tokenize","content","trim","featureSignature","globalPart","join","identity","embedPart","presetsPart","sort","map","p","include","exclude","parts","push"],"mappings":"AAAA,SAASA,aAAa,QAAQ,uBAAuB;AAErD,SAAwCC,aAAa,QAA0B,aAAa;AAE5F,OAAO,SAASC,YAAYC,GAAW;IACrC,OAAOC,OAAOC,IAAI,CAACF,IAAIG,OAAO;AAChC;AAEA,yCAAyC;AACzC,OAAO,SAASC,aAAaJ,GAAW;QACxBA;IAAd,OAAO,SAAOA,aAAAA,IAAIK,KAAK,cAATL,iCAAAA,WAAWM,KAAK,MAAK,YAAYN,IAAIK,KAAK,CAACC,KAAK,CAACC,MAAM,GAAG;AAC1E;AAEA,wFAAwF;AACxF,gGAAgG;AAChG,gGAAgG;AAChG,QAAQ;AACR,OAAO,SAASC,cAAcR,GAAW,EAAES,IAAY;QACpCT,mBAGbA;IAHJ,MAAMU,YAAWV,oBAAAA,IAAIG,OAAO,CAACM,KAAK,cAAjBT,wCAAAA,kBAAmBW,OAAO;IAC3C,IAAID,UAAU,OAAOA;IACrB,MAAMC,UAAyB;QAAEC,OAAO;IAAE;IAC1C,IAAIZ,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,cAAcc,KAAK,MAAK,OAAOH,QAAQG,KAAK,GAAG;IACnD,IAAIV,aAAaJ,MAAMW,QAAQI,OAAO,GAAG;IACzC,OAAOJ;AACT;AAEA,OAAO,SAASK,gBAAgBhB,GAAW,EAAES,IAAY,EAAEQ,MAAkB;IAC3E,OAAOT,cAAcR,KAAKS,KAAK,CAACQ,OAAO,KAAKC;AAC9C;AAEA,wGAAwG;AACxG,OAAO,SAASC,gBAAgBnB,GAAW;IACzC,OAAOD,YAAYC,KAAKoB,IAAI,CAAC,CAACX,OAASO,gBAAgBhB,KAAKS,MAAM;AACpE;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,qCAAqC;AACrC,OAAO,SAASY,eAAerB,GAAW,EAAES,IAAiB;QAE3CT;IADhB,IAAIS,SAAS,SAAS,OAAOU,gBAAgBnB;IAC7C,MAAMsB,UAAUtB,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,aAAc,CAACS,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOa,WAAWD,eAAerB,KAAK;IAC3D,OAAOsB;AACT;AAEA,OAAO,SAASC,gBAAgBvB,GAAW;IACzC,OAAOF,cAAc0B,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;AAC5D;AAEA,4FAA4F;AAC5F,8EAA8E;AAC9E,OAAO,SAASgB,cAAczB,GAAW;IACvC,MAAM0B,WAAW5B,cAAc0B,MAAM,CAAC,CAACf,OAASA,SAAS;IACzD,OAAO;QACLkB,IAAID,SAASF,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;QAClDmB,KAAKF,SAASF,MAAM,CAAC,CAACf,OAAS,CAACY,eAAerB,KAAKS;IACtD;AACF;AAEA,wFAAwF;AACxF,2FAA2F;AAC3F,0DAA0D;AAC1D,OAAO,SAASoB,YAAY7B,GAAW;QAGQ8B;IAF7C,IAAI,CAAC1B,aAAaJ,MAAM,OAAO;IAC/B,MAAM8B,IAAI9B,IAAIK,KAAK;IACnB,OAAO;QAAEC,OAAOwB,EAAExB,KAAK;QAAYyB,QAAQ,GAAED,cAAAA,EAAEC,QAAQ,cAAVD,yBAAAA,cAAc;QAAUE,KAAKF,EAAEE,GAAG;QAAEC,KAAKH,EAAEG,GAAG;QAAEC,WAAWJ,EAAEI,SAAS;IAAC;AACtH;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,SAASC,gBAAgBnC,GAAW;QACxBA;IAAjB,MAAMoC,YAAWpC,eAAAA,IAAIqC,OAAO,cAAXrC,mCAAAA,aAAaoC,QAAQ;IACtC,OAAOA,aAAalB,aAAakB,SAASE,IAAI,OAAO,KAAKpB,YAAYkB,SAASE,IAAI;AACrF;AAEA,2FAA2F;AAC3F,OAAO,SAASC,iBAAiBvC,GAAW;IAC1C,MAAMwC,aAAajB,gBAAgBvB,KAChCwB,MAAM,CAAC,CAACf,OAASA,SAAS,SAC1BgC,IAAI,CAAC;IACR,MAAMX,IAAID,YAAY7B;IACtB,uFAAuF;IACvF,+DAA+D;IAC/D,MAAM0C,WAAWZ,KAAKA,EAAEC,QAAQ,KAAK,WAAWlC,cAAciC,EAAExB,KAAK,IAAIY;IACzE,MAAMyB,YAAYb,IAAI,CAAC,MAAM,EAAEA,EAAEC,QAAQ,CAAC,CAAC,EAAED,EAAExB,KAAK,GAAGoC,aAAaxB,YAAY,CAAC,CAAC,EAAEwB,UAAU,GAAG,IAAI,GAAG;IACxG,wFAAwF;IACxF,MAAME,cAAc;WAAI7C,YAAYC;KAAK,CACtC6C,IAAI,GACJC,GAAG,CAAC,CAACrC;YAGiBsC;QAFrB,MAAMA,IAAI/C,IAAIG,OAAO,CAACM,KAAK;QAC3B,MAAMuC,UAAU;eAAID,EAAEC,OAAO;SAAC,CAACH,IAAI,GAAGJ,IAAI,CAAC;QAC3C,MAAMQ,UAAU;gBAAKF,aAAAA,EAAEE,OAAO,cAATF,wBAAAA,aAAa,EAAE;SAAE,CAACF,IAAI,GAAGJ,IAAI,CAAC;QACnD,OAAO,CAAC,OAAO,EAAEhC,KAAK,CAAC,EAAEuC,QAAQ,CAAC,EAAEC,QAAQ,CAAC,EAAEjC,gBAAgBhB,KAAKS,MAAM,aAAa,OAAO,OAAO;IACvG,GACCgC,IAAI,CAAC;IACR,MAAML,WAAWD,gBAAgBnC;IACjC,MAAMkD,QAAQ;QAAC,CAAC,SAAS,EAAEV,YAAY;QAAEG;KAAU;IACnD,IAAIP,aAAalB,WAAWgC,MAAMC,IAAI,CAAC,CAAC,SAAS,EAAEf,UAAU;IAC7Dc,MAAMC,IAAI,CAACP;IACX,OAAOM,MAAMT,IAAI,CAAC;AACpB"}

package/dist/esm/config/index.d.ts

@@ -1,5 +1,8 @@
-export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetNames, presetSemanticEnabled } from './access.js';
+export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.js';
+export type { InitOverrides } from './load.js';
 export { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.js';
 export { resolvePreset, resolveSearch } from './resolve.js';
+export type { SignalName, SignalWeights } from './signals.js';
+export { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.js';
 export type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.js';
 export { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.js';

package/dist/esm/config/index.js

@@ -1,4 +1,5 @@
-export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetNames, presetSemanticEnabled } from './access.js';
+export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.js';
 export { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.js';
 export { resolvePreset, resolveSearch } from './resolve.js';
+export { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.js';
 export { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.js';

package/dist/esm/config/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/index.ts"],"sourcesContent":["export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetNames, presetSemanticEnabled } from './access.ts';\nexport { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.ts';\nexport { resolvePreset, resolveSearch } from './resolve.ts';\nexport type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.ts';\nexport { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.ts';\n"],"names":["anyPresetEmbeds","contentTokenize","embedConfig","embedEnabled","enabledFeatures","featureEnabled","featureSignature","featureStates","presetNames","presetSemanticEnabled","findConfigPath","initConfig","loadConfig","migrateConfig","resolvePreset","resolveSearch","CONFIG_FILENAME","DEFAULT_EMBED_MODEL","STATE_DIR","SUPPORTED_CONFIG_VERSION"],"mappings":"AAAA,SAASA,eAAe,EAAEC,eAAe,EAAEC,WAAW,EAAEC,YAAY,EAAEC,eAAe,EAAEC,cAAc,EAAEC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAChM,SAASC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,QAAQ,YAAY;AAClF,SAASC,aAAa,EAAEC,aAAa,QAAQ,eAAe;AAE5D,SAASC,eAAe,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/index.ts"],"sourcesContent":["export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.ts';\nexport type { InitOverrides } from './load.ts';\nexport { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.ts';\nexport { resolvePreset, resolveSearch } from './resolve.ts';\nexport type { SignalName, SignalWeights } from './signals.ts';\nexport { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.ts';\nexport type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.ts';\nexport { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.ts';\n"],"names":["anyPresetEmbeds","contentTokenize","embedConfig","embedEnabled","enabledFeatures","featureEnabled","featureSignature","featureStates","presetHasSignal","presetNames","presetSignals","findConfigPath","initConfig","loadConfig","migrateConfig","resolvePreset","resolveSearch","SIGNAL_NAMES","SIGNAL_PREREQUISITES","CONFIG_FILENAME","DEFAULT_EMBED_MODEL","STATE_DIR","SUPPORTED_CONFIG_VERSION"],"mappings":"AAAA,SAASA,eAAe,EAAEC,eAAe,EAAEC,WAAW,EAAEC,YAAY,EAAEC,eAAe,EAAEC,cAAc,EAAEC,gBAAgB,EAAEC,aAAa,EAAEC,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,cAAc;AAEzM,SAASC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,QAAQ,YAAY;AAClF,SAASC,aAAa,EAAEC,aAAa,QAAQ,eAAe;AAE5D,SAASC,YAAY,EAAEC,oBAAoB,QAAQ,eAAe;AAElE,SAASC,eAAe,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,aAAa"}

package/dist/esm/config/load.d.ts

@@ -3,6 +3,11 @@ export declare function migrateConfig(cfg: Config): {
     cfg: Config;
     from: number;
 };
-export declare function initConfig(dir: string): string;
+export interface InitOverrides {
+    model?: string;
+    provider?: string;
+    url?: string;
+}
+export declare function initConfig(dir: string, overrides?: InitOverrides): string;
 export declare function findConfigPath(startDir: string): string | null;
 export declare function loadConfig(explicitPath?: string): ResolvedConfig;

package/dist/esm/config/load.js

@@ -130,6 +130,57 @@ const MIGRATIONS = {
             if (Object.keys(prev).length > 0) console.error('sense: every preset has "semantic": false, so v4 drops the "embed" block (vectors stay off); re-add it, or set a preset\'s semantic back to true, to turn them on');
         }
         return result;
+    },
+    // v4 -> v5: embed.type renames to embed.provider -- there are only three wire protocols
+    // now, so a provider name fits the shape better than a loader "type". "api" auto-migrates to
+    // "openai", the wire protocol it always was; any other value is carried over verbatim, since
+    // it is what v5 shape validation names the fix for.
+    //
+    // Also folded in here: a preset's `semantic` migrates to `signals`. false becomes an
+    // explicit exhaustive weight map (words, plus links when that feature is on, each at weight
+    // 1); true or absent needs no key at all, since default-on already means every signal whose
+    // prerequisites hold, each at weight 1. `semantic` leaves the config surface entirely once
+    // this step has run.
+    4: (cfg)=>{
+        var _cfg_features;
+        const prevEmbed = cfg.embed;
+        const result = {
+            ...cfg,
+            version: 5
+        };
+        if (prevEmbed) {
+            const { type, ...rest } = prevEmbed;
+            result.embed = type === undefined ? rest : {
+                ...rest,
+                provider: type === 'api' ? 'openai' : type
+            };
+        }
+        const linksOn = ((_cfg_features = cfg.features) === null || _cfg_features === void 0 ? void 0 : _cfg_features.links) !== false;
+        const presets = cfg.presets;
+        if (presets) {
+            const nextPresets = {};
+            for (const [name, preset] of Object.entries(presets)){
+                const { semantic, ...rest } = preset;
+                if (semantic === undefined) {
+                    nextPresets[name] = preset;
+                    continue;
+                }
+                if (typeof semantic !== 'boolean') {
+                    throw new SenseError('CONFIG_INVALID', `presets.${name}.semantic must be a boolean`);
+                }
+                nextPresets[name] = semantic === false ? {
+                    ...rest,
+                    signals: linksOn ? {
+                        words: 1,
+                        links: 1
+                    } : {
+                        words: 1
+                    }
+                } : rest;
+            }
+            result.presets = nextPresets;
+        }
+        return result;
     }
 };
 export function migrateConfig(cfg) {
@@ -146,7 +197,14 @@ export function migrateConfig(cfg) {
         from
     };
 }
-function starterConfig() {
+function starterConfig(overrides) {
+    const embed = {
+        model: DEFAULT_EMBED_MODEL,
+        provider: 'static'
+    };
+    if ((overrides === null || overrides === void 0 ? void 0 : overrides.model) !== undefined) embed.model = overrides.model;
+    if ((overrides === null || overrides === void 0 ? void 0 : overrides.provider) !== undefined) embed.provider = overrides.provider;
+    if ((overrides === null || overrides === void 0 ? void 0 : overrides.url) !== undefined) embed.url = overrides.url;
     return {
         $schema: 'https://unpkg.com/sensemaking/schema.json',
         version: SUPPORTED_CONFIG_VERSION,
@@ -165,21 +223,21 @@ function starterConfig() {
             }
         },
         // Written out rather than defaulted in code: this line is what makes `sense download`
-        // fetch 124 MB, so it belongs where it can be read and changed.
-        embed: {
-            model: DEFAULT_EMBED_MODEL,
-            type: 'static'
-        },
+        // fetch a model, so it belongs where it can be read and changed.
+        embed: embed,
         queries: {}
     };
 }
-// Refuses to overwrite an existing config.
-export function initConfig(dir) {
+// Refuses to overwrite an existing config. `overrides` come from `sense init`'s flags; an
+// invalid --provider fails validateConfig's own shape check before anything is written.
+export function initConfig(dir, overrides) {
     const configPath = join(dir, CONFIG_FILENAME);
     if (existsSync(configPath)) {
         throw new SenseError('CONFIG_EXISTS', `${CONFIG_FILENAME} already exists in ${dir}`);
     }
-    writeFileSync(configPath, `${JSON.stringify(starterConfig(), null, 2)}\n`);
+    const cfg = starterConfig(overrides);
+    validateConfig(cfg, configPath);
+    writeFileSync(configPath, `${JSON.stringify(cfg, null, 2)}\n`);
     return configPath;
 }
 export function findConfigPath(startDir) {