sensemaking 0.16.0 → 0.17.1

package/dist/esm/commands/search.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { contentTokenize, embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output.ts';\nimport { searchError } from '../search-error.ts';\nimport { segmentMatch } from '../segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nfunction lineRangeFor(db: DatabaseSync, path: string, line: number): string | null {\n  const row = db.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = rawScope(db, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = narrowByWhere(db, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);\n\n  // Terms pass to MATCH as written, with one transform: a run of text whose language marks no\n  // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier\n  // ahead of it retargets its _seg column), matching how it was indexed. Text that already\n  // carries boundaries comes through untouched, so this is a no-op for the languages that\n  // never needed it. Gated on the same predicate that decided whether the sidecars were\n  // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid\n  // syntax still errors rather than being rewritten.\n  // --where\n  // applies inside the candidate query (a post-filter would drop matches ranked past the pool)\n  // and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) materializeScope(db, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n  const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: Array<{ path: string; hit: string | null }> = [];\n  // A query that rewrites to nothing (e.g. only unspaced-script punctuation) has no\n  // searchable words: zero lexical rows, and the other signals still compose.\n  if (signals.words !== undefined && query.trim() !== '') {\n    try {\n      matchRows = wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  db.exec('DROP TABLE IF EXISTS _search');\n  db.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const rows = db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","contentTokenize","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","segmentMatch","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","db","path","row","prepare","get","start_line","end_line","search","cfg","opts","hits","chunkLines","chunkSimilarity","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hit","oversized","links","exec","insert","c","run","score","via","similarityCol","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,eAAe,EAAEC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AACjG,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,8EAA8E;AAC9E,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAASE,aAAaC,EAAgB,EAAEC,IAAY,EAAEJ,IAAY;IAChE,MAAMK,MAAMF,GAAGG,OAAO,CAAC,qIAAqIC,GAAG,CAACH,MAAMJ,MAAMA;IAC5K,OAAOK,MAAM,CAAC,CAAC,EAAEA,IAAIG,UAAU,CAAC,CAAC,EAAEH,IAAII,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOP,EAAgB,EAAEQ,GAAmB,EAAEjD,KAAa,EAAEkD,OAAsB,CAAC,CAAC;QA6EpCC,WAAwBC,iBAA8BC;IA5E3H,MAAMC,YAAYjF,cAAc4E,KAAKC;IACrC,MAAM,EAAEK,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAChB,GAAGG,OAAO,CAAC,kCAAkCc,GAAG,GAA+BlE,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAClH,MAAMkB,aAAa/E,SAAS4D,IAAIQ,KAAKC,MAAMO;IAC3C,MAAMI,cAAcD,WAAWpC,IAAI,GAAGiC,SAAS5D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAIiE;IACJ,IAAI;QACFA,eAAelF,cAAc6D,IAAImB,YAAYN,UAAUS,KAAK;IAC9D,EAAE,OAAOC,KAAK;QACZ,MAAMvF,YAAYuF,KAAchE,OAAOsD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQtC,KAAKuC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAInG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkBgG,IAAI;YACxB,MAAM,IAAI9F,WAAW,uBAAuB,CAAC,QAAQ,EAAE8E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAEjG,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAMkG,kBAAkBN,gBAAgBrF,mBAAmB2D,IAAIQ,KAAKa;IAEpE,4FAA4F;IAC5F,0FAA0F;IAC1F,yFAAyF;IACzF,wFAAwF;IACxF,sFAAsF;IACtF,6FAA6F;IAC7F,mDAAmD;IACnD,UAAU;IACV,6FAA6F;IAC7F,wDAAwD;IACxD,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIb,aAAalF,iBAAiB8D,IAAI,iBAAiBmB;IACvD,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAC3F,MAAMzE,QAAQlB,gBAAgB+E,SAASoB,YAAY3F,aAAasB,SAASA;IAEzE,MAAM8E,aAAa,IAAIC;IACvB,IAAIC,YAAyD,EAAE;IAC/D,kFAAkF;IAClF,4EAA4E;IAC5E,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAajF,MAAMO,IAAI,OAAO,IAAI;QACtD,IAAI;YACFqF,YAAY/F,gBAAgBwD,IAAIqC,YAAY1F,OAAOuF,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QAC1G,EAAE,OAAOjB,KAAK;YACZ,MAAMvF,YAAYuF,KAAchE,OAAO0E;QACzC;IACF;IACA,MAAMvB,OAAO,IAAI4B,IAAIC,UAAUxF,GAAG,CAAC,CAACmE,IAAM;YAACA,EAAEjB,IAAI;YAAEiB,EAAEuB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI/D,IAAI4D,UAAUpF,MAAM,CAAC,CAAC+D,IAAMA,EAAEuB,GAAG,KAAK,MAAM1F,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAEnF,IAAIc,QAAQ4B,KAAK,KAAKf,WAAWtF,gBAAgB0D,IAAIqC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ4B,KAAK;IAExH,IAAIhC,aAAa,IAAI2B;IACrB,IAAI1B,kBAAkB,IAAI0B;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAErB,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrE,kBAAkByD,IAAIQ,KAAK6B,YAAY9E,OAAOiE,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IACvI;IAEA3B,GAAG4C,IAAI,CAAC;IACR5C,GAAG4C,IAAI,CAAC;IACR,MAAMC,SAAS7C,GAAGG,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACF,MAAM6C,EAAE,IAAIT,WAAYQ,OAAOE,GAAG,CAAC9C,MAAM6C,EAAEE,KAAK,EAAEF,EAAEG,GAAG,GAAEvC,YAAAA,KAAKN,GAAG,CAACH,mBAATS,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWP,GAAG,CAACH,mBAAfU,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBR,GAAG,CAACH,mBAApBW,kCAAAA,uBAA6B;IAExJ,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMU,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAMiB,gBAAgBlB,kBAAkB,yBAAyB;IACjE,MAAMmB,OAAOnD,GACVG,OAAO,CACN,CAAC,kIAAkI,EAAE+C,cAAc;;OAElJ,EAAE5B,MAAM,oCAAoC,CAAC,EAE/CL,GAAG,CAACH;IAEP,IAAI4B,UAAU3D,IAAI,GAAG,GAAG;QACtB,MAAMqE,YAAY1G,iBAAiBa;QACnC,KAAK,MAAM2C,OAAOiD,KAAM;YACtB,IAAIjD,IAAIuC,GAAG,KAAK,QAAQ,CAACC,UAAUW,GAAG,CAACnD,IAAID,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAO1D,aAAaC,KAAKgF,IAAI8C,OAAO,EAAEpD,IAAID,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMmE;YACjDlD,IAAIuC,GAAG,GAAGrD;YACV,IAAIc,IAAIqD,KAAK,IAAI,MAAMrD,IAAIqD,KAAK,GAAG5H,eAAe6E,KAAK,cAAcT,aAAaC,IAAIE,IAAID,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QAC1I;IACF;IAEA,OAAO6D;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { contentTokenize, embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport { segmentMatch } from '../text/segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, rows get the JS excerpt.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // This is a raw substring scan, not porter-stemmed: a doc matched only through a\n    // stemmed variant (query \"negotiate\" vs doc \"negotiating\") finds no occurrence here.\n    // Fall back to the doc's start, unmarked, rather than claim a match that isn't there.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nfunction lineRangeFor(db: DatabaseSync, path: string, line: number): string | null {\n  const row = db.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = rawScope(db, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands. Runs the same where\n  // fragment matchSql does, so a bad column gets the same attributed error either way.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = narrowByWhere(db, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);\n\n  // Terms pass to MATCH as written, with one transform: a run of text whose language marks no\n  // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier\n  // ahead of it retargets its _seg column), matching how it was indexed. Text that already\n  // carries boundaries comes through untouched, so this is a no-op for the languages that\n  // never needed it. Gated on the same predicate that decided whether the sidecars were\n  // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid\n  // syntax still errors rather than being rewritten.\n  // --where\n  // applies inside the candidate query (a post-filter would drop matches ranked past the pool)\n  // and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // A scope narrower than the whole index must filter the candidate pool before LIMIT, not\n  // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the\n  // filter. A join against a temp table, not a bound parameter list: real scopes run to\n  // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.\n  if (scopeActive) materializeScope(db, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n  const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: Array<{ path: string; hit: string | null }> = [];\n  // A query that rewrites to nothing (e.g. only unspaced-script punctuation) has no\n  // searchable words: zero lexical rows, and the other signals still compose.\n  if (signals.words !== undefined && query.trim() !== '') {\n    try {\n      matchRows = wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  db.exec('DROP TABLE IF EXISTS _search');\n  db.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  // All three candidate paths (match, link, vector) already filtered to scope+where before\n  // reaching _search; this reapplies --where anyway since the join to frontmatter is already\n  // needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines is always present now: semantic rows carry their chunk's range, oversized-doc\n  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const rows = db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","contentTokenize","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","segmentMatch","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","db","path","row","prepare","get","start_line","end_line","search","cfg","opts","hits","chunkLines","chunkSimilarity","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hit","oversized","links","exec","insert","c","run","score","via","similarityCol","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,eAAe,EAAEC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AACjG,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,4BAA4B;AACxD,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,mGAAmG;AACnG,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,iFAAiF;QACjF,qFAAqF;QACrF,sFAAsF;QACtF,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,SAASE,aAAaC,EAAgB,EAAEC,IAAY,EAAEJ,IAAY;IAChE,MAAMK,MAAMF,GAAGG,OAAO,CAAC,qIAAqIC,GAAG,CAACH,MAAMJ,MAAMA;IAC5K,OAAOK,MAAM,CAAC,CAAC,EAAEA,IAAIG,UAAU,CAAC,CAAC,EAAEH,IAAII,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOP,EAAgB,EAAEQ,GAAmB,EAAEjD,KAAa,EAAEkD,OAAsB,CAAC,CAAC;QA6EpCC,WAAwBC,iBAA8BC;IA5E3H,MAAMC,YAAYjF,cAAc4E,KAAKC;IACrC,MAAM,EAAEK,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAChB,GAAGG,OAAO,CAAC,kCAAkCc,GAAG,GAA+BlE,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAClH,MAAMkB,aAAa/E,SAAS4D,IAAIQ,KAAKC,MAAMO;IAC3C,MAAMI,cAAcD,WAAWpC,IAAI,GAAGiC,SAAS5D,MAAM;IACrD,wFAAwF;IACxF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAIiE;IACJ,IAAI;QACFA,eAAelF,cAAc6D,IAAImB,YAAYN,UAAUS,KAAK;IAC9D,EAAE,OAAOC,KAAK;QACZ,MAAMvF,YAAYuF,KAAchE,OAAOsD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQtC,KAAKuC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAInG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkBgG,IAAI;YACxB,MAAM,IAAI9F,WAAW,uBAAuB,CAAC,QAAQ,EAAE8E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAEjG,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAMkG,kBAAkBN,gBAAgBrF,mBAAmB2D,IAAIQ,KAAKa;IAEpE,4FAA4F;IAC5F,0FAA0F;IAC1F,yFAAyF;IACzF,wFAAwF;IACxF,sFAAsF;IACtF,6FAA6F;IAC7F,mDAAmD;IACnD,UAAU;IACV,6FAA6F;IAC7F,wDAAwD;IACxD,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,uFAAuF;IACvF,sFAAsF;IACtF,uEAAuE;IACvE,IAAIb,aAAalF,iBAAiB8D,IAAI,iBAAiBmB;IACvD,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAC3F,MAAMzE,QAAQlB,gBAAgB+E,SAASoB,YAAY3F,aAAasB,SAASA;IAEzE,MAAM8E,aAAa,IAAIC;IACvB,IAAIC,YAAyD,EAAE;IAC/D,kFAAkF;IAClF,4EAA4E;IAC5E,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAajF,MAAMO,IAAI,OAAO,IAAI;QACtD,IAAI;YACFqF,YAAY/F,gBAAgBwD,IAAIqC,YAAY1F,OAAOuF,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QAC1G,EAAE,OAAOjB,KAAK;YACZ,MAAMvF,YAAYuF,KAAchE,OAAO0E;QACzC;IACF;IACA,MAAMvB,OAAO,IAAI4B,IAAIC,UAAUxF,GAAG,CAAC,CAACmE,IAAM;YAACA,EAAEjB,IAAI;YAAEiB,EAAEuB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI/D,IAAI4D,UAAUpF,MAAM,CAAC,CAAC+D,IAAMA,EAAEuB,GAAG,KAAK,MAAM1F,GAAG,CAAC,CAACmE,IAAMA,EAAEjB,IAAI;IAEnF,IAAIc,QAAQ4B,KAAK,KAAKf,WAAWtF,gBAAgB0D,IAAIqC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ4B,KAAK;IAExH,IAAIhC,aAAa,IAAI2B;IACrB,IAAI1B,kBAAkB,IAAI0B;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAErB,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrE,kBAAkByD,IAAIQ,KAAK6B,YAAY9E,OAAOiE,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IACvI;IAEA3B,GAAG4C,IAAI,CAAC;IACR5C,GAAG4C,IAAI,CAAC;IACR,MAAMC,SAAS7C,GAAGG,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACF,MAAM6C,EAAE,IAAIT,WAAYQ,OAAOE,GAAG,CAAC9C,MAAM6C,EAAEE,KAAK,EAAEF,EAAEG,GAAG,GAAEvC,YAAAA,KAAKN,GAAG,CAACH,mBAATS,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWP,GAAG,CAACH,mBAAfU,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBR,GAAG,CAACH,mBAApBW,kCAAAA,uBAA6B;IAExJ,yFAAyF;IACzF,2FAA2F;IAC3F,8BAA8B;IAC9B,MAAMU,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,2FAA2F;IAC3F,MAAMiB,gBAAgBlB,kBAAkB,yBAAyB;IACjE,MAAMmB,OAAOnD,GACVG,OAAO,CACN,CAAC,kIAAkI,EAAE+C,cAAc;;OAElJ,EAAE5B,MAAM,oCAAoC,CAAC,EAE/CL,GAAG,CAACH;IAEP,IAAI4B,UAAU3D,IAAI,GAAG,GAAG;QACtB,MAAMqE,YAAY1G,iBAAiBa;QACnC,KAAK,MAAM2C,OAAOiD,KAAM;YACtB,IAAIjD,IAAIuC,GAAG,KAAK,QAAQ,CAACC,UAAUW,GAAG,CAACnD,IAAID,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAO1D,aAAaC,KAAKgF,IAAI8C,OAAO,EAAEpD,IAAID,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMmE;YACjDlD,IAAIuC,GAAG,GAAGrD;YACV,IAAIc,IAAIqD,KAAK,IAAI,MAAMrD,IAAIqD,KAAK,GAAG5H,eAAe6E,KAAK,cAAcT,aAAaC,IAAIE,IAAID,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QAC1I;IACF;IAEA,OAAO6D;AACT"}

package/dist/esm/commands/signals.js

@@ -1,6 +1,6 @@
 import { semanticCandidates } from '../embed/query.js';
 import { linkEdges } from '../features/index.js';
-import { personalizedRank } from '../graph.js';
+import { personalizedRank } from '../graph/graph.js';
 // RRF composes whatever the preset declares; the constant lives here since it belongs to
 // the fusion these three functions share, not to search() itself.
 const RRF_K = 60;

package/dist/esm/commands/signals.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/signals.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { semanticCandidates } from '../embed/query.ts';\nimport { linkEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph.ts';\n\n// RRF composes whatever the preset declares; the constant lives here since it belongs to\n// the fusion these three functions share, not to search() itself.\nconst RRF_K = 60;\n\nexport type Candidates = Map<string, { score: number; via: string }>;\n\n// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared\n// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal\n// is seeded from its output, so search() runs this one first when \"words\" is declared.\nexport function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{ path: string; hit: string | null }> {\n  const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n  const SNIPPET_BOUND = 16_384;\n  const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const matchRows = db.prepare(matchSql).all(query) as Array<{ path: string; hit: string | null }>;\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: weight / (RRF_K + i), via: 'match' });\n  });\n  return matchRows;\n}\n\n// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this\n// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before\n// this was a signal.\nexport function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{ path: string; hit: string | null }>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void {\n  if (matchRows.length === 0) return;\n  const edges = linkEdges(db);\n  if (edges.length === 0) return;\n  // Gates the label only, not the score: restart mass ranks every seed without an incident\n  // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n  const linked = new Set(edges.flat());\n  const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n  const ranked = [...personalizedRank(allPaths, edges, seeds)]\n    .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, fetch);\n  ranked.forEach(([path], i) => {\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      if (linked.has(path)) existing.via = 'match+link';\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'link' });\n    }\n  });\n}\n\n// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`\n// at this signal's weight. Each row also carries its best chunk's line range and similarity,\n// for the caller.\nexport async function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{ chunkLines: Map<string, string>; chunkSimilarity: Map<string, number> }> {\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);\n  vec.forEach(({ path, lines, similarity }, i) => {\n    chunkLines.set(path, lines);\n    chunkSimilarity.set(path, similarity);\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      existing.via = `${existing.via}+vector`;\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'vector' });\n    }\n  });\n  return { chunkLines, chunkSimilarity };\n}\n"],"names":["semanticCandidates","linkEdges","personalizedRank","RRF_K","wordsCandidates","db","candidates","query","whereJoin","whereCond","scopeCond","fetch","weight","WEIGHTED_BM25","SNIPPET_BOUND","matchSql","matchRows","prepare","all","forEach","r","i","set","path","score","via","linksCandidates","allPaths","allowedPaths","length","edges","linked","Set","flat","seeds","Map","map","ranked","filter","has","sort","a","b","slice","existing","get","vectorsCandidates","cfg","terms","chunkLines","chunkSimilarity","vec","lines","similarity"],"mappings":"AAEA,SAASA,kBAAkB,QAAQ,oBAAoB;AACvD,SAASC,SAAS,QAAQ,uBAAuB;AACjD,SAASC,gBAAgB,QAAQ,cAAc;AAE/C,yFAAyF;AACzF,kEAAkE;AAClE,MAAMC,QAAQ;AAId,+FAA+F;AAC/F,gGAAgG;AAChG,uFAAuF;AACvF,OAAO,SAASC,gBAAgBC,EAAgB,EAAEC,UAAsB,EAAEC,KAAa,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,MAAc;IAC7K,MAAMC,gBAAgB;IACtB,MAAMC,gBAAgB;IACtB,MAAMC,WAAW,CAAC,+DAA+D,EAAED,cAAc,+EAA+E,EAAEN,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEG,cAAc,OAAO,EAAEF,OAAO;IACtR,MAAMK,YAAYX,GAAGY,OAAO,CAACF,UAAUG,GAAG,CAACX;IAC3CS,UAAUG,OAAO,CAAC,CAACC,GAAGC;QACpBf,WAAWgB,GAAG,CAACF,EAAEG,IAAI,EAAE;YAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;YAAII,KAAK;QAAQ;IACrE;IACA,OAAOT;AACT;AAEA,8FAA8F;AAC9F,gGAAgG;AAChG,qBAAqB;AACrB,OAAO,SAASU,gBAAgBrB,EAAgB,EAAEC,UAAsB,EAAEU,SAAsD,EAAEW,QAAkB,EAAEC,YAAyB,EAAEjB,KAAa,EAAEC,MAAc;IAC5M,IAAII,UAAUa,MAAM,KAAK,GAAG;IAC5B,MAAMC,QAAQ7B,UAAUI;IACxB,IAAIyB,MAAMD,MAAM,KAAK,GAAG;IACxB,yFAAyF;IACzF,uFAAuF;IACvF,MAAME,SAAS,IAAIC,IAAIF,MAAMG,IAAI;IACjC,MAAMC,QAAQ,IAAIC,IAAInB,UAAUoB,GAAG,CAAC,CAAChB,GAAGC,IAAM;YAACD,EAAEG,IAAI;YAAE,IAAKF,CAAAA,IAAI,CAAA;SAAG;IACnE,MAAMgB,SAAS;WAAInC,iBAAiByB,UAAUG,OAAOI;KAAO,CACzDI,MAAM,CAAC,CAAC,CAACf,MAAMC,MAAM,GAAKA,QAAQ,QAAQI,aAAaW,GAAG,CAAChB,OAC3DiB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGhC;IACZ0B,OAAOlB,OAAO,CAAC,CAAC,CAACI,KAAK,EAAEF;QACtB,MAAMuB,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpC,IAAIU,OAAOQ,GAAG,CAAChB,OAAOqB,SAASnB,GAAG,GAAG;QACvC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAO;QAClE;IACF;AACF;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,kBAAkB;AAClB,OAAO,eAAeqB,kBAAkBzC,EAAgB,EAAE0C,GAAmB,EAAEzC,UAAsB,EAAE0C,KAAa,EAAErC,KAAa,EAAEiB,YAAyB,EAAEhB,MAAc;IAC5K,MAAMqC,aAAa,IAAId;IACvB,MAAMe,kBAAkB,IAAIf;IAC5B,MAAMgB,MAAM,MAAMnD,mBAAmBK,IAAI0C,KAAKC,OAAOrC,OAAOiB;IAC5DuB,IAAIhC,OAAO,CAAC,CAAC,EAAEI,IAAI,EAAE6B,KAAK,EAAEC,UAAU,EAAE,EAAEhC;QACxC4B,WAAW3B,GAAG,CAACC,MAAM6B;QACrBF,gBAAgB5B,GAAG,CAACC,MAAM8B;QAC1B,MAAMT,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpCuB,SAASnB,GAAG,GAAG,GAAGmB,SAASnB,GAAG,CAAC,OAAO,CAAC;QACzC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAS;QACpE;IACF;IACA,OAAO;QAAEwB;QAAYC;IAAgB;AACvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/signals.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { semanticCandidates } from '../embed/query.ts';\nimport { linkEdges } from '../features/index.ts';\nimport { personalizedRank } from '../graph/graph.ts';\n\n// RRF composes whatever the preset declares; the constant lives here since it belongs to\n// the fusion these three functions share, not to search() itself.\nconst RRF_K = 60;\n\nexport type Candidates = Map<string, { score: number; via: string }>;\n\n// BM25 match rows, folded into `candidates` by reciprocal rank scaled by the preset's declared\n// weight for this signal (default 1, bit-identical to the unweighted score). Every other signal\n// is seeded from its output, so search() runs this one first when \"words\" is declared.\nexport function wordsCandidates(db: DatabaseSync, candidates: Candidates, query: string, whereJoin: string, whereCond: string, scopeCond: string, fetch: number, weight: number): Array<{ path: string; hit: string | null }> {\n  const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n  const SNIPPET_BOUND = 16_384;\n  const matchSql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const matchRows = db.prepare(matchSql).all(query) as Array<{ path: string; hit: string | null }>;\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: weight / (RRF_K + i), via: 'match' });\n  });\n  return matchRows;\n}\n\n// Personalized-PageRank expansion from the word-match seeds, folded into `candidates` at this\n// signal's weight. A no-op when there is nothing to seed from (matchRows empty), same as before\n// this was a signal.\nexport function linksCandidates(db: DatabaseSync, candidates: Candidates, matchRows: Array<{ path: string; hit: string | null }>, allPaths: string[], allowedPaths: Set<string>, fetch: number, weight: number): void {\n  if (matchRows.length === 0) return;\n  const edges = linkEdges(db);\n  if (edges.length === 0) return;\n  // Gates the label only, not the score: restart mass ranks every seed without an incident\n  // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).\n  const linked = new Set(edges.flat());\n  const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n  const ranked = [...personalizedRank(allPaths, edges, seeds)]\n    .filter(([path, score]) => score > 1e-9 && allowedPaths.has(path))\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, fetch);\n  ranked.forEach(([path], i) => {\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      if (linked.has(path)) existing.via = 'match+link';\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'link' });\n    }\n  });\n}\n\n// Vector expansion at the swept flat-region constants (pool = fetch), folded into `candidates`\n// at this signal's weight. Each row also carries its best chunk's line range and similarity,\n// for the caller.\nexport async function vectorsCandidates(db: DatabaseSync, cfg: ResolvedConfig, candidates: Candidates, terms: string, fetch: number, allowedPaths: Set<string>, weight: number): Promise<{ chunkLines: Map<string, string>; chunkSimilarity: Map<string, number> }> {\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  const vec = await semanticCandidates(db, cfg, terms, fetch, allowedPaths);\n  vec.forEach(({ path, lines, similarity }, i) => {\n    chunkLines.set(path, lines);\n    chunkSimilarity.set(path, similarity);\n    const existing = candidates.get(path);\n    if (existing) {\n      existing.score += weight / (RRF_K + i);\n      existing.via = `${existing.via}+vector`;\n    } else {\n      candidates.set(path, { score: weight / (RRF_K + i), via: 'vector' });\n    }\n  });\n  return { chunkLines, chunkSimilarity };\n}\n"],"names":["semanticCandidates","linkEdges","personalizedRank","RRF_K","wordsCandidates","db","candidates","query","whereJoin","whereCond","scopeCond","fetch","weight","WEIGHTED_BM25","SNIPPET_BOUND","matchSql","matchRows","prepare","all","forEach","r","i","set","path","score","via","linksCandidates","allPaths","allowedPaths","length","edges","linked","Set","flat","seeds","Map","map","ranked","filter","has","sort","a","b","slice","existing","get","vectorsCandidates","cfg","terms","chunkLines","chunkSimilarity","vec","lines","similarity"],"mappings":"AAEA,SAASA,kBAAkB,QAAQ,oBAAoB;AACvD,SAASC,SAAS,QAAQ,uBAAuB;AACjD,SAASC,gBAAgB,QAAQ,oBAAoB;AAErD,yFAAyF;AACzF,kEAAkE;AAClE,MAAMC,QAAQ;AAId,+FAA+F;AAC/F,gGAAgG;AAChG,uFAAuF;AACvF,OAAO,SAASC,gBAAgBC,EAAgB,EAAEC,UAAsB,EAAEC,KAAa,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,SAAiB,EAAEC,KAAa,EAAEC,MAAc;IAC7K,MAAMC,gBAAgB;IACtB,MAAMC,gBAAgB;IACtB,MAAMC,WAAW,CAAC,+DAA+D,EAAED,cAAc,+EAA+E,EAAEN,UAAU,uBAAuB,EAAEC,UAAU,CAAC,EAAEC,UAAU,UAAU,EAAEG,cAAc,OAAO,EAAEF,OAAO;IACtR,MAAMK,YAAYX,GAAGY,OAAO,CAACF,UAAUG,GAAG,CAACX;IAC3CS,UAAUG,OAAO,CAAC,CAACC,GAAGC;QACpBf,WAAWgB,GAAG,CAACF,EAAEG,IAAI,EAAE;YAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;YAAII,KAAK;QAAQ;IACrE;IACA,OAAOT;AACT;AAEA,8FAA8F;AAC9F,gGAAgG;AAChG,qBAAqB;AACrB,OAAO,SAASU,gBAAgBrB,EAAgB,EAAEC,UAAsB,EAAEU,SAAsD,EAAEW,QAAkB,EAAEC,YAAyB,EAAEjB,KAAa,EAAEC,MAAc;IAC5M,IAAII,UAAUa,MAAM,KAAK,GAAG;IAC5B,MAAMC,QAAQ7B,UAAUI;IACxB,IAAIyB,MAAMD,MAAM,KAAK,GAAG;IACxB,yFAAyF;IACzF,uFAAuF;IACvF,MAAME,SAAS,IAAIC,IAAIF,MAAMG,IAAI;IACjC,MAAMC,QAAQ,IAAIC,IAAInB,UAAUoB,GAAG,CAAC,CAAChB,GAAGC,IAAM;YAACD,EAAEG,IAAI;YAAE,IAAKF,CAAAA,IAAI,CAAA;SAAG;IACnE,MAAMgB,SAAS;WAAInC,iBAAiByB,UAAUG,OAAOI;KAAO,CACzDI,MAAM,CAAC,CAAC,CAACf,MAAMC,MAAM,GAAKA,QAAQ,QAAQI,aAAaW,GAAG,CAAChB,OAC3DiB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGhC;IACZ0B,OAAOlB,OAAO,CAAC,CAAC,CAACI,KAAK,EAAEF;QACtB,MAAMuB,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpC,IAAIU,OAAOQ,GAAG,CAAChB,OAAOqB,SAASnB,GAAG,GAAG;QACvC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAO;QAClE;IACF;AACF;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,kBAAkB;AAClB,OAAO,eAAeqB,kBAAkBzC,EAAgB,EAAE0C,GAAmB,EAAEzC,UAAsB,EAAE0C,KAAa,EAAErC,KAAa,EAAEiB,YAAyB,EAAEhB,MAAc;IAC5K,MAAMqC,aAAa,IAAId;IACvB,MAAMe,kBAAkB,IAAIf;IAC5B,MAAMgB,MAAM,MAAMnD,mBAAmBK,IAAI0C,KAAKC,OAAOrC,OAAOiB;IAC5DuB,IAAIhC,OAAO,CAAC,CAAC,EAAEI,IAAI,EAAE6B,KAAK,EAAEC,UAAU,EAAE,EAAEhC;QACxC4B,WAAW3B,GAAG,CAACC,MAAM6B;QACrBF,gBAAgB5B,GAAG,CAACC,MAAM8B;QAC1B,MAAMT,WAAWtC,WAAWuC,GAAG,CAACtB;QAChC,IAAIqB,UAAU;YACZA,SAASpB,KAAK,IAAIZ,SAAUT,CAAAA,QAAQkB,CAAAA;YACpCuB,SAASnB,GAAG,GAAG,GAAGmB,SAASnB,GAAG,CAAC,OAAO,CAAC;QACzC,OAAO;YACLnB,WAAWgB,GAAG,CAACC,MAAM;gBAAEC,OAAOZ,SAAUT,CAAAA,QAAQkB,CAAAA;gBAAII,KAAK;YAAS;QACpE;IACF;IACA,OAAO;QAAEwB;QAAYC;IAAgB;AACvC"}

package/dist/esm/config/access.d.ts

@@ -1,3 +1,4 @@
+import type { Feature } from '../features/types.js';
 import type { SignalName, SignalWeights } from './signals.js';
 import { type Config, type FeatureName } from './types.js';
 export declare function presetNames(cfg: Config): string[];
@@ -17,6 +18,7 @@ export declare function embedConfig(cfg: Config): {
     url?: string;
     key?: string;
     languages?: string[];
+    chunkTokens?: number;
 } | null;
 export declare function contentTokenize(cfg: Config): string | undefined;
-export declare function featureSignature(cfg: Config): string;
+export declare function featureSignature(cfg: Config, features: Feature[]): string;

package/dist/esm/config/access.js

@@ -1,4 +1,3 @@
-import { modelIdentity } from '../embed/identity.js';
 import { FEATURE_NAMES } from './types.js';
 export function presetNames(cfg) {
     return Object.keys(cfg.presets);
@@ -64,7 +63,8 @@ export function embedConfig(cfg) {
         provider: (_e_provider = e.provider) !== null && _e_provider !== void 0 ? _e_provider : 'static',
         url: e.url,
         key: e.key,
-        languages: e.languages
+        languages: e.languages,
+        chunkTokens: e.chunkTokens
     };
 }
 // The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than
@@ -76,13 +76,13 @@ export function contentTokenize(cfg) {
     return tokenize === undefined || tokenize.trim() === '' ? undefined : tokenize.trim();
 }
 // Cache key over everything indexing derives from, so any change to those inputs rebuilds.
-export function featureSignature(cfg) {
+// Segment order is fixed (features list, then `features` in registry order, then tokenize, then presets) since reordering alone would fire a spurious rebuild; tokenize/presets stay here because no Feature module owns either.
+export function featureSignature(cfg, features) {
     const globalPart = enabledFeatures(cfg).filter((name)=>name !== 'embed').join(',');
-    const e = embedConfig(cfg);
-    // Weight identity from identity.ts, no network: a static model's resolved sha or local
-    // size+mtime, appended once known so changed weights re-embed.
-    const identity = e && e.provider === 'static' ? modelIdentity(e.model) : undefined;
-    const embedPart = e ? `embed:${e.provider}:${e.model}${identity !== undefined ? `@${identity}` : ''}` : 'embed:off';
+    const featureParts = features.map((feature)=>{
+        var _feature_signature;
+        return (_feature_signature = feature.signature) === null || _feature_signature === void 0 ? void 0 : _feature_signature.call(feature, cfg);
+    }).filter((part)=>part !== undefined);
     // One keyed segment per preset so a rebuild notice can name exactly which preset moved.
     const presetsPart = [
         ...presetNames(cfg)
@@ -100,7 +100,7 @@ export function featureSignature(cfg) {
     const tokenize = contentTokenize(cfg);
     const parts = [
         `features:${globalPart}`,
-        embedPart
+        ...featureParts
     ];
     if (tokenize !== undefined) parts.push(`tokenize:${tokenize}`);
     parts.push(presetsPart);

package/dist/esm/config/access.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/access.ts"],"sourcesContent":["import { modelIdentity } from '../embed/identity.ts';\nimport type { SignalName, SignalWeights } from './signals.ts';\nimport { type Config, type EmbedConfig, FEATURE_NAMES, type FeatureName } from './types.ts';\n\nexport function presetNames(cfg: Config): string[] {\n  return Object.keys(cfg.presets);\n}\n\n// Whether the tree names a model at all.\nexport function embedEnabled(cfg: Config): boolean {\n  return typeof cfg.embed?.model === 'string' && cfg.embed.model.length > 0;\n}\n\n// A preset's effective signal weights: declared exhaustively when `signals` is present;\n// otherwise every signal whose prerequisite holds, each at weight 1 -- words always, links when\n// the feature is on, vectors when the tree names a model. Single source search() and status/map\n// read.\nexport function presetSignals(cfg: Config, name: string): SignalWeights {\n  const declared = cfg.presets[name]?.signals;\n  if (declared) return declared;\n  const signals: SignalWeights = { words: 1 };\n  if (cfg.features?.links !== false) signals.links = 1;\n  if (embedEnabled(cfg)) signals.vectors = 1;\n  return signals;\n}\n\nexport function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean {\n  return presetSignals(cfg, name)[signal] !== undefined;\n}\n\n// Whether embedding needs to run: a model is named AND some preset's effective signals include vectors.\nexport function anyPresetEmbeds(cfg: Config): boolean {\n  return presetNames(cfg).some((name) => presetHasSignal(cfg, name, 'vectors'));\n}\n\n// Opt-out features (default on): absent block or key means enabled. `rank` additionally\n// requires `links`. `embed` is derived, not a features-block member: on iff some preset's\n// effective signals include vectors.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  if (name === 'embed') return anyPresetEmbeds(cfg);\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Feature states, so \"off\" and \"empty\" stay distinguishable in output. `embed` is excluded:\n// it is not a features-block toggle, and `status` reports it on its own line.\nexport function featureStates(cfg: Config): { on: FeatureName[]; off: FeatureName[] } {\n  const reported = FEATURE_NAMES.filter((name) => name !== 'embed');\n  return {\n    on: reported.filter((name) => featureEnabled(cfg, name)),\n    off: reported.filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n\n// Resolved embed provider settings, or null when the config names no model. There is no\n// fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration\n// write into the file, never an implicit runtime default.\nexport function embedConfig(cfg: Config): { model: string; provider: 'static' | 'openai' | 'cohere'; url?: string; key?: string; languages?: string[] } | null {\n  if (!embedEnabled(cfg)) return null;\n  const e = cfg.embed as EmbedConfig;\n  return { model: e.model as string, provider: e.provider ?? 'static', url: e.url, key: e.key, languages: e.languages };\n}\n\n// The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than\n// the default string so featureSignature can leave the segment off entirely for trees that\n// never set it, which is what keeps existing indexes from rebuilding on upgrade.\nexport function contentTokenize(cfg: Config): string | undefined {\n  const tokenize = cfg.content?.tokenize;\n  return tokenize === undefined || tokenize.trim() === '' ? undefined : tokenize.trim();\n}\n\n// Cache key over everything indexing derives from, so any change to those inputs rebuilds.\nexport function featureSignature(cfg: Config): string {\n  const globalPart = enabledFeatures(cfg)\n    .filter((name) => name !== 'embed')\n    .join(',');\n  const e = embedConfig(cfg);\n  // Weight identity from identity.ts, no network: a static model's resolved sha or local\n  // size+mtime, appended once known so changed weights re-embed.\n  const identity = e && e.provider === 'static' ? modelIdentity(e.model) : undefined;\n  const embedPart = e ? `embed:${e.provider}:${e.model}${identity !== undefined ? `@${identity}` : ''}` : 'embed:off';\n  // One keyed segment per preset so a rebuild notice can name exactly which preset moved.\n  const presetsPart = [...presetNames(cfg)]\n    .sort()\n    .map((name) => {\n      const p = cfg.presets[name];\n      const include = [...p.include].sort().join('+');\n      const exclude = [...(p.exclude ?? [])].sort().join('+');\n      return `preset:${name}:${include}:${exclude}:${presetHasSignal(cfg, name, 'vectors') ? 'on' : 'off'}`;\n    })\n    .join('|');\n  const tokenize = contentTokenize(cfg);\n  const parts = [`features:${globalPart}`, embedPart];\n  if (tokenize !== undefined) parts.push(`tokenize:${tokenize}`);\n  parts.push(presetsPart);\n  return parts.join('|');\n}\n"],"names":["modelIdentity","FEATURE_NAMES","presetNames","cfg","Object","keys","presets","embedEnabled","embed","model","length","presetSignals","name","declared","signals","words","features","links","vectors","presetHasSignal","signal","undefined","anyPresetEmbeds","some","featureEnabled","enabled","enabledFeatures","filter","featureStates","reported","on","off","embedConfig","e","provider","url","key","languages","contentTokenize","tokenize","content","trim","featureSignature","globalPart","join","identity","embedPart","presetsPart","sort","map","p","include","exclude","parts","push"],"mappings":"AAAA,SAASA,aAAa,QAAQ,uBAAuB;AAErD,SAAwCC,aAAa,QAA0B,aAAa;AAE5F,OAAO,SAASC,YAAYC,GAAW;IACrC,OAAOC,OAAOC,IAAI,CAACF,IAAIG,OAAO;AAChC;AAEA,yCAAyC;AACzC,OAAO,SAASC,aAAaJ,GAAW;QACxBA;IAAd,OAAO,SAAOA,aAAAA,IAAIK,KAAK,cAATL,iCAAAA,WAAWM,KAAK,MAAK,YAAYN,IAAIK,KAAK,CAACC,KAAK,CAACC,MAAM,GAAG;AAC1E;AAEA,wFAAwF;AACxF,gGAAgG;AAChG,gGAAgG;AAChG,QAAQ;AACR,OAAO,SAASC,cAAcR,GAAW,EAAES,IAAY;QACpCT,mBAGbA;IAHJ,MAAMU,YAAWV,oBAAAA,IAAIG,OAAO,CAACM,KAAK,cAAjBT,wCAAAA,kBAAmBW,OAAO;IAC3C,IAAID,UAAU,OAAOA;IACrB,MAAMC,UAAyB;QAAEC,OAAO;IAAE;IAC1C,IAAIZ,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,cAAcc,KAAK,MAAK,OAAOH,QAAQG,KAAK,GAAG;IACnD,IAAIV,aAAaJ,MAAMW,QAAQI,OAAO,GAAG;IACzC,OAAOJ;AACT;AAEA,OAAO,SAASK,gBAAgBhB,GAAW,EAAES,IAAY,EAAEQ,MAAkB;IAC3E,OAAOT,cAAcR,KAAKS,KAAK,CAACQ,OAAO,KAAKC;AAC9C;AAEA,wGAAwG;AACxG,OAAO,SAASC,gBAAgBnB,GAAW;IACzC,OAAOD,YAAYC,KAAKoB,IAAI,CAAC,CAACX,OAASO,gBAAgBhB,KAAKS,MAAM;AACpE;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,qCAAqC;AACrC,OAAO,SAASY,eAAerB,GAAW,EAAES,IAAiB;QAE3CT;IADhB,IAAIS,SAAS,SAAS,OAAOU,gBAAgBnB;IAC7C,MAAMsB,UAAUtB,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,aAAc,CAACS,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOa,WAAWD,eAAerB,KAAK;IAC3D,OAAOsB;AACT;AAEA,OAAO,SAASC,gBAAgBvB,GAAW;IACzC,OAAOF,cAAc0B,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;AAC5D;AAEA,4FAA4F;AAC5F,8EAA8E;AAC9E,OAAO,SAASgB,cAAczB,GAAW;IACvC,MAAM0B,WAAW5B,cAAc0B,MAAM,CAAC,CAACf,OAASA,SAAS;IACzD,OAAO;QACLkB,IAAID,SAASF,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;QAClDmB,KAAKF,SAASF,MAAM,CAAC,CAACf,OAAS,CAACY,eAAerB,KAAKS;IACtD;AACF;AAEA,wFAAwF;AACxF,2FAA2F;AAC3F,0DAA0D;AAC1D,OAAO,SAASoB,YAAY7B,GAAW;QAGQ8B;IAF7C,IAAI,CAAC1B,aAAaJ,MAAM,OAAO;IAC/B,MAAM8B,IAAI9B,IAAIK,KAAK;IACnB,OAAO;QAAEC,OAAOwB,EAAExB,KAAK;QAAYyB,QAAQ,GAAED,cAAAA,EAAEC,QAAQ,cAAVD,yBAAAA,cAAc;QAAUE,KAAKF,EAAEE,GAAG;QAAEC,KAAKH,EAAEG,GAAG;QAAEC,WAAWJ,EAAEI,SAAS;IAAC;AACtH;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,SAASC,gBAAgBnC,GAAW;QACxBA;IAAjB,MAAMoC,YAAWpC,eAAAA,IAAIqC,OAAO,cAAXrC,mCAAAA,aAAaoC,QAAQ;IACtC,OAAOA,aAAalB,aAAakB,SAASE,IAAI,OAAO,KAAKpB,YAAYkB,SAASE,IAAI;AACrF;AAEA,2FAA2F;AAC3F,OAAO,SAASC,iBAAiBvC,GAAW;IAC1C,MAAMwC,aAAajB,gBAAgBvB,KAChCwB,MAAM,CAAC,CAACf,OAASA,SAAS,SAC1BgC,IAAI,CAAC;IACR,MAAMX,IAAID,YAAY7B;IACtB,uFAAuF;IACvF,+DAA+D;IAC/D,MAAM0C,WAAWZ,KAAKA,EAAEC,QAAQ,KAAK,WAAWlC,cAAciC,EAAExB,KAAK,IAAIY;IACzE,MAAMyB,YAAYb,IAAI,CAAC,MAAM,EAAEA,EAAEC,QAAQ,CAAC,CAAC,EAAED,EAAExB,KAAK,GAAGoC,aAAaxB,YAAY,CAAC,CAAC,EAAEwB,UAAU,GAAG,IAAI,GAAG;IACxG,wFAAwF;IACxF,MAAME,cAAc;WAAI7C,YAAYC;KAAK,CACtC6C,IAAI,GACJC,GAAG,CAAC,CAACrC;YAGiBsC;QAFrB,MAAMA,IAAI/C,IAAIG,OAAO,CAACM,KAAK;QAC3B,MAAMuC,UAAU;eAAID,EAAEC,OAAO;SAAC,CAACH,IAAI,GAAGJ,IAAI,CAAC;QAC3C,MAAMQ,UAAU;gBAAKF,aAAAA,EAAEE,OAAO,cAATF,wBAAAA,aAAa,EAAE;SAAE,CAACF,IAAI,GAAGJ,IAAI,CAAC;QACnD,OAAO,CAAC,OAAO,EAAEhC,KAAK,CAAC,EAAEuC,QAAQ,CAAC,EAAEC,QAAQ,CAAC,EAAEjC,gBAAgBhB,KAAKS,MAAM,aAAa,OAAO,OAAO;IACvG,GACCgC,IAAI,CAAC;IACR,MAAML,WAAWD,gBAAgBnC;IACjC,MAAMkD,QAAQ;QAAC,CAAC,SAAS,EAAEV,YAAY;QAAEG;KAAU;IACnD,IAAIP,aAAalB,WAAWgC,MAAMC,IAAI,CAAC,CAAC,SAAS,EAAEf,UAAU;IAC7Dc,MAAMC,IAAI,CAACP;IACX,OAAOM,MAAMT,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/access.ts"],"sourcesContent":["import type { Feature } from '../features/types.ts';\nimport type { SignalName, SignalWeights } from './signals.ts';\nimport { type Config, type EmbedConfig, FEATURE_NAMES, type FeatureName } from './types.ts';\n\nexport function presetNames(cfg: Config): string[] {\n  return Object.keys(cfg.presets);\n}\n\n// Whether the tree names a model at all.\nexport function embedEnabled(cfg: Config): boolean {\n  return typeof cfg.embed?.model === 'string' && cfg.embed.model.length > 0;\n}\n\n// A preset's effective signal weights: declared exhaustively when `signals` is present;\n// otherwise every signal whose prerequisite holds, each at weight 1 -- words always, links when\n// the feature is on, vectors when the tree names a model. Single source search() and status/map\n// read.\nexport function presetSignals(cfg: Config, name: string): SignalWeights {\n  const declared = cfg.presets[name]?.signals;\n  if (declared) return declared;\n  const signals: SignalWeights = { words: 1 };\n  if (cfg.features?.links !== false) signals.links = 1;\n  if (embedEnabled(cfg)) signals.vectors = 1;\n  return signals;\n}\n\nexport function presetHasSignal(cfg: Config, name: string, signal: SignalName): boolean {\n  return presetSignals(cfg, name)[signal] !== undefined;\n}\n\n// Whether embedding needs to run: a model is named AND some preset's effective signals include vectors.\nexport function anyPresetEmbeds(cfg: Config): boolean {\n  return presetNames(cfg).some((name) => presetHasSignal(cfg, name, 'vectors'));\n}\n\n// Opt-out features (default on): absent block or key means enabled. `rank` additionally\n// requires `links`. `embed` is derived, not a features-block member: on iff some preset's\n// effective signals include vectors.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  if (name === 'embed') return anyPresetEmbeds(cfg);\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Feature states, so \"off\" and \"empty\" stay distinguishable in output. `embed` is excluded:\n// it is not a features-block toggle, and `status` reports it on its own line.\nexport function featureStates(cfg: Config): { on: FeatureName[]; off: FeatureName[] } {\n  const reported = FEATURE_NAMES.filter((name) => name !== 'embed');\n  return {\n    on: reported.filter((name) => featureEnabled(cfg, name)),\n    off: reported.filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n\n// Resolved embed provider settings, or null when the config names no model. There is no\n// fallback to DEFAULT_EMBED_MODEL: that constant is a template `init` and the v4 migration\n// write into the file, never an implicit runtime default.\nexport function embedConfig(cfg: Config): { model: string; provider: 'static' | 'openai' | 'cohere'; url?: string; key?: string; languages?: string[]; chunkTokens?: number } | null {\n  if (!embedEnabled(cfg)) return null;\n  const e = cfg.embed as EmbedConfig;\n  return { model: e.model as string, provider: e.provider ?? 'static', url: e.url, key: e.key, languages: e.languages, chunkTokens: e.chunkTokens };\n}\n\n// The configured FTS5 tokenizer, or undefined for the built-in default. Undefined rather than\n// the default string so featureSignature can leave the segment off entirely for trees that\n// never set it, which is what keeps existing indexes from rebuilding on upgrade.\nexport function contentTokenize(cfg: Config): string | undefined {\n  const tokenize = cfg.content?.tokenize;\n  return tokenize === undefined || tokenize.trim() === '' ? undefined : tokenize.trim();\n}\n\n// Cache key over everything indexing derives from, so any change to those inputs rebuilds.\n// Segment order is fixed (features list, then `features` in registry order, then tokenize, then presets) since reordering alone would fire a spurious rebuild; tokenize/presets stay here because no Feature module owns either.\nexport function featureSignature(cfg: Config, features: Feature[]): string {\n  const globalPart = enabledFeatures(cfg)\n    .filter((name) => name !== 'embed')\n    .join(',');\n  const featureParts = features.map((feature) => feature.signature?.(cfg)).filter((part): part is string => part !== undefined);\n  // One keyed segment per preset so a rebuild notice can name exactly which preset moved.\n  const presetsPart = [...presetNames(cfg)]\n    .sort()\n    .map((name) => {\n      const p = cfg.presets[name];\n      const include = [...p.include].sort().join('+');\n      const exclude = [...(p.exclude ?? [])].sort().join('+');\n      return `preset:${name}:${include}:${exclude}:${presetHasSignal(cfg, name, 'vectors') ? 'on' : 'off'}`;\n    })\n    .join('|');\n  const tokenize = contentTokenize(cfg);\n  const parts = [`features:${globalPart}`, ...featureParts];\n  if (tokenize !== undefined) parts.push(`tokenize:${tokenize}`);\n  parts.push(presetsPart);\n  return parts.join('|');\n}\n"],"names":["FEATURE_NAMES","presetNames","cfg","Object","keys","presets","embedEnabled","embed","model","length","presetSignals","name","declared","signals","words","features","links","vectors","presetHasSignal","signal","undefined","anyPresetEmbeds","some","featureEnabled","enabled","enabledFeatures","filter","featureStates","reported","on","off","embedConfig","e","provider","url","key","languages","chunkTokens","contentTokenize","tokenize","content","trim","featureSignature","globalPart","join","featureParts","map","feature","signature","part","presetsPart","sort","p","include","exclude","parts","push"],"mappings":"AAEA,SAAwCA,aAAa,QAA0B,aAAa;AAE5F,OAAO,SAASC,YAAYC,GAAW;IACrC,OAAOC,OAAOC,IAAI,CAACF,IAAIG,OAAO;AAChC;AAEA,yCAAyC;AACzC,OAAO,SAASC,aAAaJ,GAAW;QACxBA;IAAd,OAAO,SAAOA,aAAAA,IAAIK,KAAK,cAATL,iCAAAA,WAAWM,KAAK,MAAK,YAAYN,IAAIK,KAAK,CAACC,KAAK,CAACC,MAAM,GAAG;AAC1E;AAEA,wFAAwF;AACxF,gGAAgG;AAChG,gGAAgG;AAChG,QAAQ;AACR,OAAO,SAASC,cAAcR,GAAW,EAAES,IAAY;QACpCT,mBAGbA;IAHJ,MAAMU,YAAWV,oBAAAA,IAAIG,OAAO,CAACM,KAAK,cAAjBT,wCAAAA,kBAAmBW,OAAO;IAC3C,IAAID,UAAU,OAAOA;IACrB,MAAMC,UAAyB;QAAEC,OAAO;IAAE;IAC1C,IAAIZ,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,cAAcc,KAAK,MAAK,OAAOH,QAAQG,KAAK,GAAG;IACnD,IAAIV,aAAaJ,MAAMW,QAAQI,OAAO,GAAG;IACzC,OAAOJ;AACT;AAEA,OAAO,SAASK,gBAAgBhB,GAAW,EAAES,IAAY,EAAEQ,MAAkB;IAC3E,OAAOT,cAAcR,KAAKS,KAAK,CAACQ,OAAO,KAAKC;AAC9C;AAEA,wGAAwG;AACxG,OAAO,SAASC,gBAAgBnB,GAAW;IACzC,OAAOD,YAAYC,KAAKoB,IAAI,CAAC,CAACX,OAASO,gBAAgBhB,KAAKS,MAAM;AACpE;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,qCAAqC;AACrC,OAAO,SAASY,eAAerB,GAAW,EAAES,IAAiB;QAE3CT;IADhB,IAAIS,SAAS,SAAS,OAAOU,gBAAgBnB;IAC7C,MAAMsB,UAAUtB,EAAAA,gBAAAA,IAAIa,QAAQ,cAAZb,oCAAAA,aAAc,CAACS,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOa,WAAWD,eAAerB,KAAK;IAC3D,OAAOsB;AACT;AAEA,OAAO,SAASC,gBAAgBvB,GAAW;IACzC,OAAOF,cAAc0B,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;AAC5D;AAEA,4FAA4F;AAC5F,8EAA8E;AAC9E,OAAO,SAASgB,cAAczB,GAAW;IACvC,MAAM0B,WAAW5B,cAAc0B,MAAM,CAAC,CAACf,OAASA,SAAS;IACzD,OAAO;QACLkB,IAAID,SAASF,MAAM,CAAC,CAACf,OAASY,eAAerB,KAAKS;QAClDmB,KAAKF,SAASF,MAAM,CAAC,CAACf,OAAS,CAACY,eAAerB,KAAKS;IACtD;AACF;AAEA,wFAAwF;AACxF,2FAA2F;AAC3F,0DAA0D;AAC1D,OAAO,SAASoB,YAAY7B,GAAW;QAGQ8B;IAF7C,IAAI,CAAC1B,aAAaJ,MAAM,OAAO;IAC/B,MAAM8B,IAAI9B,IAAIK,KAAK;IACnB,OAAO;QAAEC,OAAOwB,EAAExB,KAAK;QAAYyB,QAAQ,GAAED,cAAAA,EAAEC,QAAQ,cAAVD,yBAAAA,cAAc;QAAUE,KAAKF,EAAEE,GAAG;QAAEC,KAAKH,EAAEG,GAAG;QAAEC,WAAWJ,EAAEI,SAAS;QAAEC,aAAaL,EAAEK,WAAW;IAAC;AAClJ;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,SAASC,gBAAgBpC,GAAW;QACxBA;IAAjB,MAAMqC,YAAWrC,eAAAA,IAAIsC,OAAO,cAAXtC,mCAAAA,aAAaqC,QAAQ;IACtC,OAAOA,aAAanB,aAAamB,SAASE,IAAI,OAAO,KAAKrB,YAAYmB,SAASE,IAAI;AACrF;AAEA,2FAA2F;AAC3F,iOAAiO;AACjO,OAAO,SAASC,iBAAiBxC,GAAW,EAAEa,QAAmB;IAC/D,MAAM4B,aAAalB,gBAAgBvB,KAChCwB,MAAM,CAAC,CAACf,OAASA,SAAS,SAC1BiC,IAAI,CAAC;IACR,MAAMC,eAAe9B,SAAS+B,GAAG,CAAC,CAACC;YAAYA;gBAAAA,qBAAAA,QAAQC,SAAS,cAAjBD,yCAAAA,wBAAAA,SAAoB7C;OAAMwB,MAAM,CAAC,CAACuB,OAAyBA,SAAS7B;IACnH,wFAAwF;IACxF,MAAM8B,cAAc;WAAIjD,YAAYC;KAAK,CACtCiD,IAAI,GACJL,GAAG,CAAC,CAACnC;YAGiByC;QAFrB,MAAMA,IAAIlD,IAAIG,OAAO,CAACM,KAAK;QAC3B,MAAM0C,UAAU;eAAID,EAAEC,OAAO;SAAC,CAACF,IAAI,GAAGP,IAAI,CAAC;QAC3C,MAAMU,UAAU;gBAAKF,aAAAA,EAAEE,OAAO,cAATF,wBAAAA,aAAa,EAAE;SAAE,CAACD,IAAI,GAAGP,IAAI,CAAC;QACnD,OAAO,CAAC,OAAO,EAAEjC,KAAK,CAAC,EAAE0C,QAAQ,CAAC,EAAEC,QAAQ,CAAC,EAAEpC,gBAAgBhB,KAAKS,MAAM,aAAa,OAAO,OAAO;IACvG,GACCiC,IAAI,CAAC;IACR,MAAML,WAAWD,gBAAgBpC;IACjC,MAAMqD,QAAQ;QAAC,CAAC,SAAS,EAAEZ,YAAY;WAAKE;KAAa;IACzD,IAAIN,aAAanB,WAAWmC,MAAMC,IAAI,CAAC,CAAC,SAAS,EAAEjB,UAAU;IAC7DgB,MAAMC,IAAI,CAACN;IACX,OAAOK,MAAMX,IAAI,CAAC;AACpB"}

package/dist/esm/config/index.d.ts

@@ -6,3 +6,4 @@ export type { SignalName, SignalWeights } from './signals.js';
 export { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.js';
 export type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.js';
 export { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.js';
+export { KNOWN_EMBED_KEYS } from './validate.js';

package/dist/esm/config/index.js

@@ -3,3 +3,4 @@ export { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.js
 export { resolvePreset, resolveSearch } from './resolve.js';
 export { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.js';
 export { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.js';
+export { KNOWN_EMBED_KEYS } from './validate.js';

package/dist/esm/config/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/index.ts"],"sourcesContent":["export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.ts';\nexport type { InitOverrides } from './load.ts';\nexport { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.ts';\nexport { resolvePreset, resolveSearch } from './resolve.ts';\nexport type { SignalName, SignalWeights } from './signals.ts';\nexport { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.ts';\nexport type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.ts';\nexport { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.ts';\n"],"names":["anyPresetEmbeds","contentTokenize","embedConfig","embedEnabled","enabledFeatures","featureEnabled","featureSignature","featureStates","presetHasSignal","presetNames","presetSignals","findConfigPath","initConfig","loadConfig","migrateConfig","resolvePreset","resolveSearch","SIGNAL_NAMES","SIGNAL_PREREQUISITES","CONFIG_FILENAME","DEFAULT_EMBED_MODEL","STATE_DIR","SUPPORTED_CONFIG_VERSION"],"mappings":"AAAA,SAASA,eAAe,EAAEC,eAAe,EAAEC,WAAW,EAAEC,YAAY,EAAEC,eAAe,EAAEC,cAAc,EAAEC,gBAAgB,EAAEC,aAAa,EAAEC,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,cAAc;AAEzM,SAASC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,QAAQ,YAAY;AAClF,SAASC,aAAa,EAAEC,aAAa,QAAQ,eAAe;AAE5D,SAASC,YAAY,EAAEC,oBAAoB,QAAQ,eAAe;AAElE,SAASC,eAAe,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/index.ts"],"sourcesContent":["export { anyPresetEmbeds, contentTokenize, embedConfig, embedEnabled, enabledFeatures, featureEnabled, featureSignature, featureStates, presetHasSignal, presetNames, presetSignals } from './access.ts';\nexport type { InitOverrides } from './load.ts';\nexport { findConfigPath, initConfig, loadConfig, migrateConfig } from './load.ts';\nexport { resolvePreset, resolveSearch } from './resolve.ts';\nexport type { SignalName, SignalWeights } from './signals.ts';\nexport { SIGNAL_NAMES, SIGNAL_PREREQUISITES } from './signals.ts';\nexport type { Config, EffectiveSearch, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './types.ts';\nexport { CONFIG_FILENAME, DEFAULT_EMBED_MODEL, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './types.ts';\nexport { KNOWN_EMBED_KEYS } from './validate.ts';\n"],"names":["anyPresetEmbeds","contentTokenize","embedConfig","embedEnabled","enabledFeatures","featureEnabled","featureSignature","featureStates","presetHasSignal","presetNames","presetSignals","findConfigPath","initConfig","loadConfig","migrateConfig","resolvePreset","resolveSearch","SIGNAL_NAMES","SIGNAL_PREREQUISITES","CONFIG_FILENAME","DEFAULT_EMBED_MODEL","STATE_DIR","SUPPORTED_CONFIG_VERSION","KNOWN_EMBED_KEYS"],"mappings":"AAAA,SAASA,eAAe,EAAEC,eAAe,EAAEC,WAAW,EAAEC,YAAY,EAAEC,eAAe,EAAEC,cAAc,EAAEC,gBAAgB,EAAEC,aAAa,EAAEC,eAAe,EAAEC,WAAW,EAAEC,aAAa,QAAQ,cAAc;AAEzM,SAASC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,QAAQ,YAAY;AAClF,SAASC,aAAa,EAAEC,aAAa,QAAQ,eAAe;AAE5D,SAASC,YAAY,EAAEC,oBAAoB,QAAQ,eAAe;AAElE,SAASC,eAAe,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,aAAa;AACvG,SAASC,gBAAgB,QAAQ,gBAAgB"}

package/dist/esm/config/types.d.ts

@@ -10,6 +10,7 @@ export interface EmbedConfig {
     url?: string;
     key?: string;
     languages?: string[];
+    chunkTokens?: number;
 }
 export declare const DEFAULT_EMBED_MODEL = "minishlab/potion-retrieval-32M";
 export interface Preset {

package/dist/esm/config/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/types.ts"],"sourcesContent":["import type { SignalWeights } from './signals.ts';\n\nexport const CONFIG_FILENAME = 'sense.config.json';\nexport const STATE_DIR = '.sense';\n\n// Highest sense.config.json `version` this build understands. Older versions auto-migrate on load.\nexport const SUPPORTED_CONFIG_VERSION = 5;\n\n// Each feature owns its tables, parse-time extraction, and reconcile step; commands degrade when one is off.\n// links/sections/tags/rank are opt-out toggles in the top-level `features` block; embed is not a\n// member of that block -- it is on exactly when an `embed` block names a model.\nexport const FEATURE_NAMES = ['links', 'sections', 'tags', 'rank', 'embed'] as const;\nexport type FeatureName = (typeof FEATURE_NAMES)[number];\n\n// `embed` names the model and gives the tree vectors at all; a preset's `signals` says which\n// engines that scope's searches use.\nexport interface EmbedConfig {\n  model?: string;\n  provider?: 'static' | 'openai' | 'cohere';\n  url?: string; // openai provider: OpenAI-compatible base URL, e.g. http://localhost:11434/v1\n  key?: string; // openai/cohere provider: name of the env var holding the bearer token, if any\n  languages?: string[]; // owner-declared model languages; enables the language-fit check where no card exists\n}\n\nexport const DEFAULT_EMBED_MODEL = 'minishlab/potion-retrieval-32M';\n\n// A named, self-contained file-selection scope. include/exclude are globby patterns resolved\n// relative to the config file. No inheritance between presets -- every field a preset wants\n// it declares itself. Presets may overlap freely; they are views, not partitions.\nexport interface Preset {\n  include: string[];\n  exclude?: string[];\n  k?: number; // result count for `search` scoped to this preset; default 10\n  // Exhaustive when present: every signal this preset's searches compose, each mapped to\n  // its RRF weight (a finite number > 0; presence is enablement). Absent means every signal\n  // whose prerequisite holds, each at weight 1 -- words always, links when the feature is on,\n  // vectors when the tree names a model. A layer searched for exact wording (ingested sources,\n  // archives, generated output) declares {\"words\":1,\"links\":1} and costs no embedding at all.\n  signals?: SignalWeights;\n  where?: string; // standing SQL condition against frontmatter alias `f`\n}\n\n// A saved `search` invocation: `sense <name>` runs like\n// `sense search <search> [--preset] [--include] [--exclude] [--where] [--k]`.\nexport interface SavedSearch {\n  search: string;\n  preset?: string;\n  include?: string[];\n  exclude?: string[];\n  where?: string;\n  k?: number;\n}\n\n// An entry names the verb it runs, one to one with the commands: { sql } runs like\n// `sense sql`, { search } like `sense search`.\nexport type SavedQuery = { sql: string } | SavedSearch;\n\nexport interface Config {\n  // Editor-only pointer to schema.json; never read by sense.\n  $schema?: string;\n  version?: number;\n  // File selection, index-time, and per-preset search defaults. A file is indexed iff any\n  // preset's include/exclude covers it (union). `default` is used when a command names no preset.\n  presets: Record<string, Preset>;\n  // Global feature defaults; absent block or key means enabled. embed is not a member here --\n  // see the `embed` block.\n  features?: { links?: boolean; sections?: boolean; tags?: boolean; rank?: boolean };\n  // Names the embedding model. Present means every indexed file gets vectors; absent means none do.\n  embed?: EmbedConfig;\n  // Settings for the `content` FTS5 table. `tokenize` decides which languages the tree can be\n  // word-searched in at all; changing it rebuilds the index.\n  content?: { tokenize?: string };\n  queries: Record<string, SavedQuery>;\n}\n\nexport interface ResolvedConfig extends Config {\n  baseDir: string;\n  configPath: string | null;\n  // Keys the file declares that this build does not read; cli reports them.\n  unknownKeys?: string[];\n  // Set when loadConfig auto-migrated the file on disk; cli reports it.\n  migratedFrom?: number;\n}\n\nexport interface SearchOverrides {\n  preset?: string;\n  k?: number;\n  where?: string;\n  include?: string[];\n  exclude?: string[];\n  noExclude?: boolean; // drop the preset's exclude for this command; --exclude still applies\n}\n\nexport interface EffectiveSearch {\n  presetName: string;\n  k: number;\n  where?: string;\n  include: string[];\n  exclude?: string[];\n  signals: SignalWeights; // this preset's effective signal weights\n}\n"],"names":["CONFIG_FILENAME","STATE_DIR","SUPPORTED_CONFIG_VERSION","FEATURE_NAMES","DEFAULT_EMBED_MODEL"],"mappings":"AAEA,OAAO,MAAMA,kBAAkB,oBAAoB;AACnD,OAAO,MAAMC,YAAY,SAAS;AAElC,mGAAmG;AACnG,OAAO,MAAMC,2BAA2B,EAAE;AAE1C,6GAA6G;AAC7G,iGAAiG;AACjG,gFAAgF;AAChF,OAAO,MAAMC,gBAAgB;IAAC;IAAS;IAAY;IAAQ;IAAQ;CAAQ,CAAU;AAarF,OAAO,MAAMC,sBAAsB,iCAAiC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/types.ts"],"sourcesContent":["import type { SignalWeights } from './signals.ts';\n\nexport const CONFIG_FILENAME = 'sense.config.json';\nexport const STATE_DIR = '.sense';\n\n// Highest sense.config.json `version` this build understands. Older versions auto-migrate on load.\nexport const SUPPORTED_CONFIG_VERSION = 5;\n\n// Each feature owns its tables, parse-time extraction, and reconcile step; commands degrade when one is off.\n// links/sections/tags/rank are opt-out toggles in the top-level `features` block; embed is not a\n// member of that block -- it is on exactly when an `embed` block names a model.\nexport const FEATURE_NAMES = ['links', 'sections', 'tags', 'rank', 'embed'] as const;\nexport type FeatureName = (typeof FEATURE_NAMES)[number];\n\n// `embed` names the model and gives the tree vectors at all; a preset's `signals` says which\n// engines that scope's searches use.\nexport interface EmbedConfig {\n  model?: string;\n  provider?: 'static' | 'openai' | 'cohere';\n  url?: string; // openai provider: OpenAI-compatible base URL, e.g. http://localhost:11434/v1\n  key?: string; // openai/cohere provider: name of the env var holding the bearer token, if any\n  languages?: string[]; // owner-declared model languages; enables the language-fit check where no card exists\n  chunkTokens?: number; // chunk size ceiling in estimated tokens for small-context models; default 500\n}\n\nexport const DEFAULT_EMBED_MODEL = 'minishlab/potion-retrieval-32M';\n\n// A named, self-contained file-selection scope. include/exclude are globby patterns resolved\n// relative to the config file. No inheritance between presets -- every field a preset wants\n// it declares itself. Presets may overlap freely; they are views, not partitions.\nexport interface Preset {\n  include: string[];\n  exclude?: string[];\n  k?: number; // result count for `search` scoped to this preset; default 10\n  // Exhaustive when present: every signal this preset's searches compose, each mapped to\n  // its RRF weight (a finite number > 0; presence is enablement). Absent means every signal\n  // whose prerequisite holds, each at weight 1 -- words always, links when the feature is on,\n  // vectors when the tree names a model. A layer searched for exact wording (ingested sources,\n  // archives, generated output) declares {\"words\":1,\"links\":1} and costs no embedding at all.\n  signals?: SignalWeights;\n  where?: string; // standing SQL condition against frontmatter alias `f`\n}\n\n// A saved `search` invocation: `sense <name>` runs like\n// `sense search <search> [--preset] [--include] [--exclude] [--where] [--k]`.\nexport interface SavedSearch {\n  search: string;\n  preset?: string;\n  include?: string[];\n  exclude?: string[];\n  where?: string;\n  k?: number;\n}\n\n// An entry names the verb it runs, one to one with the commands: { sql } runs like\n// `sense sql`, { search } like `sense search`.\nexport type SavedQuery = { sql: string } | SavedSearch;\n\nexport interface Config {\n  // Editor-only pointer to schema.json; never read by sense.\n  $schema?: string;\n  version?: number;\n  // File selection, index-time, and per-preset search defaults. A file is indexed iff any\n  // preset's include/exclude covers it (union). `default` is used when a command names no preset.\n  presets: Record<string, Preset>;\n  // Global feature defaults; absent block or key means enabled. embed is not a member here --\n  // see the `embed` block.\n  features?: { links?: boolean; sections?: boolean; tags?: boolean; rank?: boolean };\n  // Names the embedding model. Present means every indexed file gets vectors; absent means none do.\n  embed?: EmbedConfig;\n  // Settings for the `content` FTS5 table. `tokenize` decides which languages the tree can be\n  // word-searched in at all; changing it rebuilds the index.\n  content?: { tokenize?: string };\n  queries: Record<string, SavedQuery>;\n}\n\nexport interface ResolvedConfig extends Config {\n  baseDir: string;\n  configPath: string | null;\n  // Keys the file declares that this build does not read; cli reports them.\n  unknownKeys?: string[];\n  // Set when loadConfig auto-migrated the file on disk; cli reports it.\n  migratedFrom?: number;\n}\n\nexport interface SearchOverrides {\n  preset?: string;\n  k?: number;\n  where?: string;\n  include?: string[];\n  exclude?: string[];\n  noExclude?: boolean; // drop the preset's exclude for this command; --exclude still applies\n}\n\nexport interface EffectiveSearch {\n  presetName: string;\n  k: number;\n  where?: string;\n  include: string[];\n  exclude?: string[];\n  signals: SignalWeights; // this preset's effective signal weights\n}\n"],"names":["CONFIG_FILENAME","STATE_DIR","SUPPORTED_CONFIG_VERSION","FEATURE_NAMES","DEFAULT_EMBED_MODEL"],"mappings":"AAEA,OAAO,MAAMA,kBAAkB,oBAAoB;AACnD,OAAO,MAAMC,YAAY,SAAS;AAElC,mGAAmG;AACnG,OAAO,MAAMC,2BAA2B,EAAE;AAE1C,6GAA6G;AAC7G,iGAAiG;AACjG,gFAAgF;AAChF,OAAO,MAAMC,gBAAgB;IAAC;IAAS;IAAY;IAAQ;IAAQ;CAAQ,CAAU;AAcrF,OAAO,MAAMC,sBAAsB,iCAAiC"}

package/dist/esm/config/validate.d.ts

@@ -1,4 +1,5 @@
 import type { Config } from './types.js';
+export declare const KNOWN_EMBED_KEYS: Set<string>;
 export declare function unknownConfigKeys(cfg: Record<string, unknown>): string[];
 export declare function validateLegacyScan(parsed: unknown, configPath: string): void;
 export declare function validateConfig(parsed: unknown, configPath: string): Config;

package/dist/esm/config/validate.js

@@ -24,12 +24,15 @@ const KNOWN_FEATURE_KEYS = new Set([
     'tags',
     'rank'
 ]);
-const KNOWN_EMBED_KEYS = new Set([
+// Exported so a docs test can assert every key here has a schema.json property -- one source
+// of truth for what the embed block accepts, checked against the other for drift.
+export const KNOWN_EMBED_KEYS = new Set([
     'model',
     'provider',
     'url',
     'key',
-    'languages'
+    'languages',
+    'chunkTokens'
 ]);
 const KNOWN_CONTENT_KEYS = new Set([
     'tokenize'
@@ -121,6 +124,9 @@ function validateEmbedBlock(value, configPath) {
     if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l)=>typeof l !== 'string' || l === ''))) {
         throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. ["en", "zh"])`);
     }
+    if (embed.chunkTokens !== undefined && (typeof embed.chunkTokens !== 'number' || !Number.isInteger(embed.chunkTokens) || embed.chunkTokens <= 0)) {
+        throw new SenseError('CONFIG_INVALID', `${configPath}: embed.chunkTokens must be a positive integer`);
+    }
 }
 function isSignalName(value) {
     return typeof value === 'string' && SIGNAL_NAMES.includes(value);

package/dist/esm/config/validate.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { SIGNAL_NAMES, SIGNAL_PREREQUISITES, type SignalName } from './signals.ts';\nimport type { Config } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'content', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'signals', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\nconst KNOWN_EMBED_KEYS = new Set(['model', 'provider', 'url', 'key', 'languages']);\nconst KNOWN_CONTENT_KEYS = new Set(['tokenize']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\n// Only the shape is checked here. Whether the linked SQLite accepts the tokenizer is settled\n// by probing it in db.ts, so this never has to carry a table of which version added what.\nfunction validateContentBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content must be an object of { tokenize?: string }`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_CONTENT_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content has unknown key(s) ${unknown.join(', ')}; content takes tokenize`);\n  }\n  if (block.tokenize !== undefined && (typeof block.tokenize !== 'string' || block.tokenize.trim() === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize must be a non-empty string, e.g. \"trigram\" or \"unicode61 tokenchars '-_'\"`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, provider?: \"static\"|\"openai\"|\"cohere\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.provider !== undefined && embed.provider !== 'static' && embed.provider !== 'openai' && embed.provider !== 'cohere') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider must be \"static\", \"openai\", or \"cohere\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n  // openai has no default host, so a missing url would otherwise only surface at the first\n  // vector search; a config error belongs at config time.\n  if (embed.provider === 'openai' && (embed.url === undefined || embed.url === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider \"openai\" requires embed.url (e.g. \"http://localhost:11434/v1\" for Ollama)`);\n  }\n  if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l) => typeof l !== 'string' || l === ''))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. [\"en\", \"zh\"])`);\n  }\n}\n\nfunction isSignalName(value: unknown): value is SignalName {\n  return typeof value === 'string' && (SIGNAL_NAMES as readonly string[]).includes(value);\n}\n\nfunction isSignalWeight(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0;\n}\n\n// Shape (an object of signal name -> weight) plus the one cross-signal prerequisite that\n// doesn't need the top-level embed block: links is seeded by word-match rows, so declaring it\n// without words is always empty.\nfunction validatePresetSignals(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length === 0) {\n    throw new SenseError(\n      'CONFIG_INVALID',\n      `${configPath}: presets.${name}.signals must be a non-empty object of signal name -> weight (a finite number > 0), naming ${SIGNAL_NAMES.join(', ')} -- a preset that searches nothing is a mistake; name at least \"words\", or omit \"signals\" to use every signal whose prerequisites hold at weight 1`\n    );\n  }\n  const entries = Object.entries(value as Record<string, unknown>);\n  const unknown = entries.filter(([k]) => !isSignalName(k)).map(([k]) => k);\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals names unknown signal(s) ${unknown.join(', ')}; valid signals are ${SIGNAL_NAMES.join(', ')}`);\n  }\n  for (const [signal, weight] of entries) {\n    if (!isSignalWeight(weight)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals.${signal} must be a finite number > 0`);\n    }\n  }\n  const names = new Set(entries.map(([k]) => k));\n  for (const [signal, requires] of Object.entries(SIGNAL_PREREQUISITES) as Array<[SignalName, SignalName]>) {\n    if (names.has(signal) && !names.has(requires)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals has \"${signal}\" without \"${requires}\"; add \"${requires}\" to presets.${name}.signals, or drop \"${signal}\"`);\n    }\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, signals, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.signals !== undefined) validatePresetSignals(name, preset.signals, configPath);\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string used to mean SQL. It now fails rather than being inferred: an entry says\n  // which of the two verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, signals?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.content !== undefined) {\n    validateContentBlock(cfg.content, configPath);\n  }\n\n  // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).\n  if (cfg.embed === undefined) {\n    for (const [name, value] of Object.entries(presets)) {\n      const signals = (value as Record<string, unknown>).signals as Record<string, unknown> | undefined;\n      if (signals && 'vectors' in signals) {\n        throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals includes \"vectors\", but no \"embed\" block names a model; add one, or drop \"vectors\" from presets.${name}.signals`);\n      }\n    }\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","SIGNAL_NAMES","SIGNAL_PREREQUISITES","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","KNOWN_CONTENT_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateContentBlock","block","unknown","join","tokenize","undefined","trim","validateFeaturesBlock","name","v","entries","validateEmbedBlock","embed","model","provider","url","key","languages","some","l","isSignalName","includes","isSignalWeight","Number","isFinite","validatePresetSignals","map","signal","weight","names","requires","validatePreset","preset","exclude","signals","isInteger","where","validateSavedQuery","JSON","stringify","slice","entry","sql","search","validateConfig","checks","presets","default","queries","features","content"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,oBAAoB,QAAyB,eAAe;AAGnF,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAW;CAAU;AACvG,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAW;CAAQ;AACjF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAY;IAAO;IAAO;CAAY;AACjF,MAAMI,qBAAqB,IAAIJ,IAAI;IAAC;CAAW;AAC/C,MAAMK,oBAAoB,IAAIL,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASM,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACZ,WAAWa,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASG,qBAAqBX,KAAc,EAAEQ,UAAkB;IAC9D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,oDAAoD,CAAC;IAC5G;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACP,mBAAmBQ,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,6BAA6B,EAAEK,QAAQC,IAAI,CAAC,MAAM,wBAAwB,CAAC;IAClI;IACA,IAAIF,MAAMG,QAAQ,KAAKC,aAAc,CAAA,OAAOJ,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,CAACE,IAAI,OAAO,EAAC,GAAI;QACxG,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,4FAA4F,CAAC;IACpJ;AACF;AAEA,SAASU,sBAAsBlB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACT,mBAAmBU,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,EAAEK,QAAQC,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACK,MAAMC,EAAE,IAAI1B,OAAO2B,OAAO,CAACT,OAAQ;QAC7C,IAAI,OAAOQ,MAAM,WAAW;YAC1B,MAAM,IAAItC,WAAW,kBAAkB,GAAG0B,WAAW,WAAW,EAAEW,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASG,mBAAmBtB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,MAAMe,QAAQvB;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAAC4B,OAAO3B,MAAM,CAAC,CAACC,IAAM,CAACR,iBAAiBS,GAAG,CAACD;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,2BAA2B,EAAEK,QAAQC,IAAI,CAAC,OAAO;IACxG;IACA,IAAIS,MAAMC,KAAK,KAAKR,aAAa,OAAOO,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIe,MAAMC,KAAK,KAAKR,aAAaO,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIe,MAAME,QAAQ,KAAKT,aAAaO,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,UAAU;QAC7H,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIe,MAAMG,GAAG,KAAKV,aAAa,OAAOO,MAAMG,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIe,MAAMI,GAAG,KAAKX,aAAa,OAAOO,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,yFAAyF;IACzF,wDAAwD;IACxD,IAAIe,MAAME,QAAQ,KAAK,YAAaF,CAAAA,MAAMG,GAAG,KAAKV,aAAaO,MAAMG,GAAG,KAAK,EAAC,GAAI;QAChF,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,IAAIe,MAAMK,SAAS,KAAKZ,aAAc,CAAA,CAACf,MAAMC,OAAO,CAACqB,MAAMK,SAAS,KAAKL,MAAMK,SAAS,CAACzB,MAAM,KAAK,KAAKoB,MAAMK,SAAS,CAACC,IAAI,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,MAAM,GAAE,GAAI;QACxK,MAAM,IAAIhD,WAAW,kBAAkB,GAAG0B,WAAW,gFAAgF,CAAC;IACxI;AACF;AAEA,SAASuB,aAAa/B,KAAc;IAClC,OAAO,OAAOA,UAAU,YAAY,AAACjB,aAAmCiD,QAAQ,CAAChC;AACnF;AAEA,SAASiC,eAAejC,KAAc;IACpC,OAAO,OAAOA,UAAU,YAAYkC,OAAOC,QAAQ,CAACnC,UAAUA,QAAQ;AACxE;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,iCAAiC;AACjC,SAASoC,sBAAsBjB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC7E,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,UAAUN,OAAOC,IAAI,CAACK,OAAOG,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WACR,kBACA,GAAG0B,WAAW,UAAU,EAAEW,KAAK,2FAA2F,EAAEpC,aAAa+B,IAAI,CAAC,MAAM,kJAAkJ,CAAC;IAE3S;IACA,MAAMO,UAAU3B,OAAO2B,OAAO,CAACrB;IAC/B,MAAMa,UAAUQ,QAAQzB,MAAM,CAAC,CAAC,CAACC,EAAE,GAAK,CAACkC,aAAalC,IAAIwC,GAAG,CAAC,CAAC,CAACxC,EAAE,GAAKA;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,iCAAiC,EAAEN,QAAQC,IAAI,CAAC,MAAM,oBAAoB,EAAE/B,aAAa+B,IAAI,CAAC,OAAO;IAC7K;IACA,KAAK,MAAM,CAACwB,QAAQC,OAAO,IAAIlB,QAAS;QACtC,IAAI,CAACY,eAAeM,SAAS;YAC3B,MAAM,IAAIzD,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,SAAS,EAAEmB,OAAO,4BAA4B,CAAC;QACvH;IACF;IACA,MAAME,QAAQ,IAAItD,IAAImC,QAAQgB,GAAG,CAAC,CAAC,CAACxC,EAAE,GAAKA;IAC3C,KAAK,MAAM,CAACyC,QAAQG,SAAS,IAAI/C,OAAO2B,OAAO,CAACrC,sBAA0D;QACxG,IAAIwD,MAAM1C,GAAG,CAACwC,WAAW,CAACE,MAAM1C,GAAG,CAAC2C,WAAW;YAC7C,MAAM,IAAI3D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,cAAc,EAAEmB,OAAO,WAAW,EAAEG,SAAS,QAAQ,EAAEA,SAAS,aAAa,EAAEtB,KAAK,mBAAmB,EAAEmB,OAAO,CAAC,CAAC;QAC1L;IACF;AACF;AAEA,SAASI,eAAevB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAMwB,SAAS3C;IACf,MAAMa,UAAUnB,OAAOC,IAAI,CAACgD,QAAQ/C,MAAM,CAAC,CAACC,IAAM,CAACV,kBAAkBW,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,oDAAoD,CAAC;IACtK;IACA,IAAI,CAACf,sBAAsB4C,OAAOjC,OAAO,GAAG;QAC1C,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAIwB,OAAOC,OAAO,KAAK5B,aAAa,CAACjB,sBAAsB4C,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAIwB,OAAOE,OAAO,KAAK7B,WAAWoB,sBAAsBjB,MAAMwB,OAAOE,OAAO,EAAErC;IAC9E,IAAImC,OAAO9C,CAAC,KAAKmB,aAAc,CAAA,OAAO2B,OAAO9C,CAAC,KAAK,YAAY,CAACqC,OAAOY,SAAS,CAACH,OAAO9C,CAAC,KAAK8C,OAAO9C,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAIwB,OAAOI,KAAK,KAAK/B,aAAa,OAAO2B,OAAOI,KAAK,KAAK,UAAU;QAClE,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAAS6B,mBAAmB7B,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC1E,yFAAyF;IACzF,0EAA0E;IAC1E,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uCAAuC,EAAE8B,KAAKC,SAAS,CAAClD,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMmD,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGnD,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMiC,QAAQpD;IACd,IAAI,SAASoD,OAAO;QAClB,MAAMvC,UAAUnB,OAAOC,IAAI,CAACyD,OAAOxD,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOsC,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACpC,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYiC,OAAO;QACrB,MAAMvC,UAAUnB,OAAOC,IAAI,CAACyD,OAAOxD,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOsC,MAAME,MAAM,KAAK,YAAYF,MAAME,MAAM,CAACrC,IAAI,OAAO,IAAI;YAClE,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIiC,MAAMT,MAAM,KAAK3B,aAAc,CAAA,OAAOoC,MAAMT,MAAM,KAAK,YAAYS,MAAMT,MAAM,CAACxC,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIiC,MAAM1C,OAAO,KAAKM,aAAa,CAACjB,sBAAsBqD,MAAM1C,OAAO,GAAG;YACxE,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIiC,MAAMR,OAAO,KAAK5B,aAAa,CAACjB,sBAAsBqD,MAAMR,OAAO,GAAG;YACxE,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIiC,MAAML,KAAK,KAAK/B,aAAa,OAAOoC,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIiC,MAAMvD,CAAC,KAAKmB,aAAc,CAAA,OAAOoC,MAAMvD,CAAC,KAAK,YAAY,CAACqC,OAAOY,SAAS,CAACM,MAAMvD,CAAC,KAAKuD,MAAMvD,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAIrC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASoC,eAAehD,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIzB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAI+D,MAAM,KAAKxC,WAAW;QAC5B,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yHAAyH,CAAC;IACjL;IAEA,MAAMiD,UAAUhE,IAAIgE,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAYxD,MAAMC,OAAO,CAACuD,YAAY/D,OAAOC,IAAI,CAAC8D,SAAStD,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,kGAAkG,CAAC;IAC1J;IACA,KAAK,MAAM,CAACW,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACoC,SAAU;QACnDf,eAAevB,MAAMnB,OAAOQ;IAC9B;IACA,IAAIiD,QAAQC,OAAO,KAAK1C,WAAW;QACjC,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAIkE,OAAO,KAAK3C,WAAWvB,IAAIkE,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOlE,IAAIkE,OAAO,KAAK,YAAYlE,IAAIkE,OAAO,KAAK,QAAQ1D,MAAMC,OAAO,CAACT,IAAIkE,OAAO,GAAG;QACzF,MAAM,IAAI7E,WAAW,kBAAkB,GAAG0B,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMmD,UAAUlE,IAAIkE,OAAO;IAC3B,KAAK,MAAM,CAACxC,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACsC,SAAU;QACnDX,mBAAmB7B,MAAMnB,OAAOQ;IAClC;IAEA,IAAIf,IAAImE,QAAQ,KAAK5C,WAAW;QAC9BE,sBAAsBzB,IAAImE,QAAQ,EAAEpD;IACtC;IACA,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3BM,mBAAmB7B,IAAI8B,KAAK,EAAEf;IAChC;IACA,IAAIf,IAAIoE,OAAO,KAAK7C,WAAW;QAC7BL,qBAAqBlB,IAAIoE,OAAO,EAAErD;IACpC;IAEA,8FAA8F;IAC9F,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3B,KAAK,MAAM,CAACG,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACoC,SAAU;YACnD,MAAMZ,UAAU,AAAC7C,MAAkC6C,OAAO;YAC1D,IAAIA,WAAW,aAAaA,SAAS;gBACnC,MAAM,IAAI/D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,yGAAyG,EAAEA,KAAK,QAAQ,CAAC;YACjM;QACF;IACF;IAEA,OAAO1B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { SIGNAL_NAMES, SIGNAL_PREREQUISITES, type SignalName } from './signals.ts';\nimport type { Config } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'content', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'signals', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\n// Exported so a docs test can assert every key here has a schema.json property -- one source\n// of truth for what the embed block accepts, checked against the other for drift.\nexport const KNOWN_EMBED_KEYS = new Set(['model', 'provider', 'url', 'key', 'languages', 'chunkTokens']);\nconst KNOWN_CONTENT_KEYS = new Set(['tokenize']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\n// Only the shape is checked here. Whether the linked SQLite accepts the tokenizer is settled\n// by probing it in db.ts, so this never has to carry a table of which version added what.\nfunction validateContentBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content must be an object of { tokenize?: string }`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_CONTENT_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content has unknown key(s) ${unknown.join(', ')}; content takes tokenize`);\n  }\n  if (block.tokenize !== undefined && (typeof block.tokenize !== 'string' || block.tokenize.trim() === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize must be a non-empty string, e.g. \"trigram\" or \"unicode61 tokenchars '-_'\"`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, provider?: \"static\"|\"openai\"|\"cohere\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.provider !== undefined && embed.provider !== 'static' && embed.provider !== 'openai' && embed.provider !== 'cohere') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider must be \"static\", \"openai\", or \"cohere\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n  // openai has no default host, so a missing url would otherwise only surface at the first\n  // vector search; a config error belongs at config time.\n  if (embed.provider === 'openai' && (embed.url === undefined || embed.url === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider \"openai\" requires embed.url (e.g. \"http://localhost:11434/v1\" for Ollama)`);\n  }\n  if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l) => typeof l !== 'string' || l === ''))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. [\"en\", \"zh\"])`);\n  }\n  if (embed.chunkTokens !== undefined && (typeof embed.chunkTokens !== 'number' || !Number.isInteger(embed.chunkTokens) || embed.chunkTokens <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.chunkTokens must be a positive integer`);\n  }\n}\n\nfunction isSignalName(value: unknown): value is SignalName {\n  return typeof value === 'string' && (SIGNAL_NAMES as readonly string[]).includes(value);\n}\n\nfunction isSignalWeight(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0;\n}\n\n// Shape (an object of signal name -> weight) plus the one cross-signal prerequisite that\n// doesn't need the top-level embed block: links is seeded by word-match rows, so declaring it\n// without words is always empty.\nfunction validatePresetSignals(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length === 0) {\n    throw new SenseError(\n      'CONFIG_INVALID',\n      `${configPath}: presets.${name}.signals must be a non-empty object of signal name -> weight (a finite number > 0), naming ${SIGNAL_NAMES.join(', ')} -- a preset that searches nothing is a mistake; name at least \"words\", or omit \"signals\" to use every signal whose prerequisites hold at weight 1`\n    );\n  }\n  const entries = Object.entries(value as Record<string, unknown>);\n  const unknown = entries.filter(([k]) => !isSignalName(k)).map(([k]) => k);\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals names unknown signal(s) ${unknown.join(', ')}; valid signals are ${SIGNAL_NAMES.join(', ')}`);\n  }\n  for (const [signal, weight] of entries) {\n    if (!isSignalWeight(weight)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals.${signal} must be a finite number > 0`);\n    }\n  }\n  const names = new Set(entries.map(([k]) => k));\n  for (const [signal, requires] of Object.entries(SIGNAL_PREREQUISITES) as Array<[SignalName, SignalName]>) {\n    if (names.has(signal) && !names.has(requires)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals has \"${signal}\" without \"${requires}\"; add \"${requires}\" to presets.${name}.signals, or drop \"${signal}\"`);\n    }\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, signals, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.signals !== undefined) validatePresetSignals(name, preset.signals, configPath);\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string used to mean SQL. It now fails rather than being inferred: an entry says\n  // which of the two verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, signals?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.content !== undefined) {\n    validateContentBlock(cfg.content, configPath);\n  }\n\n  // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).\n  if (cfg.embed === undefined) {\n    for (const [name, value] of Object.entries(presets)) {\n      const signals = (value as Record<string, unknown>).signals as Record<string, unknown> | undefined;\n      if (signals && 'vectors' in signals) {\n        throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals includes \"vectors\", but no \"embed\" block names a model; add one, or drop \"vectors\" from presets.${name}.signals`);\n      }\n    }\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","SIGNAL_NAMES","SIGNAL_PREREQUISITES","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","KNOWN_CONTENT_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateContentBlock","block","unknown","join","tokenize","undefined","trim","validateFeaturesBlock","name","v","entries","validateEmbedBlock","embed","model","provider","url","key","languages","some","l","chunkTokens","Number","isInteger","isSignalName","includes","isSignalWeight","isFinite","validatePresetSignals","map","signal","weight","names","requires","validatePreset","preset","exclude","signals","where","validateSavedQuery","JSON","stringify","slice","entry","sql","search","validateConfig","checks","presets","default","queries","features","content"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,oBAAoB,QAAyB,eAAe;AAGnF,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAW;CAAU;AACvG,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAW;CAAQ;AACjF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,6FAA6F;AAC7F,kFAAkF;AAClF,OAAO,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAY;IAAO;IAAO;IAAa;CAAc,EAAE;AACzG,MAAMI,qBAAqB,IAAIJ,IAAI;IAAC;CAAW;AAC/C,MAAMK,oBAAoB,IAAIL,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASM,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACZ,WAAWa,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASG,qBAAqBX,KAAc,EAAEQ,UAAkB;IAC9D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,oDAAoD,CAAC;IAC5G;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACP,mBAAmBQ,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,6BAA6B,EAAEK,QAAQC,IAAI,CAAC,MAAM,wBAAwB,CAAC;IAClI;IACA,IAAIF,MAAMG,QAAQ,KAAKC,aAAc,CAAA,OAAOJ,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,CAACE,IAAI,OAAO,EAAC,GAAI;QACxG,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,4FAA4F,CAAC;IACpJ;AACF;AAEA,SAASU,sBAAsBlB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACT,mBAAmBU,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,EAAEK,QAAQC,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACK,MAAMC,EAAE,IAAI1B,OAAO2B,OAAO,CAACT,OAAQ;QAC7C,IAAI,OAAOQ,MAAM,WAAW;YAC1B,MAAM,IAAItC,WAAW,kBAAkB,GAAG0B,WAAW,WAAW,EAAEW,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASG,mBAAmBtB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,MAAMe,QAAQvB;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAAC4B,OAAO3B,MAAM,CAAC,CAACC,IAAM,CAACR,iBAAiBS,GAAG,CAACD;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,2BAA2B,EAAEK,QAAQC,IAAI,CAAC,OAAO;IACxG;IACA,IAAIS,MAAMC,KAAK,KAAKR,aAAa,OAAOO,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIe,MAAMC,KAAK,KAAKR,aAAaO,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIe,MAAME,QAAQ,KAAKT,aAAaO,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,UAAU;QAC7H,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIe,MAAMG,GAAG,KAAKV,aAAa,OAAOO,MAAMG,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIe,MAAMI,GAAG,KAAKX,aAAa,OAAOO,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,yFAAyF;IACzF,wDAAwD;IACxD,IAAIe,MAAME,QAAQ,KAAK,YAAaF,CAAAA,MAAMG,GAAG,KAAKV,aAAaO,MAAMG,GAAG,KAAK,EAAC,GAAI;QAChF,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,IAAIe,MAAMK,SAAS,KAAKZ,aAAc,CAAA,CAACf,MAAMC,OAAO,CAACqB,MAAMK,SAAS,KAAKL,MAAMK,SAAS,CAACzB,MAAM,KAAK,KAAKoB,MAAMK,SAAS,CAACC,IAAI,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,MAAM,GAAE,GAAI;QACxK,MAAM,IAAIhD,WAAW,kBAAkB,GAAG0B,WAAW,gFAAgF,CAAC;IACxI;IACA,IAAIe,MAAMQ,WAAW,KAAKf,aAAc,CAAA,OAAOO,MAAMQ,WAAW,KAAK,YAAY,CAACC,OAAOC,SAAS,CAACV,MAAMQ,WAAW,KAAKR,MAAMQ,WAAW,IAAI,CAAA,GAAI;QAChJ,MAAM,IAAIjD,WAAW,kBAAkB,GAAG0B,WAAW,8CAA8C,CAAC;IACtG;AACF;AAEA,SAAS0B,aAAalC,KAAc;IAClC,OAAO,OAAOA,UAAU,YAAY,AAACjB,aAAmCoD,QAAQ,CAACnC;AACnF;AAEA,SAASoC,eAAepC,KAAc;IACpC,OAAO,OAAOA,UAAU,YAAYgC,OAAOK,QAAQ,CAACrC,UAAUA,QAAQ;AACxE;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,iCAAiC;AACjC,SAASsC,sBAAsBnB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC7E,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,UAAUN,OAAOC,IAAI,CAACK,OAAOG,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WACR,kBACA,GAAG0B,WAAW,UAAU,EAAEW,KAAK,2FAA2F,EAAEpC,aAAa+B,IAAI,CAAC,MAAM,kJAAkJ,CAAC;IAE3S;IACA,MAAMO,UAAU3B,OAAO2B,OAAO,CAACrB;IAC/B,MAAMa,UAAUQ,QAAQzB,MAAM,CAAC,CAAC,CAACC,EAAE,GAAK,CAACqC,aAAarC,IAAI0C,GAAG,CAAC,CAAC,CAAC1C,EAAE,GAAKA;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,iCAAiC,EAAEN,QAAQC,IAAI,CAAC,MAAM,oBAAoB,EAAE/B,aAAa+B,IAAI,CAAC,OAAO;IAC7K;IACA,KAAK,MAAM,CAAC0B,QAAQC,OAAO,IAAIpB,QAAS;QACtC,IAAI,CAACe,eAAeK,SAAS;YAC3B,MAAM,IAAI3D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,SAAS,EAAEqB,OAAO,4BAA4B,CAAC;QACvH;IACF;IACA,MAAME,QAAQ,IAAIxD,IAAImC,QAAQkB,GAAG,CAAC,CAAC,CAAC1C,EAAE,GAAKA;IAC3C,KAAK,MAAM,CAAC2C,QAAQG,SAAS,IAAIjD,OAAO2B,OAAO,CAACrC,sBAA0D;QACxG,IAAI0D,MAAM5C,GAAG,CAAC0C,WAAW,CAACE,MAAM5C,GAAG,CAAC6C,WAAW;YAC7C,MAAM,IAAI7D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,cAAc,EAAEqB,OAAO,WAAW,EAAEG,SAAS,QAAQ,EAAEA,SAAS,aAAa,EAAExB,KAAK,mBAAmB,EAAEqB,OAAO,CAAC,CAAC;QAC1L;IACF;AACF;AAEA,SAASI,eAAezB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAM0B,SAAS7C;IACf,MAAMa,UAAUnB,OAAOC,IAAI,CAACkD,QAAQjD,MAAM,CAAC,CAACC,IAAM,CAACV,kBAAkBW,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,oDAAoD,CAAC;IACtK;IACA,IAAI,CAACf,sBAAsB8C,OAAOnC,OAAO,GAAG;QAC1C,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI0B,OAAOC,OAAO,KAAK9B,aAAa,CAACjB,sBAAsB8C,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI0B,OAAOE,OAAO,KAAK/B,WAAWsB,sBAAsBnB,MAAM0B,OAAOE,OAAO,EAAEvC;IAC9E,IAAIqC,OAAOhD,CAAC,KAAKmB,aAAc,CAAA,OAAO6B,OAAOhD,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACY,OAAOhD,CAAC,KAAKgD,OAAOhD,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAI0B,OAAOG,KAAK,KAAKhC,aAAa,OAAO6B,OAAOG,KAAK,KAAK,UAAU;QAClE,MAAM,IAAIlE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAAS8B,mBAAmB9B,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC1E,yFAAyF;IACzF,0EAA0E;IAC1E,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uCAAuC,EAAE+B,KAAKC,SAAS,CAACnD,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMoD,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGpD,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMkC,QAAQrD;IACd,IAAI,SAASqD,OAAO;QAClB,MAAMxC,UAAUnB,OAAOC,IAAI,CAAC0D,OAAOzD,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOuC,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACrC,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYkC,OAAO;QACrB,MAAMxC,UAAUnB,OAAOC,IAAI,CAAC0D,OAAOzD,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOuC,MAAME,MAAM,KAAK,YAAYF,MAAME,MAAM,CAACtC,IAAI,OAAO,IAAI;YAClE,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIkC,MAAMR,MAAM,KAAK7B,aAAc,CAAA,OAAOqC,MAAMR,MAAM,KAAK,YAAYQ,MAAMR,MAAM,CAAC1C,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIkC,MAAM3C,OAAO,KAAKM,aAAa,CAACjB,sBAAsBsD,MAAM3C,OAAO,GAAG;YACxE,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIkC,MAAMP,OAAO,KAAK9B,aAAa,CAACjB,sBAAsBsD,MAAMP,OAAO,GAAG;YACxE,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIkC,MAAML,KAAK,KAAKhC,aAAa,OAAOqC,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAIlE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIkC,MAAMxD,CAAC,KAAKmB,aAAc,CAAA,OAAOqC,MAAMxD,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACoB,MAAMxD,CAAC,KAAKwD,MAAMxD,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAIrC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASqC,eAAejD,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIzB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAIgE,MAAM,KAAKzC,WAAW;QAC5B,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yHAAyH,CAAC;IACjL;IAEA,MAAMkD,UAAUjE,IAAIiE,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAYzD,MAAMC,OAAO,CAACwD,YAAYhE,OAAOC,IAAI,CAAC+D,SAASvD,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,kGAAkG,CAAC;IAC1J;IACA,KAAK,MAAM,CAACW,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACqC,SAAU;QACnDd,eAAezB,MAAMnB,OAAOQ;IAC9B;IACA,IAAIkD,QAAQC,OAAO,KAAK3C,WAAW;QACjC,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAImE,OAAO,KAAK5C,WAAWvB,IAAImE,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOnE,IAAImE,OAAO,KAAK,YAAYnE,IAAImE,OAAO,KAAK,QAAQ3D,MAAMC,OAAO,CAACT,IAAImE,OAAO,GAAG;QACzF,MAAM,IAAI9E,WAAW,kBAAkB,GAAG0B,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMoD,UAAUnE,IAAImE,OAAO;IAC3B,KAAK,MAAM,CAACzC,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACuC,SAAU;QACnDX,mBAAmB9B,MAAMnB,OAAOQ;IAClC;IAEA,IAAIf,IAAIoE,QAAQ,KAAK7C,WAAW;QAC9BE,sBAAsBzB,IAAIoE,QAAQ,EAAErD;IACtC;IACA,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3BM,mBAAmB7B,IAAI8B,KAAK,EAAEf;IAChC;IACA,IAAIf,IAAIqE,OAAO,KAAK9C,WAAW;QAC7BL,qBAAqBlB,IAAIqE,OAAO,EAAEtD;IACpC;IAEA,8FAA8F;IAC9F,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3B,KAAK,MAAM,CAACG,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACqC,SAAU;YACnD,MAAMX,UAAU,AAAC/C,MAAkC+C,OAAO;YAC1D,IAAIA,WAAW,aAAaA,SAAS;gBACnC,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,yGAAyG,EAAEA,KAAK,QAAQ,CAAC;YACjM;QACF;IACF;IAEA,OAAO1B;AACT"}

package/dist/esm/db/open.d.ts

@@ -1,7 +1,7 @@
 import { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig } from '../config/index.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "17";
+export declare const SCHEMA_VERSION = "18";
 export interface OpenResult {
     db: DatabaseSync;
     cfg: ResolvedConfig;

package/dist/esm/db/open.js

@@ -5,14 +5,14 @@ import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { contentTokenize, featureSignature, STATE_DIR } from '../config/index.js';
 import { SenseError } from '../errors.js';
-import { activeFeatures } from '../features/index.js';
-import { registerFunctions } from '../sql-functions.js';
+import { activeFeatures, FEATURES } from '../features/index.js';
 import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.js';
 import { getMeta, setMeta } from './shared.js';
+import { registerFunctions } from './sql-functions.js';
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '17';
+export const SCHEMA_VERSION = '18';
 // Stemming is English-only, but the segmentation underneath it is what decides coverage:
 // unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)
 // indexes a whole run as one token and word search finds nothing. `content.tokenize` is how
@@ -61,7 +61,7 @@ function ensureSchema(db, cfg, tokenize) {
     db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
     for (const feature of activeFeatures(cfg))feature.schema(db);
     if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);
-    if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));
+    if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg, FEATURES));
 }
 export function docCount(db) {
     const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get();
@@ -88,7 +88,7 @@ export function open(cfg) {
     // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).
     const version = getMeta(db, 'schema_version');
     const features = getMeta(db, 'features');
-    const wantFeatures = featureSignature(cfg);
+    const wantFeatures = featureSignature(cfg, FEATURES);
     let tokenizeOnlyRebuild = false;
     if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
         // Indexing derives from presets, so a config edit rebuilding the cache must say so and