sensemaking 0.23.0 → 0.24.0

package/dist/esm/commands/search.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md). Past this bound, rows get the JS excerpt.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  for (const term of terms) {\n    let idx = 0;\n    for (;;) {\n      const found = haystackLower.indexOf(term, idx);\n      if (found === -1) break;\n      occ.push({ start: found, end: found + term.length, term });\n      idx = found + term.length;\n    }\n  }\n  // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n  // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n  let best = occ[0].start;\n  let bestDistinct = 0;\n  let bestCount = 0;\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + windowSize;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n      best = occ[i].start;\n      bestDistinct = seen.size;\n      bestCount = count;\n    }\n  }\n  return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n  const occ = findOccurrences(text.toLowerCase(), terms);\n  if (occ.length === 0) {\n    // Raw substring scan, not porter-stemmed: a doc matched only through a stemmed variant\n    // finds no occurrence here, so fall back to the doc's start, unmarked.\n    const end = Math.min(text.length, EXCERPT_WINDOW);\n    return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n  }\n  const start = bestWindowStart(occ, EXCERPT_WINDOW);\n  const end = Math.min(text.length, start + EXCERPT_WINDOW);\n  let out = '';\n  let cursor = start;\n  for (const o of occ) {\n    if (o.start < start || o.end > end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, end);\n  const prefix = start > 0 ? '…' : '';\n  const suffix = end < text.length ? '…' : '';\n  return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nasync function lineRangeFor(store: Store, path: string, line: number): Promise<string | null> {\n  const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');\n  const row = (await stmt.get(path, line, line)) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(store: Store, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = await rawScope(store, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = await narrowByWhere(store, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && (await scopeHasEmbeddings(store, cfg, allowedPaths));\n\n  // --where applies inside the candidate query (a post-filter would drop matches ranked past\n  // the pool) and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // Filtered before LIMIT, not after, or scoped notes ranking below the global top-`fetch`\n  // never reach the filter; joined against a temp table since real scopes exceed SQLITE_MAX_VARIABLE_NUMBER.\n  if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: LexicalHit[] = [];\n  // A query that is only whitespace has no searchable words: zero lexical rows, and the other\n  // signals compose normally.\n  if (signals.words !== undefined && terms.trim() !== '') {\n    try {\n      matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n  // from via='link' rows (never in matchRows, so absent from this set) below.\n  const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n  if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  await store.exec('DROP TABLE IF EXISTS _search');\n  // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the\n  // rounded score/similarity are printed at full precision in rows.\n  await store.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score DOUBLE, via TEXT, hit TEXT, lines TEXT, similarity DOUBLE)');\n  if (candidates.size > 0) {\n    await store.runBatch(\n      'INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)',\n      [...candidates].map(([path, c]) => [path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null])\n    );\n  }\n\n  // Reapplies --where even though _search is already scope+where filtered, since the join to\n  // frontmatter is already needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines: semantic rows carry their chunk's range, oversized-doc lexical rows gain one\n  // below, everything else stays null; similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  const selectStmt = await store.prepare(\n    `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n  );\n  const rows = (await selectStmt.all(k)) as Row[];\n\n  if (oversized.size > 0) {\n    const bareTerms = extractBareTerms(terms);\n    for (const row of rows) {\n      if (row.hit !== null || !oversized.has(row.path as string)) continue;\n      let text: string;\n      try {\n        text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n      } catch {\n        continue; // vanished since the match; leave hit/lines null rather than throw\n      }\n      const { excerpt, offset } = computeExcerpt(text, bareTerms);\n      row.hit = excerpt;\n      if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path as string, lineNumberAt(text, offset)) : null;\n    }\n  }\n\n  return rows;\n}\n"],"names":["readFileSync","join","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","store","path","stmt","prepare","row","get","start_line","end_line","search","cfg","opts","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hits","hit","oversized","links","chunkLines","chunkSimilarity","exec","runBatch","c","score","via","similarityCol","selectStmt","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AAChF,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,4BAA4B;AAExD,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,0GAA0G;AAC1G,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,uFAAuF;QACvF,uEAAuE;QACvE,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,eAAeE,aAAaC,KAAY,EAAEC,IAAY,EAAEJ,IAAY;IAClE,MAAMK,OAAO,MAAMF,MAAMG,OAAO,CAAC;IACjC,MAAMC,MAAO,MAAMF,KAAKG,GAAG,CAACJ,MAAMJ,MAAMA;IACxC,OAAOO,MAAM,CAAC,CAAC,EAAEA,IAAIE,UAAU,CAAC,CAAC,EAAEF,IAAIG,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOR,KAAY,EAAES,GAAmB,EAAElD,KAAa,EAAEmD,OAAsB,CAAC,CAAC;IACrG,MAAMC,YAAY9E,cAAc4E,KAAKC;IACrC,MAAM,EAAEE,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMd,MAAMG,OAAO,CAAC,iCAAgC,EAAGY,GAAG,EAAC,EAA+BhE,GAAG,CAAC,CAACiE,IAAMA,EAAEf,IAAI;IACrI,MAAMgB,aAAa,MAAM7E,SAAS4D,OAAOS,KAAKC,MAAMI;IACpD,MAAMI,cAAcD,WAAWlC,IAAI,GAAG+B,SAAS1D,MAAM;IACrD,wFAAwF;IACxF,wEAAwE;IACxE,IAAI+D;IACJ,IAAI;QACFA,eAAe,MAAMhF,cAAc6D,OAAOiB,YAAYN,UAAUS,KAAK;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAMpF,YAAYoF,KAAc9D,OAAOoD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQpC,KAAKqC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAIhG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkB6F,IAAI;YACxB,MAAM,IAAI3F,WAAW,uBAAuB,CAAC,QAAQ,EAAE2E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAE9F,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAM+F,kBAAkBN,gBAAiB,MAAMnF,mBAAmB2D,OAAOS,KAAKU;IAE9E,2FAA2F;IAC3F,kEAAkE;IAClE,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,2GAA2G;IAC3G,IAAIb,aAAa,MAAMhF,iBAAiB8D,OAAO,iBAAiBiB;IAChE,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAE3F,MAAMiB,aAAa,IAAIC;IACvB,IAAIC,YAA0B,EAAE;IAChC,4FAA4F;IAC5F,4BAA4B;IAC5B,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAanE,MAAML,IAAI,OAAO,IAAI;QACtD,IAAI;YACFmF,YAAY,MAAM7F,gBAAgBwD,OAAOmC,YAAY5E,OAAOyE,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QACnH,EAAE,OAAOjB,KAAK;YACZ,MAAMpF,YAAYoF,KAAc9D,OAAOwE;QACzC;IACF;IACA,MAAMQ,OAAO,IAAIH,IAAIC,UAAUtF,GAAG,CAAC,CAACiE,IAAM;YAACA,EAAEf,IAAI;YAAEe,EAAEwB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI9D,IAAI0D,UAAUlF,MAAM,CAAC,CAAC6D,IAAMA,EAAEwB,GAAG,KAAK,MAAMzF,GAAG,CAAC,CAACiE,IAAMA,EAAEf,IAAI;IAEnF,IAAIY,QAAQ6B,KAAK,KAAKhB,WAAW,MAAMpF,gBAAgB0D,OAAOmC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ6B,KAAK;IAEjI,IAAIC,aAAa,IAAIP;IACrB,IAAIQ,kBAAkB,IAAIR;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAEa,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrG,kBAAkByD,OAAOS,KAAK0B,YAAY5E,OAAO+D,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IAC1I;IAEA,MAAMzB,MAAM6C,IAAI,CAAC;IACjB,8FAA8F;IAC9F,kEAAkE;IAClE,MAAM7C,MAAM6C,IAAI,CAAC;IACjB,IAAIV,WAAWpD,IAAI,GAAG,GAAG;QACvB,MAAMiB,MAAM8C,QAAQ,CAClB,8FACA;eAAIX;SAAW,CAACpF,GAAG,CAAC,CAAC,CAACkD,MAAM8C,EAAE;gBAA4BR,WAAwBI,iBAA8BC;mBAA7E;gBAAC3C;gBAAM8C,EAAEC,KAAK;gBAAED,EAAEE,GAAG;iBAAEV,YAAAA,KAAKlC,GAAG,CAACJ,mBAATsC,uBAAAA,YAAkB;iBAAMI,kBAAAA,WAAWtC,GAAG,CAACJ,mBAAf0C,6BAAAA,kBAAwB;iBAAMC,uBAAAA,gBAAgBvC,GAAG,CAACJ,mBAApB2C,kCAAAA,uBAA6B;aAAK;;IAEtJ;IAEA,2FAA2F;IAC3F,qDAAqD;IACrD,MAAMxB,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,qEAAqE;IACrE,MAAMmB,gBAAgBpB,kBAAkB,yBAAyB;IACjE,MAAMqB,aAAa,MAAMnD,MAAMG,OAAO,CACpC,CAAC,kIAAkI,EAAE+C,cAAc;;OAEhJ,EAAE9B,MAAM,oCAAoC,CAAC;IAElD,MAAMgC,OAAQ,MAAMD,WAAWpC,GAAG,CAACH;IAEnC,IAAI6B,UAAU1D,IAAI,GAAG,GAAG;QACtB,MAAMsE,YAAY3G,iBAAiBa;QACnC,KAAK,MAAM6C,OAAOgD,KAAM;YACtB,IAAIhD,IAAIoC,GAAG,KAAK,QAAQ,CAACC,UAAUa,GAAG,CAAClD,IAAIH,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAOxD,aAAaC,KAAK+E,IAAI8C,OAAO,EAAEnD,IAAIH,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMoE;YACjDjD,IAAIoC,GAAG,GAAGpD;YACV,IAAIgB,IAAIoD,KAAK,IAAI,MAAMpD,IAAIoD,KAAK,GAAG5H,eAAe6E,KAAK,cAAc,MAAMV,aAAaC,OAAOI,IAAIH,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QACnJ;IACF;IAEA,OAAO8D;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { stemmer } from 'stemmer';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { bareTermSyntaxError, searchError } from '../output/search-error.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\nimport { foldForSearch, hasUnspacedRun, searchTokens, unspacedRuns } from '../text/segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// Default budget behind --snippet-char-limit/--snippet-count-limit: the limit is whatever the\n// caller passes, these only say what happens when they pass nothing (DESIGN.md \"Search snippets\").\nexport const SNIPPET_CHAR_LIMIT_DEFAULT = 80;\nexport const SNIPPET_COUNT_LIMIT_DEFAULT = 1;\n\n// Bare terms from an FTS5 query string: strips operators/quoting, not real terms, so the\n// snippet scan matches what the query matched -- FTS5 operators are uppercase-only, so this is too.\nfunction extractBareTerms(query: string): string[] {\n  const cleaned = query\n    .replace(/\"/g, ' ')\n    .replace(/[()*]/g, ' ')\n    .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/g, ' ');\n  return cleaned\n    .split(/\\s+/)\n    .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n    .flatMap((tok) => searchTokens(tok).map(({ text }) => text))\n    .map((tok) => tok.toLowerCase().trim())\n    .filter((tok) => tok.length > 0);\n}\n\n// House convention (src/chunk/group.ts): Intl.Segmenter word boundaries, not a bare \\b, which\n// is ASCII-only. Built on first use: construction is a few ms, paid only when a term matches.\nlet wordSegmenter: Intl.Segmenter | undefined;\n\nfunction findOccurrences(text: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n  const occ: Array<{ start: number; end: number; term: string }> = [];\n  const textLower = text.toLowerCase();\n\n  // Unspaced scripts (UNSPACED_SCRIPTS) mark no word boundaries: the sense-setup skill promises\n  // substring semantics there, so a matching term is found wherever it occurs in the run.\n  const unspacedTerms = terms.filter(hasUnspacedRun);\n  if (unspacedTerms.length > 0) {\n    for (const run of unspacedRuns(text)) {\n      const runLower = textLower.slice(run.start, run.end);\n      for (const term of unspacedTerms) {\n        let idx = 0;\n        for (;;) {\n          const found = runLower.indexOf(term, idx);\n          if (found === -1) break;\n          occ.push({ start: run.start + found, end: run.start + found + term.length, term });\n          idx = found + term.length;\n        }\n      }\n    }\n  }\n\n  // Spaced scripts: a word is marked when its porter stem equals a query term's stem, the same\n  // algorithm both indexes tokenize with (sqlite's TOKENIZE = 'porter unicode61'; duckdb's fts default).\n  const spacedTerms = terms.filter((t) => !hasUnspacedRun(t));\n  if (spacedTerms.length > 0) {\n    const termByStem = new Map(spacedTerms.map((t) => [stemmer(foldForSearch(t)), t]));\n    const stemInitials = new Set(Array.from(termByStem.keys(), (term) => term[0]));\n    for (const match of searchTokens(text)) {\n      const segment = match.text;\n      if (match.unspaced) continue;\n      const folded = foldForSearch(segment);\n      // Porter stemming changes suffixes only, so a different initial cannot match a query stem.\n      if (!stemInitials.has(folded[0])) continue;\n      const term = termByStem.get(stemmer(folded));\n      if (term !== undefined) occ.push({ start: match.start, end: match.end, term });\n    }\n  }\n\n  return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Segment boundaries of the whole text, word granularity: the only cut points a snippet edge\n// may land on. One rule for every script (DESIGN.md \"Cutting\") -- a regex \\b is ASCII-only and\n// would mis-cut Japanese, Chinese and Thai, which Intl.Segmenter divides by dictionary instead.\nfunction wordBoundaries(text: string): number[] {\n  wordSegmenter ??= new Intl.Segmenter('en', { granularity: 'word' });\n  const bounds = [0];\n  for (const { index, segment } of wordSegmenter.segment(text)) bounds.push(index + segment.length);\n  return bounds;\n}\n\n// bounds is sorted: snaps are binary searches, since each runs once per candidate window.\nfunction snapForward(bounds: number[], pos: number): number {\n  let lo = 0;\n  let hi = bounds.length - 1;\n  while (lo < hi) {\n    const mid = (lo + hi) >> 1;\n    if (bounds[mid] < pos) lo = mid + 1;\n    else hi = mid;\n  }\n  return bounds[lo];\n}\n\nfunction snapBackward(bounds: number[], pos: number): number {\n  let lo = 0;\n  let hi = bounds.length - 1;\n  let result = bounds[0];\n  while (lo < hi) {\n    const mid = (lo + hi + 1) >> 1;\n    if (bounds[mid] <= pos) {\n      lo = mid;\n      result = bounds[mid];\n    } else hi = mid - 1;\n  }\n  return result;\n}\n\nfunction firstBoundAfter(bounds: number[], pos: number): number | undefined {\n  let lo = 0;\n  let hi = bounds.length;\n  while (lo < hi) {\n    const mid = (lo + hi) >> 1;\n    if (bounds[mid] <= pos) lo = mid + 1;\n    else hi = mid;\n  }\n  return bounds[lo];\n}\n\n// A raw, unmarked upper bound: [start, end) inside [anchor, anchor+charLimit), edges on\n// boundaries. Markup (guillemets, ellipses) added on top of this can push the rendered result\n// past charLimit, which is what fitEnd() below corrects for; this is only the starting guess.\n// A single segment spanning the whole target window (a word longer than the char limit) is the\n// one case where holding the boundary rule means the budget is not a hard ceiling.\nfunction cutWindow(text: string, bounds: number[], anchor: number, charLimit: number): { start: number; end: number } {\n  const rawEnd = Math.min(text.length, anchor + charLimit);\n  const start = snapForward(bounds, anchor);\n  let end = snapBackward(bounds, rawEnd);\n  if (end <= start) end = firstBoundAfter(bounds, start) ?? text.length;\n  return { start, end: Math.min(end, text.length) };\n}\n\n// The largest boundary <= upperBound, > floor, whose rendered [*, end) fits charLimit. Markup\n// only ever adds characters as end grows, so rendered length is non-decreasing in end and\n// shrinking from upperBound is guaranteed to find the true fit rather than guess at it.\n// `floor` is the one end this never shrinks past (the anchor's own marked word, or the first\n// word in the unmarked fallback): DESIGN.md's budget is a ceiling except here, where returning\n// nothing is worse than a single word that overruns it.\nfunction fitEnd(render: (end: number) => string, bounds: number[], floor: number, upperBound: number, charLimit: number): number {\n  let lo = 0;\n  let hi = bounds.length - 1;\n  while (lo < hi) {\n    const mid = (lo + hi + 1) >> 1;\n    if (bounds[mid] <= upperBound) lo = mid;\n    else hi = mid - 1;\n  }\n  for (; lo >= 0 && bounds[lo] > floor; lo--) {\n    if (render(bounds[lo]).length <= charLimit) return bounds[lo];\n  }\n  return floor;\n}\n\ninterface Span {\n  start: number;\n  end: number;\n}\n\ninterface SnippetWindow extends Span {\n  distinct: number;\n  count: number;\n}\n\n// occ is start-sorted: a window's first candidate occurrence is a binary search, not a walk.\nfunction firstOccAtOrAfter(occ: Array<{ start: number; end: number; term: string }>, pos: number): number {\n  let lo = 0;\n  let hi = occ.length;\n  while (lo < hi) {\n    const mid = (lo + hi) >> 1;\n    if (occ[mid].start < pos) lo = mid + 1;\n    else hi = mid;\n  }\n  return lo;\n}\n\nfunction renderWindow(text: string, occ: Array<{ start: number; end: number; term: string }>, win: Span): string {\n  let out = '';\n  let cursor = win.start;\n  for (let i = firstOccAtOrAfter(occ, win.start); i < occ.length && occ[i].start < win.end; i++) {\n    const o = occ[i];\n    if (o.end > win.end) continue;\n    // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n    // emitting each would duplicate document text. Keep the first, absorb the rest.\n    if (o.start < cursor) continue;\n    out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n    cursor = o.end;\n  }\n  out += text.slice(cursor, win.end);\n  const prefix = win.start > 0 ? '…' : '';\n  const suffix = win.end < text.length ? '…' : '';\n  return `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`;\n}\n\n// Scores every occurrence-anchored candidate window: most distinct terms wins, ties broken by\n// most total hits -- coverage now that occurrences are real words, not the false-fragment counts\n// a substring scan produced. Selection itself is unevaluated (DESIGN.md \"Selection, unsettled\").\n// The window's `end` is fitted to the *rendered* text, markup included, not the raw span: a\n// window dense with marks (every guillemet pair adds two characters) shrinks to fit the same\n// budget a sparse one fills almost exactly.\nfunction scoreWindows(occ: Array<{ start: number; end: number; term: string }>, bounds: number[], text: string, charLimit: number): SnippetWindow[] {\n  const candidates: SnippetWindow[] = [];\n  for (let i = 0; i < occ.length; i++) {\n    const winEnd = occ[i].start + charLimit;\n    const seen = new Set<string>();\n    let count = 0;\n    for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n      seen.add(occ[j].term);\n      count++;\n    }\n    const start = snapForward(bounds, occ[i].start);\n    const floor = Math.max(start, occ[i].end);\n    const upperBound = Math.max(floor, cutWindow(text, bounds, occ[i].start, charLimit).end);\n    const end = fitEnd((e) => renderWindow(text, occ, { start, end: e }), bounds, floor, upperBound, charLimit);\n    candidates.push({ start, end, distinct: seen.size, count });\n  }\n  return candidates;\n}\n\nfunction overlaps(a: Span, b: Span): boolean {\n  return a.start < b.end && b.start < a.end;\n}\n\n// Same scoring function as a single snippet, kept rather than discarded (DESIGN.md \"Several\n// snippets per note\"): rank by score, take the top non-overlapping windows, then reorder to the\n// note's own order, since several passages from one note read as an abridged note.\nfunction selectWindows(candidates: SnippetWindow[], countLimit: number): { chosen: SnippetWindow[]; best: SnippetWindow } {\n  const ranked = candidates.slice().sort((a, b) => b.distinct - a.distinct || b.count - a.count);\n  const chosen: SnippetWindow[] = [];\n  for (const c of ranked) {\n    if (chosen.length >= countLimit) break;\n    if (chosen.some((s) => overlaps(s, c))) continue;\n    chosen.push(c);\n  }\n  chosen.sort((a, b) => a.start - b.start);\n  return { chosen, best: ranked[0] };\n}\n\n// Builds bounded marked passages and returns the best window's offset for section lookup.\nexport function computeSnippets(text: string, terms: string[], charLimit: number, countLimit: number): { snippets: string[]; offset: number } {\n  const occ = findOccurrences(text, terms);\n  const bounds = wordBoundaries(text);\n  if (occ.length === 0) {\n    // A term matched only via title/summary, not this body text, finds no occurrence here;\n    // fall back to the doc's start, unmarked.\n    const render = (end: number): string => `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`;\n    const floor = bounds.find((b) => b > 0) ?? text.length;\n    const upperBound = Math.max(floor, cutWindow(text, bounds, 0, charLimit).end);\n    const end = fitEnd(render, bounds, floor, upperBound, charLimit);\n    return { snippets: [render(end)], offset: 0 };\n  }\n  const candidates = scoreWindows(occ, bounds, text, charLimit);\n  const { chosen, best } = selectWindows(candidates, countLimit);\n  return { snippets: chosen.map((win) => renderWindow(text, occ, win)), offset: best.start };\n}\n\n// Converts an offset in the original text to its one-based line number.\nexport function lineNumberAt(text: string, offset: number): number {\n  let line = 1;\n  for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n  return line;\n}\n\nasync function lineRangeFor(store: Store, path: string, line: number): Promise<string | null> {\n  const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');\n  const row = (await stmt.get(path, line, line)) as { start_line: number; end_line: number } | undefined;\n  return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n  include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n  exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n  noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n  snippetCharLimit?: number; // --snippet-char-limit; defaults to SNIPPET_CHAR_LIMIT_DEFAULT\n  snippetCountLimit?: number; // --snippet-count-limit; defaults to SNIPPET_COUNT_LIMIT_DEFAULT\n}\n\n// Hydrates already-ranked rows with snippets and containing-section ranges without reranking.\nexport async function hydrateSearchRows(store: Store, cfg: ResolvedConfig, rows: Row[], matchedPaths: Set<string>, terms: string, opts: Pick<SearchOptions, 'snippetCharLimit' | 'snippetCountLimit'> = {}): Promise<void> {\n  if (matchedPaths.size === 0) return;\n  const bareTerms = extractBareTerms(terms);\n  const charLimit = opts.snippetCharLimit ?? SNIPPET_CHAR_LIMIT_DEFAULT;\n  const countLimit = opts.snippetCountLimit ?? SNIPPET_COUNT_LIMIT_DEFAULT;\n  for (const row of rows) {\n    if (!matchedPaths.has(row.path as string)) continue;\n    let text: string;\n    try {\n      text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n    } catch {\n      continue; // vanished since the match; leave snippets/lines empty rather than throw\n    }\n    const { snippets, offset } = computeSnippets(text, bareTerms, charLimit, countLimit);\n    row.snippets = snippets;\n    if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path as string, lineNumberAt(text, offset)) : null;\n  }\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(store: Store, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n  const effective = resolveSearch(cfg, opts);\n  const { k, signals } = effective;\n\n  const allPaths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n  const scopePaths = await rawScope(store, cfg, opts, allPaths);\n  const scopeActive = scopePaths.size < allPaths.length;\n  // The set every candidate pool must be filtered to before truncation: scope narrowed by\n  // --where, the same composition scopedPaths() gives the other commands.\n  let allowedPaths: Set<string>;\n  try {\n    allowedPaths = await narrowByWhere(store, scopePaths, effective.where);\n  } catch (err) {\n    throw searchError(err as Error, terms, effective.where);\n  }\n  const fetch = Math.max(k * 3, 30);\n\n  // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n  // local path with missing files errors here, since nothing will ever fetch it for itself.\n  const wantsVectors = signals.vectors !== undefined;\n  if (wantsVectors) {\n    const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n    if (localModelMissing(e)) {\n      throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n    }\n  }\n  const semanticEnabled = wantsVectors && (await scopeHasEmbeddings(store, cfg, allowedPaths));\n\n  // --where applies inside the candidate query (a post-filter would drop matches ranked past\n  // the pool) and again on the final select, for link-derived rows.\n  const scope = effective.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  // Filtered before LIMIT, not after, or scoped notes ranking below the global top-`fetch`\n  // never reach the filter; joined against a temp table since real scopes exceed SQLITE_MAX_VARIABLE_NUMBER.\n  if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);\n  const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n\n  const candidates = new Map<string, { score: number; via: string }>();\n  let matchRows: LexicalHit[] = [];\n  const hasEmptyPhrase = [...terms.matchAll(/\"([^\"]*)\"/g)].some((match) => !/[\\p{L}\\p{N}_]/u.test(match[1]));\n  const hasBareSyntaxOnly = /[()]/u.test(terms.replace(/\"[^\"]*\"/g, ' '));\n  // A query that is only whitespace or punctuation has no lexical rows; other declared signals\n  // compose normally. In particular, unspaced-script punctuation can rewrite to an empty MATCH.\n  if (signals.words !== undefined && terms.trim() !== '' && (/[\\p{L}\\p{N}_]/u.test(terms) || hasBareSyntaxOnly) && !hasEmptyPhrase) {\n    const syntaxError = bareTermSyntaxError(terms);\n    if (syntaxError) throw syntaxError;\n    try {\n      matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);\n    } catch (err) {\n      throw searchError(err as Error, terms, scope);\n    }\n  }\n  // Every store's lexical `snippets` is NULL; the snippet text is always computed below.\n  // Distinguish from via='link' rows, which never appear in matchRows.\n  const matchedPaths = new Set(matchRows.map((r) => r.path));\n\n  if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n  let chunkLines = new Map<string, string>();\n  let chunkSimilarity = new Map<string, number>();\n  if (semanticEnabled) {\n    ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n  }\n\n  await store.exec('DROP TABLE IF EXISTS _search');\n  // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the\n  // rounded score/similarity are printed at full precision in rows.\n  await store.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score DOUBLE, via TEXT, lines TEXT, similarity DOUBLE)');\n  if (candidates.size > 0) {\n    await store.runBatch(\n      'INSERT INTO _search (\"path\", score, via, lines, similarity) VALUES (?, ?, ?, ?, ?)',\n      [...candidates].map(([path, c]) => [path, c.score, c.via, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null])\n    );\n  }\n\n  // Reapplies --where even though _search is already scope+where filtered, since the join to\n  // frontmatter is already needed for the path column.\n  const where = scope ? `WHERE (${scope})` : '';\n  // lines: semantic rows carry their chunk's range, lexical rows gain one below,\n  // everything else stays null; similarity stays semantic-only.\n  const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n  // NULL AS snippets holds this column's position (between summary and via) for every row;\n  // overwritten below, always to an array, never left null (DESIGN.md).\n  const selectStmt = await store.prepare(\n    `SELECT f.\"path\" AS path, content.title, content.summary, NULL AS snippets, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n       FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n       ${where} ORDER BY _search.score DESC LIMIT ?`\n  );\n  const rows = (await selectStmt.all(k)) as Row[];\n  for (const row of rows) row.snippets = [];\n\n  await hydrateSearchRows(store, cfg, rows, matchedPaths, terms, opts);\n\n  return rows;\n}\n"],"names":["readFileSync","join","stemmer","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","bareTermSyntaxError","searchError","foldForSearch","hasUnspacedRun","searchTokens","unspacedRuns","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","SNIPPET_CHAR_LIMIT_DEFAULT","SNIPPET_COUNT_LIMIT_DEFAULT","extractBareTerms","query","cleaned","replace","split","map","tok","flatMap","text","toLowerCase","trim","filter","length","wordSegmenter","findOccurrences","terms","occ","textLower","unspacedTerms","run","runLower","slice","start","end","term","idx","found","indexOf","push","spacedTerms","t","termByStem","Map","stemInitials","Set","Array","from","keys","match","segment","unspaced","folded","has","get","undefined","sort","a","b","wordBoundaries","Intl","Segmenter","granularity","bounds","index","snapForward","pos","lo","hi","mid","snapBackward","result","firstBoundAfter","cutWindow","anchor","charLimit","rawEnd","Math","min","fitEnd","render","floor","upperBound","firstOccAtOrAfter","renderWindow","win","out","cursor","i","o","prefix","suffix","scoreWindows","candidates","winEnd","seen","count","j","add","max","e","distinct","size","overlaps","selectWindows","countLimit","ranked","chosen","c","some","s","best","computeSnippets","find","snippets","offset","lineNumberAt","line","charCodeAt","lineRangeFor","store","path","stmt","prepare","row","start_line","end_line","hydrateSearchRows","cfg","rows","matchedPaths","opts","bareTerms","snippetCharLimit","snippetCountLimit","baseDir","lines","search","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","wantsVectors","vectors","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","matchRows","hasEmptyPhrase","matchAll","test","hasBareSyntaxOnly","words","syntaxError","links","chunkLines","chunkSimilarity","exec","runBatch","score","via","similarityCol","selectStmt"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,OAAO,QAAQ,UAAU;AAElC,SAASC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AAChF,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,mBAAmB,EAAEC,WAAW,QAAQ,4BAA4B;AAE7E,SAASC,aAAa,EAAEC,cAAc,EAAEC,YAAY,EAAEC,YAAY,QAAQ,qBAAqB;AAC/F,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAMC,6BAA6B,GAAG;AAC7C,OAAO,MAAMC,8BAA8B,EAAE;AAE7C,yFAAyF;AACzF,oGAAoG;AACpG,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,kCAAkC;IAC7C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFI,OAAO,CAAC,CAACD,MAAQjB,aAAaiB,KAAKD,GAAG,CAAC,CAAC,EAAEG,IAAI,EAAE,GAAKA,OACrDH,GAAG,CAAC,CAACC,MAAQA,IAAIG,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACL,MAAQA,IAAIM,MAAM,GAAG;AAClC;AAEA,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAIC;AAEJ,SAASC,gBAAgBN,IAAY,EAAEO,KAAe;IACpD,MAAMC,MAA2D,EAAE;IACnE,MAAMC,YAAYT,KAAKC,WAAW;IAElC,8FAA8F;IAC9F,wFAAwF;IACxF,MAAMS,gBAAgBH,MAAMJ,MAAM,CAACvB;IACnC,IAAI8B,cAAcN,MAAM,GAAG,GAAG;QAC5B,KAAK,MAAMO,OAAO7B,aAAakB,MAAO;YACpC,MAAMY,WAAWH,UAAUI,KAAK,CAACF,IAAIG,KAAK,EAAEH,IAAII,GAAG;YACnD,KAAK,MAAMC,QAAQN,cAAe;gBAChC,IAAIO,MAAM;gBACV,OAAS;oBACP,MAAMC,QAAQN,SAASO,OAAO,CAACH,MAAMC;oBACrC,IAAIC,UAAU,CAAC,GAAG;oBAClBV,IAAIY,IAAI,CAAC;wBAAEN,OAAOH,IAAIG,KAAK,GAAGI;wBAAOH,KAAKJ,IAAIG,KAAK,GAAGI,QAAQF,KAAKZ,MAAM;wBAAEY;oBAAK;oBAChFC,MAAMC,QAAQF,KAAKZ,MAAM;gBAC3B;YACF;QACF;IACF;IAEA,6FAA6F;IAC7F,uGAAuG;IACvG,MAAMiB,cAAcd,MAAMJ,MAAM,CAAC,CAACmB,IAAM,CAAC1C,eAAe0C;IACxD,IAAID,YAAYjB,MAAM,GAAG,GAAG;QAC1B,MAAMmB,aAAa,IAAIC,IAAIH,YAAYxB,GAAG,CAAC,CAACyB,IAAM;gBAACpD,QAAQS,cAAc2C;gBAAKA;aAAE;QAChF,MAAMG,eAAe,IAAIC,IAAIC,MAAMC,IAAI,CAACL,WAAWM,IAAI,IAAI,CAACb,OAASA,IAAI,CAAC,EAAE;QAC5E,KAAK,MAAMc,SAASjD,aAAamB,MAAO;YACtC,MAAM+B,UAAUD,MAAM9B,IAAI;YAC1B,IAAI8B,MAAME,QAAQ,EAAE;YACpB,MAAMC,SAAStD,cAAcoD;YAC7B,2FAA2F;YAC3F,IAAI,CAACN,aAAaS,GAAG,CAACD,MAAM,CAAC,EAAE,GAAG;YAClC,MAAMjB,OAAOO,WAAWY,GAAG,CAACjE,QAAQ+D;YACpC,IAAIjB,SAASoB,WAAW5B,IAAIY,IAAI,CAAC;gBAAEN,OAAOgB,MAAMhB,KAAK;gBAAEC,KAAKe,MAAMf,GAAG;gBAAEC;YAAK;QAC9E;IACF;IAEA,OAAOR,IAAI6B,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAExB,KAAK,GAAGyB,EAAEzB,KAAK,IAAIyB,EAAExB,GAAG,GAAGuB,EAAEvB,GAAG;AAC9D;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,gGAAgG;AAChG,SAASyB,eAAexC,IAAY;IAClCK,0BAAAA,2BAAAA,gBAAAA,gBAAkB,IAAIoC,KAAKC,SAAS,CAAC,MAAM;QAAEC,aAAa;IAAO;IACjE,MAAMC,SAAS;QAAC;KAAE;IAClB,KAAK,MAAM,EAAEC,KAAK,EAAEd,OAAO,EAAE,IAAI1B,cAAc0B,OAAO,CAAC/B,MAAO4C,OAAOxB,IAAI,CAACyB,QAAQd,QAAQ3B,MAAM;IAChG,OAAOwC;AACT;AAEA,0FAA0F;AAC1F,SAASE,YAAYF,MAAgB,EAAEG,GAAW;IAChD,IAAIC,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM,GAAG;IACzB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,MAAO;QACzB,IAAIL,MAAM,CAACM,IAAI,GAAGH,KAAKC,KAAKE,MAAM;aAC7BD,KAAKC;IACZ;IACA,OAAON,MAAM,CAACI,GAAG;AACnB;AAEA,SAASG,aAAaP,MAAgB,EAAEG,GAAW;IACjD,IAAIC,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM,GAAG;IACzB,IAAIgD,SAASR,MAAM,CAAC,EAAE;IACtB,MAAOI,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,KAAK,KAAM;QAC7B,IAAIL,MAAM,CAACM,IAAI,IAAIH,KAAK;YACtBC,KAAKE;YACLE,SAASR,MAAM,CAACM,IAAI;QACtB,OAAOD,KAAKC,MAAM;IACpB;IACA,OAAOE;AACT;AAEA,SAASC,gBAAgBT,MAAgB,EAAEG,GAAW;IACpD,IAAIC,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM;IACtB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,MAAO;QACzB,IAAIL,MAAM,CAACM,IAAI,IAAIH,KAAKC,KAAKE,MAAM;aAC9BD,KAAKC;IACZ;IACA,OAAON,MAAM,CAACI,GAAG;AACnB;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,8FAA8F;AAC9F,+FAA+F;AAC/F,mFAAmF;AACnF,SAASM,UAAUtD,IAAY,EAAE4C,MAAgB,EAAEW,MAAc,EAAEC,SAAiB;QAI1DH;IAHxB,MAAMI,SAASC,KAAKC,GAAG,CAAC3D,KAAKI,MAAM,EAAEmD,SAASC;IAC9C,MAAM1C,QAAQgC,YAAYF,QAAQW;IAClC,IAAIxC,MAAMoC,aAAaP,QAAQa;IAC/B,IAAI1C,OAAOD,OAAOC,OAAMsC,mBAAAA,gBAAgBT,QAAQ9B,oBAAxBuC,8BAAAA,mBAAkCrD,KAAKI,MAAM;IACrE,OAAO;QAAEU;QAAOC,KAAK2C,KAAKC,GAAG,CAAC5C,KAAKf,KAAKI,MAAM;IAAE;AAClD;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,wFAAwF;AACxF,6FAA6F;AAC7F,+FAA+F;AAC/F,wDAAwD;AACxD,SAASwD,OAAOC,MAA+B,EAAEjB,MAAgB,EAAEkB,KAAa,EAAEC,UAAkB,EAAEP,SAAiB;IACrH,IAAIR,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM,GAAG;IACzB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,KAAK,KAAM;QAC7B,IAAIL,MAAM,CAACM,IAAI,IAAIa,YAAYf,KAAKE;aAC/BD,KAAKC,MAAM;IAClB;IACA,MAAOF,MAAM,KAAKJ,MAAM,CAACI,GAAG,GAAGc,OAAOd,KAAM;QAC1C,IAAIa,OAAOjB,MAAM,CAACI,GAAG,EAAE5C,MAAM,IAAIoD,WAAW,OAAOZ,MAAM,CAACI,GAAG;IAC/D;IACA,OAAOc;AACT;AAYA,6FAA6F;AAC7F,SAASE,kBAAkBxD,GAAwD,EAAEuC,GAAW;IAC9F,IAAIC,KAAK;IACT,IAAIC,KAAKzC,IAAIJ,MAAM;IACnB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,MAAO;QACzB,IAAIzC,GAAG,CAAC0C,IAAI,CAACpC,KAAK,GAAGiC,KAAKC,KAAKE,MAAM;aAChCD,KAAKC;IACZ;IACA,OAAOF;AACT;AAEA,SAASiB,aAAajE,IAAY,EAAEQ,GAAwD,EAAE0D,GAAS;IACrG,IAAIC,MAAM;IACV,IAAIC,SAASF,IAAIpD,KAAK;IACtB,IAAK,IAAIuD,IAAIL,kBAAkBxD,KAAK0D,IAAIpD,KAAK,GAAGuD,IAAI7D,IAAIJ,MAAM,IAAII,GAAG,CAAC6D,EAAE,CAACvD,KAAK,GAAGoD,IAAInD,GAAG,EAAEsD,IAAK;QAC7F,MAAMC,IAAI9D,GAAG,CAAC6D,EAAE;QAChB,IAAIC,EAAEvD,GAAG,GAAGmD,IAAInD,GAAG,EAAE;QACrB,sFAAsF;QACtF,gFAAgF;QAChF,IAAIuD,EAAExD,KAAK,GAAGsD,QAAQ;QACtBD,OAAO,GAAGnE,KAAKa,KAAK,CAACuD,QAAQE,EAAExD,KAAK,EAAE,CAAC,EAAEd,KAAKa,KAAK,CAACyD,EAAExD,KAAK,EAAEwD,EAAEvD,GAAG,EAAE,CAAC,CAAC;QACtEqD,SAASE,EAAEvD,GAAG;IAChB;IACAoD,OAAOnE,KAAKa,KAAK,CAACuD,QAAQF,IAAInD,GAAG;IACjC,MAAMwD,SAASL,IAAIpD,KAAK,GAAG,IAAI,MAAM;IACrC,MAAM0D,SAASN,IAAInD,GAAG,GAAGf,KAAKI,MAAM,GAAG,MAAM;IAC7C,OAAO,GAAGmE,SAASJ,IAAIxE,OAAO,CAAC,QAAQ,OAAO6E,QAAQ;AACxD;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,iGAAiG;AACjG,4FAA4F;AAC5F,6FAA6F;AAC7F,4CAA4C;AAC5C,SAASC,aAAajE,GAAwD,EAAEoC,MAAgB,EAAE5C,IAAY,EAAEwD,SAAiB;IAC/H,MAAMkB,aAA8B,EAAE;IACtC,IAAK,IAAIL,IAAI,GAAGA,IAAI7D,IAAIJ,MAAM,EAAEiE,IAAK;QACnC,MAAMM,SAASnE,GAAG,CAAC6D,EAAE,CAACvD,KAAK,GAAG0C;QAC9B,MAAMoB,OAAO,IAAIlD;QACjB,IAAImD,QAAQ;QACZ,IAAK,IAAIC,IAAIT,GAAGS,IAAItE,IAAIJ,MAAM,IAAII,GAAG,CAACsE,EAAE,CAAChE,KAAK,GAAG6D,QAAQG,IAAK;YAC5DF,KAAKG,GAAG,CAACvE,GAAG,CAACsE,EAAE,CAAC9D,IAAI;YACpB6D;QACF;QACA,MAAM/D,QAAQgC,YAAYF,QAAQpC,GAAG,CAAC6D,EAAE,CAACvD,KAAK;QAC9C,MAAMgD,QAAQJ,KAAKsB,GAAG,CAAClE,OAAON,GAAG,CAAC6D,EAAE,CAACtD,GAAG;QACxC,MAAMgD,aAAaL,KAAKsB,GAAG,CAAClB,OAAOR,UAAUtD,MAAM4C,QAAQpC,GAAG,CAAC6D,EAAE,CAACvD,KAAK,EAAE0C,WAAWzC,GAAG;QACvF,MAAMA,MAAM6C,OAAO,CAACqB,IAAMhB,aAAajE,MAAMQ,KAAK;gBAAEM;gBAAOC,KAAKkE;YAAE,IAAIrC,QAAQkB,OAAOC,YAAYP;QACjGkB,WAAWtD,IAAI,CAAC;YAAEN;YAAOC;YAAKmE,UAAUN,KAAKO,IAAI;YAAEN;QAAM;IAC3D;IACA,OAAOH;AACT;AAEA,SAASU,SAAS9C,CAAO,EAAEC,CAAO;IAChC,OAAOD,EAAExB,KAAK,GAAGyB,EAAExB,GAAG,IAAIwB,EAAEzB,KAAK,GAAGwB,EAAEvB,GAAG;AAC3C;AAEA,4FAA4F;AAC5F,gGAAgG;AAChG,mFAAmF;AACnF,SAASsE,cAAcX,UAA2B,EAAEY,UAAkB;IACpE,MAAMC,SAASb,WAAW7D,KAAK,GAAGwB,IAAI,CAAC,CAACC,GAAGC,IAAMA,EAAE2C,QAAQ,GAAG5C,EAAE4C,QAAQ,IAAI3C,EAAEsC,KAAK,GAAGvC,EAAEuC,KAAK;IAC7F,MAAMW,SAA0B,EAAE;IAClC,KAAK,MAAMC,KAAKF,OAAQ;QACtB,IAAIC,OAAOpF,MAAM,IAAIkF,YAAY;QACjC,IAAIE,OAAOE,IAAI,CAAC,CAACC,IAAMP,SAASO,GAAGF,KAAK;QACxCD,OAAOpE,IAAI,CAACqE;IACd;IACAD,OAAOnD,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAExB,KAAK,GAAGyB,EAAEzB,KAAK;IACvC,OAAO;QAAE0E;QAAQI,MAAML,MAAM,CAAC,EAAE;IAAC;AACnC;AAEA,0FAA0F;AAC1F,OAAO,SAASM,gBAAgB7F,IAAY,EAAEO,KAAe,EAAEiD,SAAiB,EAAE8B,UAAkB;IAClG,MAAM9E,MAAMF,gBAAgBN,MAAMO;IAClC,MAAMqC,SAASJ,eAAexC;IAC9B,IAAIQ,IAAIJ,MAAM,KAAK,GAAG;YAINwC;QAHd,uFAAuF;QACvF,0CAA0C;QAC1C,MAAMiB,SAAS,CAAC9C,MAAwB,GAAGf,KAAKa,KAAK,CAAC,GAAGE,KAAKpB,OAAO,CAAC,QAAQ,KAAKO,IAAI,KAAKa,MAAMf,KAAKI,MAAM,GAAG,MAAM,IAAI;QAC1H,MAAM0D,SAAQlB,eAAAA,OAAOkD,IAAI,CAAC,CAACvD,IAAMA,IAAI,gBAAvBK,0BAAAA,eAA6B5C,KAAKI,MAAM;QACtD,MAAM2D,aAAaL,KAAKsB,GAAG,CAAClB,OAAOR,UAAUtD,MAAM4C,QAAQ,GAAGY,WAAWzC,GAAG;QAC5E,MAAMA,MAAM6C,OAAOC,QAAQjB,QAAQkB,OAAOC,YAAYP;QACtD,OAAO;YAAEuC,UAAU;gBAAClC,OAAO9C;aAAK;YAAEiF,QAAQ;QAAE;IAC9C;IACA,MAAMtB,aAAaD,aAAajE,KAAKoC,QAAQ5C,MAAMwD;IACnD,MAAM,EAAEgC,MAAM,EAAEI,IAAI,EAAE,GAAGP,cAAcX,YAAYY;IACnD,OAAO;QAAES,UAAUP,OAAO3F,GAAG,CAAC,CAACqE,MAAQD,aAAajE,MAAMQ,KAAK0D;QAAO8B,QAAQJ,KAAK9E,KAAK;IAAC;AAC3F;AAEA,wEAAwE;AACxE,OAAO,SAASmF,aAAajG,IAAY,EAAEgG,MAAc;IACvD,IAAIE,OAAO;IACX,IAAK,IAAI7B,IAAI,GAAGA,IAAI2B,QAAQ3B,IAAK,IAAIrE,KAAKmG,UAAU,CAAC9B,OAAO,IAAI6B;IAChE,OAAOA;AACT;AAEA,eAAeE,aAAaC,KAAY,EAAEC,IAAY,EAAEJ,IAAY;IAClE,MAAMK,OAAO,MAAMF,MAAMG,OAAO,CAAC;IACjC,MAAMC,MAAO,MAAMF,KAAKpE,GAAG,CAACmE,MAAMJ,MAAMA;IACxC,OAAOO,MAAM,CAAC,CAAC,EAAEA,IAAIC,UAAU,CAAC,CAAC,EAAED,IAAIE,QAAQ,EAAE,GAAG;AACtD;AAaA,8FAA8F;AAC9F,OAAO,eAAeC,kBAAkBP,KAAY,EAAEQ,GAAmB,EAAEC,IAAW,EAAEC,YAAyB,EAAExG,KAAa,EAAEyG,OAAsE,CAAC,CAAC;QAGtLA,wBACCA;IAHnB,IAAID,aAAa5B,IAAI,KAAK,GAAG;IAC7B,MAAM8B,YAAYzH,iBAAiBe;IACnC,MAAMiD,aAAYwD,yBAAAA,KAAKE,gBAAgB,cAArBF,oCAAAA,yBAAyB1H;IAC3C,MAAMgG,cAAa0B,0BAAAA,KAAKG,iBAAiB,cAAtBH,qCAAAA,0BAA0BzH;IAC7C,KAAK,MAAMkH,OAAOK,KAAM;QACtB,IAAI,CAACC,aAAa7E,GAAG,CAACuE,IAAIH,IAAI,GAAa;QAC3C,IAAItG;QACJ,IAAI;YACFA,OAAOhC,aAAaC,KAAK4I,IAAIO,OAAO,EAAEX,IAAIH,IAAI,GAAa;QAC7D,EAAE,OAAM;YACN,UAAU,yEAAyE;QACrF;QACA,MAAM,EAAEP,QAAQ,EAAEC,MAAM,EAAE,GAAGH,gBAAgB7F,MAAMiH,WAAWzD,WAAW8B;QACzEmB,IAAIV,QAAQ,GAAGA;QACf,IAAIU,IAAIY,KAAK,IAAI,MAAMZ,IAAIY,KAAK,GAAGjJ,eAAeyI,KAAK,cAAc,MAAMT,aAAaC,OAAOI,IAAIH,IAAI,EAAYL,aAAajG,MAAMgG,WAAW;IACnJ;AACF;AAEA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAesB,OAAOjB,KAAY,EAAEQ,GAAmB,EAAEtG,KAAa,EAAEyG,OAAsB,CAAC,CAAC;IACrG,MAAMO,YAAYlJ,cAAcwI,KAAKG;IACrC,MAAM,EAAEQ,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMrB,MAAMG,OAAO,CAAC,iCAAgC,EAAGmB,GAAG,EAAC,EAA+B9H,GAAG,CAAC,CAAC+H,IAAMA,EAAEtB,IAAI;IACrI,MAAMuB,aAAa,MAAM5I,SAASoH,OAAOQ,KAAKG,MAAMU;IACpD,MAAMI,cAAcD,WAAW1C,IAAI,GAAGuC,SAAStH,MAAM;IACrD,wFAAwF;IACxF,wEAAwE;IACxE,IAAI2H;IACJ,IAAI;QACFA,eAAe,MAAM/I,cAAcqH,OAAOwB,YAAYN,UAAUS,KAAK;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAMvJ,YAAYuJ,KAAc1H,OAAOgH,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQxE,KAAKsB,GAAG,CAACwC,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMW,eAAeV,QAAQW,OAAO,KAAKhG;IACzC,IAAI+F,cAAc;QAChB,MAAMlD,IAAI9G,YAAY0I,MAAM,oEAAoE;QAChG,IAAIvI,kBAAkB2G,IAAI;YACxB,MAAM,IAAIzG,WAAW,uBAAuB,CAAC,QAAQ,EAAE+I,UAAUc,UAAU,CAAC,mDAAmD,EAAEpD,EAAEqD,KAAK,CAAC,aAAa,EAAE/J,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAMgK,kBAAkBJ,gBAAiB,MAAMjJ,mBAAmBmH,OAAOQ,KAAKkB;IAE9E,2FAA2F;IAC3F,kEAAkE;IAClE,MAAMS,QAAQjB,UAAUS,KAAK;IAC7B,MAAMS,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,2GAA2G;IAC3G,IAAIV,aAAa,MAAM/I,iBAAiBsH,OAAO,iBAAiBwB;IAChE,MAAMc,YAAYb,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAE3F,MAAMpD,aAAa,IAAIlD;IACvB,IAAIoH,YAA0B,EAAE;IAChC,MAAMC,iBAAiB;WAAItI,MAAMuI,QAAQ,CAAC;KAAc,CAACpD,IAAI,CAAC,CAAC5D,QAAU,CAAC,iBAAiBiH,IAAI,CAACjH,KAAK,CAAC,EAAE;IACxG,MAAMkH,oBAAoB,QAAQD,IAAI,CAACxI,MAAMZ,OAAO,CAAC,YAAY;IACjE,6FAA6F;IAC7F,8FAA8F;IAC9F,IAAI8H,QAAQwB,KAAK,KAAK7G,aAAa7B,MAAML,IAAI,OAAO,MAAO,CAAA,iBAAiB6I,IAAI,CAACxI,UAAUyI,iBAAgB,KAAM,CAACH,gBAAgB;QAChI,MAAMK,cAAczK,oBAAoB8B;QACxC,IAAI2I,aAAa,MAAMA;QACvB,IAAI;YACFN,YAAY,MAAMvJ,gBAAgBgH,OAAO3B,YAAYnE,OAAOkI,WAAWC,WAAWC,WAAWT,OAAOT,QAAQwB,KAAK;QACnH,EAAE,OAAOhB,KAAK;YACZ,MAAMvJ,YAAYuJ,KAAc1H,OAAOiI;QACzC;IACF;IACA,uFAAuF;IACvF,qEAAqE;IACrE,MAAMzB,eAAe,IAAIrF,IAAIkH,UAAU/I,GAAG,CAAC,CAAC+H,IAAMA,EAAEtB,IAAI;IAExD,IAAImB,QAAQ0B,KAAK,KAAK/G,WAAW,MAAMjD,gBAAgBkH,OAAO3B,YAAYkE,WAAWlB,UAAUK,cAAcG,OAAOT,QAAQ0B,KAAK;IAEjI,IAAIC,aAAa,IAAI5H;IACrB,IAAI6H,kBAAkB,IAAI7H;IAC1B,IAAI+G,iBAAiB;QAClB,CAAA,EAAEa,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMjK,kBAAkBiH,OAAOQ,KAAKnC,YAAYnE,OAAO2H,OAAOH,cAAcN,QAAQW,OAAO,CAAU;IAC1I;IAEA,MAAM/B,MAAMiD,IAAI,CAAC;IACjB,8FAA8F;IAC9F,kEAAkE;IAClE,MAAMjD,MAAMiD,IAAI,CAAC;IACjB,IAAI5E,WAAWS,IAAI,GAAG,GAAG;QACvB,MAAMkB,MAAMkD,QAAQ,CAClB,sFACA;eAAI7E;SAAW,CAAC7E,GAAG,CAAC,CAAC,CAACyG,MAAMb,EAAE;gBAA4B2D,iBAA8BC;mBAArD;gBAAC/C;gBAAMb,EAAE+D,KAAK;gBAAE/D,EAAEgE,GAAG;iBAAEL,kBAAAA,WAAWjH,GAAG,CAACmE,mBAAf8C,6BAAAA,kBAAwB;iBAAMC,uBAAAA,gBAAgBlH,GAAG,CAACmE,mBAApB+C,kCAAAA,uBAA6B;aAAK;;IAE9H;IAEA,2FAA2F;IAC3F,qDAAqD;IACrD,MAAMrB,QAAQQ,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,+EAA+E;IAC/E,8DAA8D;IAC9D,MAAMkB,gBAAgBnB,kBAAkB,yBAAyB;IACjE,yFAAyF;IACzF,sEAAsE;IACtE,MAAMoB,aAAa,MAAMtD,MAAMG,OAAO,CACpC,CAAC,uIAAuI,EAAEkD,cAAc;;OAErJ,EAAE1B,MAAM,oCAAoC,CAAC;IAElD,MAAMlB,OAAQ,MAAM6C,WAAWhC,GAAG,CAACH;IACnC,KAAK,MAAMf,OAAOK,KAAML,IAAIV,QAAQ,GAAG,EAAE;IAEzC,MAAMa,kBAAkBP,OAAOQ,KAAKC,MAAMC,cAAcxG,OAAOyG;IAE/D,OAAOF;AACT"}

package/dist/esm/output/output.js

@@ -7,8 +7,10 @@ function cell(value) {
 }
 // RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every
 // character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.
+// An array (e.g. snippets) joins with the one delimiter our content provably cannot contain,
+// inside the quoted field newlines already trigger (DESIGN.md "Rendering a list of snippets").
 function csvField(value) {
-    const text = value === null || value === undefined ? '' : String(value);
+    const text = value === null || value === undefined ? '' : Array.isArray(value) ? value.join('\n') : String(value);
     return /["\r\n,]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
 }
 function csvLine(values) {
@@ -122,19 +124,40 @@ function fitWidths(widths, limit) {
         fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));
     }
 }
+// An array cell (snippets) becomes one physical line per element; every other value stays the
+// single line cell() already produced. No cell in any other command holds one today, so a row
+// with no array-valued cell yields exactly one physical line, unchanged from before this existed.
+function cellLines(value) {
+    if (Array.isArray(value)) return value.length === 0 ? [
+        ''
+    ] : value.map((v)=>cell(v));
+    return [
+        cell(value)
+    ];
+}
 function renderRows(rows, format, width = process.stdout.columns) {
     if (format === 'json') return stringifyJson(rows, 2);
     if (rows.length === 0) return '(0 rows)';
     const columns = Object.keys(rows[0]);
-    const cells = rows.map((row)=>columns.map((col)=>cell(row[col])));
-    const natural = columns.map((col, i)=>Math.max(col.length, ...cells.map((row)=>row[i].length)));
+    const linesByCell = rows.map((row)=>columns.map((col)=>cellLines(row[col])));
+    const heights = linesByCell.map((cellsInRow)=>Math.max(1, ...cellsInRow.map((lines)=>lines.length)));
+    const natural = columns.map((col, i)=>Math.max(col.length, ...linesByCell.flatMap((cellsInRow)=>cellsInRow[i]).map((line)=>line.length)));
     const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;
     const clip = (value, w)=>value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`;
     const formatRow = (values)=>values.map((value, i)=>clip(value, widths[i])).join('  ').trimEnd();
+    // A row becomes as many physical lines as its tallest cell; other columns blank-pad on
+    // continuation lines (DESIGN.md "Rendering a list of snippets").
+    const body = [];
+    for(let r = 0; r < rows.length; r++){
+        for(let li = 0; li < heights[r]; li++)body.push(formatRow(columns.map((_, c)=>{
+            var _linesByCell_r_c_li;
+            return (_linesByCell_r_c_li = linesByCell[r][c][li]) !== null && _linesByCell_r_c_li !== void 0 ? _linesByCell_r_c_li : '';
+        })));
+    }
     return [
         formatRow(columns),
         widths.map((w)=>'-'.repeat(w)).join('  '),
-        ...cells.map(formatRow)
+        ...body
     ].join('\n');
 }
 // Text renderers for the top-level commands; cli.ts prints what these return.

package/dist/esm/output/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output keeps them as-is.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every\n// character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// JSON.stringify throws on BigInt, and stores return int64 as BigInt (DuckDB does): a value\n// that fits a safe integer stays a number, a larger one becomes its decimal string.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nconst bigintSafe = (_key: string, value: unknown): unknown => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n\nexport function stringifyJson(value: unknown, indent?: number): string {\n  return JSON.stringify(value, bigintSafe, indent);\n}\n\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Names from a row wherever there is one: the statement's column list keeps duplicates the\n    // row collapsed. Nothing at all with no row and no columns -- a bare newline is an empty record.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: the result is never held as\n// rows and again as a rendered string. Table buffers (column width depends on the last row); json/csv write row by row.\nexport async function printRowStream(rows: AsyncIterable<Row>, format: RowFormat, columns: string[]): Promise<void> {\n  const iterator = rows[Symbol.asyncIterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = await iterator.next();\n  const rest: AsyncIterable<Row> = { [Symbol.asyncIterator]: () => iterator };\n\n  if (format === 'table') {\n    const collected: Row[] = first.done ? [] : [first.value];\n    for await (const row of rest) collected.push(row);\n    printRows(collected, format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    const indent = (row: Row) =>\n      stringifyJson(row, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for await (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for await (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return stringifyJson(rows, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\n// name, or name:weight when the weight is not the default 1.\nfunction signalLabels(signals: Record<string, number>): string {\n  return Object.entries(signals)\n    .map(([name, weight]) => (weight === 1 ? name : `${name}:${weight}`))\n    .join(', ');\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a preset without the vectors signal says what it uses instead.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)];\n}\n\nexport function renderMap(result: {\n  docs: { count: number; bytes: number };\n  fields: Row[];\n  fieldsTotal: number;\n  features: { on: string[]; off: string[] };\n  presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>;\n  hubs: Row[];\n  recent: Row[];\n  recentCaveat: string | null;\n}): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","test","csvLine","values","map","join","warnOversize","bytes","rowCount","console","warn","Math","round","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","stringifyJson","indent","JSON","stringify","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","asyncIterator","first","next","rest","collected","done","push","write","process","stdout","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","cells","col","natural","i","clip","w","padEnd","slice","formatRow","trimEnd","repeat","featureOffHint","featuresLine","features","off","on","signalLabels","signals","entries","weight","presetsLines","presets","p","files","embedded","vectors","renderMap","result","parts","docs","count","fields","fieldsTotal","hubs","includes","recentCaveat","recent","renderPeek","lines","path","tokens","parseError","key","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,sGAAsG;AACtG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKJ,OAAOD;IACjE,OAAO,WAAWM,IAAI,CAACF,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASG,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACN,UAAUO,IAAI,CAAC;AACnC;AAEA,SAASC,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAASd,gBAAgB;IAC7BgB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,4FAA4F;AAC5F,oFAAoF;AACpF,MAAMK,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,MAAMC,aAAa,CAACC,MAAcvB,QAA6B,OAAOA,UAAU,WAAYA,SAAS,CAACkB,YAAYlB,SAASkB,WAAWE,OAAOpB,SAASA,MAAMwB,QAAQ,KAAMxB;AAE1K,OAAO,SAASyB,cAAczB,KAAc,EAAE0B,MAAe;IAC3D,OAAOC,KAAKC,SAAS,CAAC5B,OAAOsB,YAAYI;AAC3C;AAEA,OAAO,SAASG,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,2FAA2F;QAC3F,iGAAiG;QACjG,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAAC/B,QAAQ0B;eAAWH,KAAKrB,GAAG,CAAC,CAAC8B,MAAQhC,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASD,GAAG,CAACC,KAAK;SAAI,CAAC9B,IAAI,CAAC;QACtGI,QAAQ2B,GAAG,CAACH;QACZ,yDAAyD;QACzD3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCjB,QAAQ2B,GAAG,CAACH;IACZ3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,+FAA+F;AAC/F,wHAAwH;AACxH,OAAO,eAAeW,eAAef,IAAwB,EAAEC,MAAiB,EAAEC,OAAiB;IACjG,MAAMc,WAAWhB,IAAI,CAACiB,OAAOC,aAAa,CAAC;IAC3C,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMC,QAAQ,MAAMH,SAASI,IAAI;IACjC,MAAMC,OAA2B;QAAE,CAACJ,OAAOC,aAAa,CAAC,EAAE,IAAMF;IAAS;IAE1E,IAAIf,WAAW,SAAS;QACtB,MAAMqB,YAAmBH,MAAMI,IAAI,GAAG,EAAE,GAAG;YAACJ,MAAMjD,KAAK;SAAC;QACxD,WAAW,MAAMuC,OAAOY,KAAMC,UAAUE,IAAI,CAACf;QAC7CV,UAAUuB,WAAWrB,QAAQC;QAC7B;IACF;IAEA,IAAIpB,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM0C,QAAQ,CAACnD;QACb,yDAAyD;QACzDQ,SAAS8B,OAAOC,UAAU,CAACvC;QAC3BoD,QAAQC,MAAM,CAACF,KAAK,CAACnD;IACvB;IAEA,IAAI2B,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,MAAML,SAAS,CAACa,MACdd,cAAcc,KAAK,GAChBmB,KAAK,CAAC,MACNjD,GAAG,CAAC,CAACkD,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzBjD,IAAI,CAAC;QACV,IAAIuC,MAAMI,IAAI,EAAE;YACdE,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAM7B,OAAOuB,MAAMjD,KAAK;YACxBa,WAAW;YACX,WAAW,MAAM0B,OAAOY,KAAM;gBAC5BI,MAAM;gBACNA,MAAM7B,OAAOa;gBACb1B;YACF;YACA0C,MAAM;QACR;IACF,OAAO,IAAIN,MAAMI,IAAI,EAAE;QACrB,MAAMpB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGqB,MAAM,GAAGhD,QAAQ0B,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACa,MAAMjD,KAAK;QACrCuD,MAAM,GAAGhD,QAAQ0B,OAAO,EAAE,CAAC;QAC3BsB,MAAM,GAAGhD,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASS,MAAMjD,KAAK,CAACwC,KAAK,GAAG,EAAE,CAAC;QAC5D3B,WAAW;QACX,WAAW,MAAM0B,OAAOY,KAAM;YAC5BI,MAAM,GAAGhD,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpD3B;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM+C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAO7B,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMgC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACxD,KAAKyD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASpB,WAAWd,IAAW,EAAEC,MAAc,EAAE2C,QAAQlB,QAAQC,MAAM,CAACzB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAON,cAAcK,MAAM;IAClD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAM6C,QAAQ7C,KAAKrB,GAAG,CAAC,CAAC8B,MAAQP,QAAQvB,GAAG,CAAC,CAACmE,MAAQ7E,KAAKwC,GAAG,CAACqC,IAAI;IAClE,MAAMC,UAAU7C,QAAQvB,GAAG,CAAC,CAACmE,KAAKE,IAAM9D,KAAKyD,GAAG,CAACG,IAAI1C,MAAM,KAAKyC,MAAMlE,GAAG,CAAC,CAAC8B,MAAQA,GAAG,CAACuC,EAAE,CAAC5C,MAAM;IAChG,MAAM6B,SAASW,SAASA,QAAQd,aAAaE,UAAUe,SAASH,SAASG;IAEzE,MAAME,OAAO,CAAC/E,OAAegF,IAAehF,MAAMkC,MAAM,IAAI8C,IAAIhF,MAAMiF,MAAM,CAACD,KAAK,GAAGhF,MAAMkF,KAAK,CAAC,GAAGF,IAAI,KAAKnB,WAAW;IACxH,MAAMsB,YAAY,CAAC3E,SACjBA,OACGC,GAAG,CAAC,CAACT,OAAO8E,IAAMC,KAAK/E,OAAO+D,MAAM,CAACe,EAAE,GACvCpE,IAAI,CAAC,MACL0E,OAAO;IAEZ,OAAO;QAACD,UAAUnD;QAAU+B,OAAOtD,GAAG,CAAC,CAACuE,IAAM,IAAIK,MAAM,CAACL,IAAItE,IAAI,CAAC;WAAUiE,MAAMlE,GAAG,CAAC0E;KAAW,CAACzE,IAAI,CAAC;AACzG;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAAS4E,eAAe9C,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAAS+C,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAACvD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEsD,SAASC,GAAG,CAAChF,GAAG,CAAC,CAAC+B,OAAS,GAAGA,KAAK,EAAE,EAAE8C,eAAe9C,MAAM,CAAC,CAAC,EAAE9B,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE8E,SAASE,EAAE,CAAChF,IAAI,CAAC,QAAQ+E,KAAK;AACpD;AAEA,6DAA6D;AAC7D,SAASE,aAAaC,OAA+B;IACnD,OAAOzD,OAAO0D,OAAO,CAACD,SACnBnF,GAAG,CAAC,CAAC,CAAC+B,MAAMsD,OAAO,GAAMA,WAAW,IAAItD,OAAO,GAAGA,KAAK,CAAC,EAAEsD,QAAQ,EAClEpF,IAAI,CAAC;AACV;AAEA,OAAO,SAASqF,aAAaC,OAAkG;IAC7H,6FAA6F;IAC7F,8EAA8E;IAC9E,OAAO;QAAC;WAAeA,QAAQvF,GAAG,CAAC,CAACwF,IAAM,CAAC,EAAE,EAAEA,EAAEzD,IAAI,CAAC,EAAE,EAAEyD,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEL,OAAO,CAACQ,OAAO,KAAK/F,YAAY,KAAK,CAAC,WAAW,EAAEsF,aAAaM,EAAEL,OAAO,EAAE,CAAC,CAAC,EAAE;KAAE;AACzL;AAEA,OAAO,SAASS,UAAUC,MASzB;IACC,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEzF,KAAKC,KAAK,CAACqF,OAAOE,IAAI,CAAC5F,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE2E,aAAae,OAAOd,QAAQ;QAAG;WAAOO,aAAaO,OAAON,OAAO;QAAG;QAAIpD,WAAW0D,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACxE,MAAM,EAAEqE,MAAMjD,IAAI,CAAC,CAAC,EAAE,EAAEgD,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACxE,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIoE,OAAOM,IAAI,CAAC1E,MAAM,GAAG,GAAGqE,MAAMjD,IAAI,CAAC,0BAA0BV,WAAW0D,OAAOM,IAAI,EAAE;SACpF,IAAIN,OAAOd,QAAQ,CAACC,GAAG,CAACoB,QAAQ,CAAC,SAASN,MAAMjD,IAAI,CAAC,CAAC,aAAa,EAAEgC,eAAe,QAAQ,CAAC,CAAC;IACnGiB,MAAMjD,IAAI,CAAC;IACX,IAAIgD,OAAOQ,YAAY,EAAEP,MAAMjD,IAAI,CAACgD,OAAOQ,YAAY;IACvDP,MAAMjD,IAAI,CAACV,WAAW0D,OAAOS,MAAM,EAAE;IACrC,OAAOR,MAAM7F,IAAI,CAAC;AACpB;AAEA,OAAO,SAASsG,WAAWV,MAAoR;IAC7S,MAAMW,QAAQ;QAAC,GAAGX,OAAOY,IAAI,CAAC,IAAI,EAAEZ,OAAOa,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAIb,OAAOc,UAAU,EAAEH,MAAM3D,IAAI,CAAC,CAAC,wDAAwD,EAAEgD,OAAOc,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAKrH,MAAM,IAAImC,OAAO0D,OAAO,CAACS,OAAOgB,WAAW,EAAGL,MAAM3D,IAAI,CAAC,CAAC,EAAE,EAAE+D,IAAI,EAAE,EAAErH,OAAO;IAC9F,IAAIsG,OAAOiB,QAAQ,CAACrF,MAAM,GAAG,GAAG;QAC9B+E,MAAM3D,IAAI,CAAC,IAAI;QACf,KAAK,MAAMkE,KAAKlB,OAAOiB,QAAQ,CAAEN,MAAM3D,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI+B,MAAM,CAACmC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAIb,OAAOuB,aAAa,GAAGvB,OAAOiB,QAAQ,CAACrF,MAAM,EAAE+E,MAAM3D,IAAI,CAAC,CAAC,IAAI,EAAEgD,OAAOuB,aAAa,GAAGvB,OAAOiB,QAAQ,CAACrF,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAIoE,OAAOb,GAAG,CAACoB,QAAQ,CAAC,aAAaI,MAAM3D,IAAI,CAAC,IAAI;IAC3D,IAAIgD,OAAOb,GAAG,CAACoB,QAAQ,CAAC,UAAU;QAChCI,MAAM3D,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMwE,WAAW,CAACC,OAAeC,OAAiB7D;YAChD,IAAIA,QAAQ,GAAG8C,MAAM3D,IAAI,CAAC,GAAGyE,MAAM,EAAE,EAAE5D,MAAM,GAAG,EAAE6D,MAAMtH,IAAI,CAAC,QAAQyD,QAAQ6D,MAAM9F,MAAM,GAAG,CAAC,GAAG,EAAEiC,QAAQ6D,MAAM9F,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAIoE,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGlB,MAAM3D,IAAI,CAAC;QAC1FwE,SAAS,aAAaxB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;QAC3DH,SAAS,cAAcxB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;QAChEJ,SAAS,aAAaxB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC/D;IACA,OAAOlB,MAAMvG,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output keeps them as-is.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every\n// character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.\n// An array (e.g. snippets) joins with the one delimiter our content provably cannot contain,\n// inside the quoted field newlines already trigger (DESIGN.md \"Rendering a list of snippets\").\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : Array.isArray(value) ? value.join('\\n') : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// JSON.stringify throws on BigInt, and stores return int64 as BigInt (DuckDB does): a value\n// that fits a safe integer stays a number, a larger one becomes its decimal string.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nconst bigintSafe = (_key: string, value: unknown): unknown => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n\nexport function stringifyJson(value: unknown, indent?: number): string {\n  return JSON.stringify(value, bigintSafe, indent);\n}\n\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Names from a row wherever there is one: the statement's column list keeps duplicates the\n    // row collapsed. Nothing at all with no row and no columns -- a bare newline is an empty record.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: the result is never held as\n// rows and again as a rendered string. Table buffers (column width depends on the last row); json/csv write row by row.\nexport async function printRowStream(rows: AsyncIterable<Row>, format: RowFormat, columns: string[]): Promise<void> {\n  const iterator = rows[Symbol.asyncIterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = await iterator.next();\n  const rest: AsyncIterable<Row> = { [Symbol.asyncIterator]: () => iterator };\n\n  if (format === 'table') {\n    const collected: Row[] = first.done ? [] : [first.value];\n    for await (const row of rest) collected.push(row);\n    printRows(collected, format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    const indent = (row: Row) =>\n      stringifyJson(row, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for await (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for await (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\n// An array cell (snippets) becomes one physical line per element; every other value stays the\n// single line cell() already produced. No cell in any other command holds one today, so a row\n// with no array-valued cell yields exactly one physical line, unchanged from before this existed.\nfunction cellLines(value: unknown): string[] {\n  if (Array.isArray(value)) return value.length === 0 ? [''] : value.map((v) => cell(v));\n  return [cell(value)];\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return stringifyJson(rows, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const linesByCell = rows.map((row) => columns.map((col) => cellLines(row[col])));\n  const heights = linesByCell.map((cellsInRow) => Math.max(1, ...cellsInRow.map((lines) => lines.length)));\n  const natural = columns.map((col, i) => Math.max(col.length, ...linesByCell.flatMap((cellsInRow) => cellsInRow[i]).map((line) => line.length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  // A row becomes as many physical lines as its tallest cell; other columns blank-pad on\n  // continuation lines (DESIGN.md \"Rendering a list of snippets\").\n  const body: string[] = [];\n  for (let r = 0; r < rows.length; r++) {\n    for (let li = 0; li < heights[r]; li++) body.push(formatRow(columns.map((_, c) => linesByCell[r][c][li] ?? '')));\n  }\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...body].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\n// name, or name:weight when the weight is not the default 1.\nfunction signalLabels(signals: Record<string, number>): string {\n  return Object.entries(signals)\n    .map(([name, weight]) => (weight === 1 ? name : `${name}:${weight}`))\n    .join(', ');\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a preset without the vectors signal says what it uses instead.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)];\n}\n\nexport function renderMap(result: {\n  docs: { count: number; bytes: number };\n  fields: Row[];\n  fieldsTotal: number;\n  features: { on: string[]; off: string[] };\n  presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>;\n  hubs: Row[];\n  recent: Row[];\n  recentCaveat: string | null;\n}): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","Array","isArray","join","test","csvLine","values","map","warnOversize","bytes","rowCount","console","warn","Math","round","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","stringifyJson","indent","JSON","stringify","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","asyncIterator","first","next","rest","collected","done","push","write","process","stdout","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","cellLines","v","width","linesByCell","col","heights","cellsInRow","lines","natural","i","flatMap","clip","w","padEnd","slice","formatRow","trimEnd","body","r","li","_","c","repeat","featureOffHint","featuresLine","features","off","on","signalLabels","signals","entries","weight","presetsLines","presets","p","files","embedded","vectors","renderMap","result","parts","docs","count","fields","fieldsTotal","hubs","includes","recentCaveat","recent","renderPeek","path","tokens","parseError","key","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,sGAAsG;AACtG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,6FAA6F;AAC7F,+FAA+F;AAC/F,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKC,MAAMC,OAAO,CAACP,SAASA,MAAMQ,IAAI,CAAC,QAAQP,OAAOD;IAC3G,OAAO,WAAWS,IAAI,CAACL,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASM,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACT,UAAUK,IAAI,CAAC;AACnC;AAEA,SAASK,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAAShB,gBAAgB;IAC7BkB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,4FAA4F;AAC5F,oFAAoF;AACpF,MAAMK,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,MAAMC,aAAa,CAACC,MAAczB,QAA6B,OAAOA,UAAU,WAAYA,SAAS,CAACoB,YAAYpB,SAASoB,WAAWE,OAAOtB,SAASA,MAAM0B,QAAQ,KAAM1B;AAE1K,OAAO,SAAS2B,cAAc3B,KAAc,EAAE4B,MAAe;IAC3D,OAAOC,KAAKC,SAAS,CAAC9B,OAAOwB,YAAYI;AAC3C;AAEA,OAAO,SAASG,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,2FAA2F;QAC3F,iGAAiG;QACjG,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAAC9B,QAAQyB;eAAWH,KAAKpB,GAAG,CAAC,CAAC6B,MAAQ/B,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASD,GAAG,CAACC,KAAK;SAAI,CAAClC,IAAI,CAAC;QACtGQ,QAAQ2B,GAAG,CAACH;QACZ,yDAAyD;QACzD3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCjB,QAAQ2B,GAAG,CAACH;IACZ3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,+FAA+F;AAC/F,wHAAwH;AACxH,OAAO,eAAeW,eAAef,IAAwB,EAAEC,MAAiB,EAAEC,OAAiB;IACjG,MAAMc,WAAWhB,IAAI,CAACiB,OAAOC,aAAa,CAAC;IAC3C,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMC,QAAQ,MAAMH,SAASI,IAAI;IACjC,MAAMC,OAA2B;QAAE,CAACJ,OAAOC,aAAa,CAAC,EAAE,IAAMF;IAAS;IAE1E,IAAIf,WAAW,SAAS;QACtB,MAAMqB,YAAmBH,MAAMI,IAAI,GAAG,EAAE,GAAG;YAACJ,MAAMnD,KAAK;SAAC;QACxD,WAAW,MAAMyC,OAAOY,KAAMC,UAAUE,IAAI,CAACf;QAC7CV,UAAUuB,WAAWrB,QAAQC;QAC7B;IACF;IAEA,IAAIpB,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM0C,QAAQ,CAACrD;QACb,yDAAyD;QACzDU,SAAS8B,OAAOC,UAAU,CAACzC;QAC3BsD,QAAQC,MAAM,CAACF,KAAK,CAACrD;IACvB;IAEA,IAAI6B,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,MAAML,SAAS,CAACa,MACdd,cAAcc,KAAK,GAChBmB,KAAK,CAAC,MACNhD,GAAG,CAAC,CAACiD,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzBrD,IAAI,CAAC;QACV,IAAI2C,MAAMI,IAAI,EAAE;YACdE,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAM7B,OAAOuB,MAAMnD,KAAK;YACxBe,WAAW;YACX,WAAW,MAAM0B,OAAOY,KAAM;gBAC5BI,MAAM;gBACNA,MAAM7B,OAAOa;gBACb1B;YACF;YACA0C,MAAM;QACR;IACF,OAAO,IAAIN,MAAMI,IAAI,EAAE;QACrB,MAAMpB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGqB,MAAM,GAAG/C,QAAQyB,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACa,MAAMnD,KAAK;QACrCyD,MAAM,GAAG/C,QAAQyB,OAAO,EAAE,CAAC;QAC3BsB,MAAM,GAAG/C,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASS,MAAMnD,KAAK,CAAC0C,KAAK,GAAG,EAAE,CAAC;QAC5D3B,WAAW;QACX,WAAW,MAAM0B,OAAOY,KAAM;YAC5BI,MAAM,GAAG/C,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpD3B;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM+C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAO7B,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMgC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACxD,KAAKyD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,8FAA8F;AAC9F,8FAA8F;AAC9F,kGAAkG;AAClG,SAASU,UAAU5E,KAAc;IAC/B,IAAIM,MAAMC,OAAO,CAACP,QAAQ,OAAOA,MAAMoC,MAAM,KAAK,IAAI;QAAC;KAAG,GAAGpC,MAAMY,GAAG,CAAC,CAACiE,IAAM9E,KAAK8E;IACnF,OAAO;QAAC9E,KAAKC;KAAO;AACtB;AAEA,SAAS8C,WAAWd,IAAW,EAAEC,MAAc,EAAE6C,QAAQpB,QAAQC,MAAM,CAACzB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAON,cAAcK,MAAM;IAClD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAM+C,cAAc/C,KAAKpB,GAAG,CAAC,CAAC6B,MAAQP,QAAQtB,GAAG,CAAC,CAACoE,MAAQJ,UAAUnC,GAAG,CAACuC,IAAI;IAC7E,MAAMC,UAAUF,YAAYnE,GAAG,CAAC,CAACsE,aAAehE,KAAKyD,GAAG,CAAC,MAAMO,WAAWtE,GAAG,CAAC,CAACuE,QAAUA,MAAM/C,MAAM;IACrG,MAAMgD,UAAUlD,QAAQtB,GAAG,CAAC,CAACoE,KAAKK,IAAMnE,KAAKyD,GAAG,CAACK,IAAI5C,MAAM,KAAK2C,YAAYO,OAAO,CAAC,CAACJ,aAAeA,UAAU,CAACG,EAAE,EAAEzE,GAAG,CAAC,CAACiD,OAASA,KAAKzB,MAAM;IAC5I,MAAM6B,SAASa,SAASA,QAAQhB,aAAaE,UAAUoB,SAASN,SAASM;IAEzE,MAAMG,OAAO,CAACvF,OAAewF,IAAexF,MAAMoC,MAAM,IAAIoD,IAAIxF,MAAMyF,MAAM,CAACD,KAAK,GAAGxF,MAAM0F,KAAK,CAAC,GAAGF,IAAI,KAAKzB,WAAW;IACxH,MAAM4B,YAAY,CAAChF,SACjBA,OACGC,GAAG,CAAC,CAACZ,OAAOqF,IAAME,KAAKvF,OAAOiE,MAAM,CAACoB,EAAE,GACvC7E,IAAI,CAAC,MACLoF,OAAO;IAEZ,uFAAuF;IACvF,iEAAiE;IACjE,MAAMC,OAAiB,EAAE;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAI9D,KAAKI,MAAM,EAAE0D,IAAK;QACpC,IAAK,IAAIC,KAAK,GAAGA,KAAKd,OAAO,CAACa,EAAE,EAAEC,KAAMF,KAAKrC,IAAI,CAACmC,UAAUzD,QAAQtB,GAAG,CAAC,CAACoF,GAAGC;gBAAMlB;oBAAAA,sBAAAA,WAAW,CAACe,EAAE,CAACG,EAAE,CAACF,GAAG,cAArBhB,iCAAAA,sBAAyB;;IAC7G;IAEA,OAAO;QAACY,UAAUzD;QAAU+B,OAAOrD,GAAG,CAAC,CAAC4E,IAAM,IAAIU,MAAM,CAACV,IAAIhF,IAAI,CAAC;WAAUqF;KAAK,CAACrF,IAAI,CAAC;AACzF;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAAS2F,eAAezD,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAAS0D,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAAClE,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEiE,SAASC,GAAG,CAAC1F,GAAG,CAAC,CAAC8B,OAAS,GAAGA,KAAK,EAAE,EAAEyD,eAAezD,MAAM,CAAC,CAAC,EAAElC,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE6F,SAASE,EAAE,CAAC/F,IAAI,CAAC,QAAQ8F,KAAK;AACpD;AAEA,6DAA6D;AAC7D,SAASE,aAAaC,OAA+B;IACnD,OAAOpE,OAAOqE,OAAO,CAACD,SACnB7F,GAAG,CAAC,CAAC,CAAC8B,MAAMiE,OAAO,GAAMA,WAAW,IAAIjE,OAAO,GAAGA,KAAK,CAAC,EAAEiE,QAAQ,EAClEnG,IAAI,CAAC;AACV;AAEA,OAAO,SAASoG,aAAaC,OAAkG;IAC7H,6FAA6F;IAC7F,8EAA8E;IAC9E,OAAO;QAAC;WAAeA,QAAQjG,GAAG,CAAC,CAACkG,IAAM,CAAC,EAAE,EAAEA,EAAEpE,IAAI,CAAC,EAAE,EAAEoE,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEL,OAAO,CAACQ,OAAO,KAAK5G,YAAY,KAAK,CAAC,WAAW,EAAEmG,aAAaM,EAAEL,OAAO,EAAE,CAAC,CAAC,EAAE;KAAE;AACzL;AAEA,OAAO,SAASS,UAAUC,MASzB;IACC,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEpG,KAAKC,KAAK,CAACgG,OAAOE,IAAI,CAACvG,KAAK,GAAG,MAAM,IAAI,CAAC;QAAEsF,aAAae,OAAOd,QAAQ;QAAG;WAAOO,aAAaO,OAAON,OAAO;QAAG;QAAI/D,WAAWqE,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACnF,MAAM,EAAEgF,MAAM5D,IAAI,CAAC,CAAC,EAAE,EAAE2D,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACnF,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI+E,OAAOM,IAAI,CAACrF,MAAM,GAAG,GAAGgF,MAAM5D,IAAI,CAAC,0BAA0BV,WAAWqE,OAAOM,IAAI,EAAE;SACpF,IAAIN,OAAOd,QAAQ,CAACC,GAAG,CAACoB,QAAQ,CAAC,SAASN,MAAM5D,IAAI,CAAC,CAAC,aAAa,EAAE2C,eAAe,QAAQ,CAAC,CAAC;IACnGiB,MAAM5D,IAAI,CAAC;IACX,IAAI2D,OAAOQ,YAAY,EAAEP,MAAM5D,IAAI,CAAC2D,OAAOQ,YAAY;IACvDP,MAAM5D,IAAI,CAACV,WAAWqE,OAAOS,MAAM,EAAE;IACrC,OAAOR,MAAM5G,IAAI,CAAC;AACpB;AAEA,OAAO,SAASqH,WAAWV,MAAoR;IAC7S,MAAMhC,QAAQ;QAAC,GAAGgC,OAAOW,IAAI,CAAC,IAAI,EAAEX,OAAOY,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAIZ,OAAOa,UAAU,EAAE7C,MAAM3B,IAAI,CAAC,CAAC,wDAAwD,EAAE2D,OAAOa,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAKjI,MAAM,IAAIqC,OAAOqE,OAAO,CAACS,OAAOe,WAAW,EAAG/C,MAAM3B,IAAI,CAAC,CAAC,EAAE,EAAEyE,IAAI,EAAE,EAAEjI,OAAO;IAC9F,IAAImH,OAAOgB,QAAQ,CAAC/F,MAAM,GAAG,GAAG;QAC9B+C,MAAM3B,IAAI,CAAC,IAAI;QACf,KAAK,MAAM4E,KAAKjB,OAAOgB,QAAQ,CAAEhD,MAAM3B,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI0C,MAAM,CAACkC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAIZ,OAAOsB,aAAa,GAAGtB,OAAOgB,QAAQ,CAAC/F,MAAM,EAAE+C,MAAM3B,IAAI,CAAC,CAAC,IAAI,EAAE2D,OAAOsB,aAAa,GAAGtB,OAAOgB,QAAQ,CAAC/F,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAI+E,OAAOb,GAAG,CAACoB,QAAQ,CAAC,aAAavC,MAAM3B,IAAI,CAAC,IAAI;IAC3D,IAAI2D,OAAOb,GAAG,CAACoB,QAAQ,CAAC,UAAU;QAChCvC,MAAM3B,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMkF,WAAW,CAACC,OAAeC,OAAiBvE;YAChD,IAAIA,QAAQ,GAAGc,MAAM3B,IAAI,CAAC,GAAGmF,MAAM,EAAE,EAAEtE,MAAM,GAAG,EAAEuE,MAAMpI,IAAI,CAAC,QAAQ6D,QAAQuE,MAAMxG,MAAM,GAAG,CAAC,GAAG,EAAEiC,QAAQuE,MAAMxG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAI+E,OAAO0B,aAAa,GAAG1B,OAAO2B,eAAe,GAAG3B,OAAO4B,cAAc,GAAG,GAAG5D,MAAM3B,IAAI,CAAC;QAC1FkF,SAAS,aAAavB,OAAO6B,QAAQ,EAAE7B,OAAO0B,aAAa;QAC3DH,SAAS,cAAcvB,OAAO8B,UAAU,EAAE9B,OAAO2B,eAAe;QAChEJ,SAAS,aAAavB,OAAO+B,SAAS,EAAE/B,OAAO4B,cAAc;IAC/D;IACA,OAAO5D,MAAM3E,IAAI,CAAC;AACpB"}

package/dist/esm/output/search-error.d.ts

@@ -1 +1,2 @@
+export declare function bareTermSyntaxError(terms: string): Error | null;
 export declare function searchError(err: Error, terms: string, scope?: string): Error;

package/dist/esm/output/search-error.js

@@ -1,7 +1,23 @@
 import { SenseError } from '../errors.js';
+import { hasUnspacedRun } from '../text/segment.js';
 // FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and
 // errors `no such column: to` -- true about the parse, misleading about the input.
-const FTS5_PUNCTUATION = /[^\p{L}\p{N}_\s]/u;
+const FTS5_PUNCTUATION = /[^\p{L}\p{M}\p{N}_\s]/u;
+// SQLite treats punctuation in an unquoted spaced-script token as MATCH syntax. Validate that
+// boundary before dispatch so DuckDB and Tantivy do not silently interpret the same input as text.
+export function bareTermSyntaxError(terms) {
+    var _outsideQuotes_match;
+    const outsideQuotes = terms.replace(/"[^"]*"/g, ' ');
+    const suspects = ((_outsideQuotes_match = outsideQuotes.match(/\S+/g)) !== null && _outsideQuotes_match !== void 0 ? _outsideQuotes_match : []).filter((token)=>{
+        if (!FTS5_PUNCTUATION.test(token) || hasUnspacedRun(token)) return false;
+        if (/^[\p{L}_]\w*\s*:/u.test(token)) return false;
+        if (/^[\p{L}\p{N}_]+\*(?:\s|$)/u.test(token)) return false;
+        if (/^(?:AND|OR|NOT)$/u.test(token)) return false;
+        return true;
+    });
+    if (suspects.length === 0) return null;
+    return new SenseError('SEARCH_SYNTAX', `the punctuation in ${suspects.map((term)=>`\`${term}\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '"${suspects[0]}"'. Searchable columns are title, summary, text.`);
+}
 export function searchError(err, terms, scope) {
     var _terms_match;
     var _exec;

package/dist/esm/output/search-error.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/search-error.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\n\n// FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and\n// errors `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_PUNCTUATION = /[^\\p{L}\\p{N}_\\s]/u;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_PUNCTUATION.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where can raise \"no such column\" through this same statement.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","FTS5_PUNCTUATION","searchError","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","map","join"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAE1C,0FAA0F;AAC1F,mFAAmF;AACnF,MAAMC,mBAAmB;AAEzB,OAAO,SAASC,YAAYC,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAGN;IALZ,MAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,MAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQX,iBAAiBM,IAAI,CAACI;IACvG,wFAAwF;IACxF,qEAAqE;IACrE,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,CAACP,IAAMA,EAAEQ,KAAK,CAAC,mBAAmBC,QAAQ,CAACP;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIjB,WAAW,iBAAiB,GAAGM,QAAQ,uBAAuB,EAAEE,SAASa,GAAG,CAAC,CAACV,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEW,IAAI,CAAC,MAAM,gFAAgF,EAAEd,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIhB,WAAW,iBAAiB,GAAGM,QAAQ,yBAAyB,EAAED,MAAM,iHAAiH,CAAC;IACvM;IACA,OAAO,IAAIL,WAAW,iBAAiB,GAAGM,QAAQ,4JAA4J,CAAC;AACjN"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/search-error.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { hasUnspacedRun } from '../text/segment.ts';\n\n// FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and\n// errors `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_PUNCTUATION = /[^\\p{L}\\p{M}\\p{N}_\\s]/u;\n\n// SQLite treats punctuation in an unquoted spaced-script token as MATCH syntax. Validate that\n// boundary before dispatch so DuckDB and Tantivy do not silently interpret the same input as text.\nexport function bareTermSyntaxError(terms: string): Error | null {\n  const outsideQuotes = terms.replace(/\"[^\"]*\"/g, ' ');\n  const suspects = (outsideQuotes.match(/\\S+/g) ?? []).filter((token) => {\n    if (!FTS5_PUNCTUATION.test(token) || hasUnspacedRun(token)) return false;\n    if (/^[\\p{L}_]\\w*\\s*:/u.test(token)) return false;\n    if (/^[\\p{L}\\p{N}_]+\\*(?:\\s|$)/u.test(token)) return false;\n    if (/^(?:AND|OR|NOT)$/u.test(token)) return false;\n    return true;\n  });\n  if (suspects.length === 0) return null;\n  return new SenseError('SEARCH_SYNTAX', `the punctuation in ${suspects.map((term) => `\\`${term}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n}\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_PUNCTUATION.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where can raise \"no such column\" through this same statement.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","hasUnspacedRun","FTS5_PUNCTUATION","bareTermSyntaxError","terms","outsideQuotes","replace","suspects","match","filter","token","test","length","map","term","join","searchError","err","scope","message","t","startsWith","col","exec","fromTerms","undefined","some","split","includes"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,qBAAqB;AAEpD,0FAA0F;AAC1F,mFAAmF;AACnF,MAAMC,mBAAmB;AAEzB,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,SAASC,oBAAoBC,KAAa;QAE7BC;IADlB,MAAMA,gBAAgBD,MAAME,OAAO,CAAC,YAAY;IAChD,MAAMC,WAAW,EAACF,uBAAAA,cAAcG,KAAK,CAAC,qBAApBH,kCAAAA,uBAA+B,EAAE,EAAEI,MAAM,CAAC,CAACC;QAC3D,IAAI,CAACR,iBAAiBS,IAAI,CAACD,UAAUT,eAAeS,QAAQ,OAAO;QACnE,IAAI,oBAAoBC,IAAI,CAACD,QAAQ,OAAO;QAC5C,IAAI,6BAA6BC,IAAI,CAACD,QAAQ,OAAO;QACrD,IAAI,oBAAoBC,IAAI,CAACD,QAAQ,OAAO;QAC5C,OAAO;IACT;IACA,IAAIH,SAASK,MAAM,KAAK,GAAG,OAAO;IAClC,OAAO,IAAIZ,WAAW,iBAAiB,CAAC,mBAAmB,EAAEO,SAASM,GAAG,CAAC,CAACC,OAAS,CAAC,EAAE,EAAEA,KAAK,EAAE,CAAC,EAAEC,IAAI,CAAC,MAAM,gFAAgF,EAAER,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;AAC/P;AAEA,OAAO,SAASS,YAAYC,GAAU,EAAEb,KAAa,EAAEc,KAAc;QAGjDd;QAGN;IALZ,MAAMe,UAAUF,IAAIE,OAAO;IAC3B,IAAI,CAAC,oDAAoDR,IAAI,CAACQ,UAAU,OAAOF;IAC/E,MAAMV,WAAW,EAACH,eAAAA,MAAMI,KAAK,CAAC,qBAAZJ,0BAAAA,eAAuB,EAAE,EAAEK,MAAM,CAAC,CAACW,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQnB,iBAAiBS,IAAI,CAACS;IACvG,wFAAwF;IACxF,qEAAqE;IACrE,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACJ,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMK,YAAYF,QAAQG,YAAYlB,SAASK,MAAM,GAAG,IAAIL,SAASmB,IAAI,CAAC,CAACN,IAAMA,EAAEO,KAAK,CAAC,mBAAmBC,QAAQ,CAACN;IACrH,IAAIE,aAAajB,SAASK,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIZ,WAAW,iBAAiB,GAAGmB,QAAQ,uBAAuB,EAAEZ,SAASM,GAAG,CAAC,CAACO,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEL,IAAI,CAAC,MAAM,gFAAgF,EAAER,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIe,QAAQG,aAAaP,UAAUO,WAAW;QAC5C,OAAO,IAAIzB,WAAW,iBAAiB,GAAGmB,QAAQ,yBAAyB,EAAED,MAAM,iHAAiH,CAAC;IACvM;IACA,OAAO,IAAIlB,WAAW,iBAAiB,GAAGmB,QAAQ,4JAA4J,CAAC;AACjN"}

package/dist/esm/scan/frontmatter.js

@@ -92,7 +92,7 @@ function warnStringifiedKeys(relPath, doc, warnings) {
     if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);
 }
 // Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.
-// yaml's message continues onto a source excerpt, so the first line is the sentence, minus the colon that introduced the part being dropped.
+// yaml's message continues onto a source snippet, so the first line is the sentence, minus the colon that introduced the part being dropped.
 function firstLine(message) {
     return message.split('\n')[0].replace(/:\s*$/, '');
 }

package/dist/esm/scan/frontmatter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source excerpt, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","countLines","s","n","i","charCodeAt","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,qGAAqG;AACrG,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gGAAgG;AAChG,qDAAqD;AACrD,OAAO,SAAS6B,WAAWC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEb,MAAM,EAAEe,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAEA,gEAAgE;AAChE,OAAO,SAASG,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIjB,KAAK,CAACkB,IAAI,CAAC,EAAE,CAACnB,MAAM;IACrC,MAAMwB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKzC,WAAW,OAAO;QAAEqC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKtB,KAAK,CAAC,GAAGuB,MAAMC,KAAK;QAAGH,MAAMC,KAAKtB,KAAK,CAACuB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACxB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAAS0B,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCpD,MAAMkD,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACzD,aAAayD,KAAKC,GAAG,GAAG,OAAOlD;YACpC8C,QAAQ;YACR,OAAOpD,MAAMyD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAACrD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASsD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMnD,cAAc4C,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAChE,oBAAoBiE,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQ1C,MAAM,GAAG,GAAG;QACtB,MAAMgD,SAASN,QAAQ1C,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE0C,QAAQ1C,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMiD,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASlE,WAAW,OAAO;QAAEkE,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source snippet, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","countLines","s","n","i","charCodeAt","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,qGAAqG;AACrG,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gGAAgG;AAChG,qDAAqD;AACrD,OAAO,SAAS6B,WAAWC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEb,MAAM,EAAEe,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAEA,gEAAgE;AAChE,OAAO,SAASG,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIjB,KAAK,CAACkB,IAAI,CAAC,EAAE,CAACnB,MAAM;IACrC,MAAMwB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKzC,WAAW,OAAO;QAAEqC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKtB,KAAK,CAAC,GAAGuB,MAAMC,KAAK;QAAGH,MAAMC,KAAKtB,KAAK,CAACuB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACxB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAAS0B,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCpD,MAAMkD,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACzD,aAAayD,KAAKC,GAAG,GAAG,OAAOlD;YACpC8C,QAAQ;YACR,OAAOpD,MAAMyD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAACrD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASsD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMnD,cAAc4C,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAChE,oBAAoBiE,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQ1C,MAAM,GAAG,GAAG;QACtB,MAAMgD,SAASN,QAAQ1C,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE0C,QAAQ1C,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMiD,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASlE,WAAW,OAAO;QAAEkE,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}

package/dist/esm/scan/index.js

@@ -6,6 +6,7 @@ import { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_
 export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
 export { listFiles, toPosixPath } from './list.js';
 export function parseFile(file, extractors = [], cfg) {
+    // Preserve the authored bytes: feature extractors and section offsets use the raw document.
     const raw = readFileSync(file.absPath, 'utf8');
     const warnings = [];
     const { fm, body: content } = splitFrontmatter(raw);

package/dist/esm/scan/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors and section offsets use the raw document.\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,4FAA4F;IAC5F,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}

package/dist/esm/store/batch.d.ts

@@ -0,0 +1,4 @@
+export declare function rewriteInsert(sql: string, rowCount: number): {
+    sql: string;
+    width: number;
+} | null;

package/dist/esm/store/batch.js

@@ -0,0 +1,21 @@
+// Rewrites a single-row parameterized INSERT into one multi-row statement. Engine-neutral string
+// work, shared by duckdb (batch.ts's rewriteBatch) and turso (turso/connection.ts's runBatch).
+const INSERT_RE = /^(INSERT\s+(?:OR\s+\w+\s+)?INTO\s+\S+\s*\([^)]*\)\s*VALUES\s*)\(([^()]*)\)(.*)$/is;
+function placeholderCount(tuple) {
+    var _tuple_match;
+    return ((_tuple_match = tuple.match(/\?/g)) !== null && _tuple_match !== void 0 ? _tuple_match : []).length;
+}
+export function rewriteInsert(sql, rowCount) {
+    const m = INSERT_RE.exec(sql.trim());
+    if (!m) return null;
+    const [, head, tuple, tail] = m;
+    const width = placeholderCount(tuple);
+    if (width === 0) return null;
+    const rows = Array.from({
+        length: rowCount
+    }, ()=>`(${tuple})`).join(', ');
+    return {
+        sql: `${head}${rows}${tail}`,
+        width
+    };
+}

package/dist/esm/store/batch.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/batch.ts"],"sourcesContent":["// Rewrites a single-row parameterized INSERT into one multi-row statement. Engine-neutral string\n// work, shared by duckdb (batch.ts's rewriteBatch) and turso (turso/connection.ts's runBatch).\n\nconst INSERT_RE = /^(INSERT\\s+(?:OR\\s+\\w+\\s+)?INTO\\s+\\S+\\s*\\([^)]*\\)\\s*VALUES\\s*)\\(([^()]*)\\)(.*)$/is;\n\nfunction placeholderCount(tuple: string): number {\n  return (tuple.match(/\\?/g) ?? []).length;\n}\n\nexport function rewriteInsert(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = INSERT_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, head, tuple, tail] = m;\n  const width = placeholderCount(tuple);\n  if (width === 0) return null;\n  const rows = Array.from({ length: rowCount }, () => `(${tuple})`).join(', ');\n  return { sql: `${head}${rows}${tail}`, width };\n}\n"],"names":["INSERT_RE","placeholderCount","tuple","match","length","rewriteInsert","sql","rowCount","m","exec","trim","head","tail","width","rows","Array","from","join"],"mappings":"AAAA,iGAAiG;AACjG,+FAA+F;AAE/F,MAAMA,YAAY;AAElB,SAASC,iBAAiBC,KAAa;QAC7BA;IAAR,OAAO,EAACA,eAAAA,MAAMC,KAAK,CAAC,oBAAZD,0BAAAA,eAAsB,EAAE,EAAEE,MAAM;AAC1C;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAEC,QAAgB;IACzD,MAAMC,IAAIR,UAAUS,IAAI,CAACH,IAAII,IAAI;IACjC,IAAI,CAACF,GAAG,OAAO;IACf,MAAM,GAAGG,MAAMT,OAAOU,KAAK,GAAGJ;IAC9B,MAAMK,QAAQZ,iBAAiBC;IAC/B,IAAIW,UAAU,GAAG,OAAO;IACxB,MAAMC,OAAOC,MAAMC,IAAI,CAAC;QAAEZ,QAAQG;IAAS,GAAG,IAAM,CAAC,CAAC,EAAEL,MAAM,CAAC,CAAC,EAAEe,IAAI,CAAC;IACvE,OAAO;QAAEX,KAAK,GAAGK,OAAOG,OAAOF,MAAM;QAAEC;IAAM;AAC/C"}

package/dist/esm/store/duckdb/batch.d.ts

@@ -1,7 +1,3 @@
-export declare function rewriteInsert(sql: string, rowCount: number): {
-    sql: string;
-    width: number;
-} | null;
 export declare function rewriteUpdate(sql: string, rowCount: number): {
     sql: string;
     width: number;

package/dist/esm/store/duckdb/batch.js

@@ -1,12 +1,9 @@
 // Rewrites the store's single-row parameterized INSERT/UPDATE/DELETE into one multi-row statement --
 // DuckDB's columnar engine pays for row-at-a-time writes, not just the round trip (PRINCIPLES: documented-means-tested).
-const INSERT_RE = /^(INSERT\s+(?:OR\s+\w+\s+)?INTO\s+\S+\s*\([^)]*\)\s*VALUES\s*)\(([^()]*)\)(.*)$/is;
+// rewriteInsert itself lives in the shared ../batch.ts: turso folds INSERTs the same way (3.36 B).
+import { rewriteInsert } from '../batch.js';
 const UPDATE_RE = /^UPDATE\s+(\S+)\s+SET\s+(.+?)\s+WHERE\s+(.+)$/is;
 const DELETE_RE = /^DELETE\s+FROM\s+(\S+)\s+WHERE\s+(.+)$/is;
-function placeholderCount(tuple) {
-    var _tuple_match;
-    return ((_tuple_match = tuple.match(/\?/g)) !== null && _tuple_match !== void 0 ? _tuple_match : []).length;
-}
 // "a = ?, b = ?" or "a = ? AND b = ?" -> ["a", "b"]. A clause that doesn't fit yields a shorter
 // list, which rewriteUpdate/rewriteDelete reject as a width mismatch.
 function equalityColumns(clause, sep) {
@@ -27,20 +24,6 @@ function tuples(rowCount, width) {
         length: rowCount
     }, ()=>row).join(', ');
 }
-export function rewriteInsert(sql, rowCount) {
-    const m = INSERT_RE.exec(sql.trim());
-    if (!m) return null;
-    const [, head, tuple, tail] = m;
-    const width = placeholderCount(tuple);
-    if (width === 0) return null;
-    const rows = Array.from({
-        length: rowCount
-    }, ()=>`(${tuple})`).join(', ');
-    return {
-        sql: `${head}${rows}${tail}`,
-        width
-    };
-}
 export function rewriteUpdate(sql, rowCount) {
     const m = UPDATE_RE.exec(sql.trim());
     if (!m) return null;