sensemaking 0.15.3 → 0.16.0

package/dist/esm/output.d.ts

@@ -12,7 +12,7 @@ export declare function presetsLines(presets: Array<{
     name: string;
     files: number;
     embedded: number;
-    semantic?: boolean;
+    signals: Record<string, number>;
 }>): string[];
 export declare function renderMap(result: {
     docs: {
@@ -29,7 +29,7 @@ export declare function renderMap(result: {
         name: string;
         files: number;
         embedded: number;
-        semantic?: boolean;
+        signals: Record<string, number>;
     }>;
     hubs: Row[];
     recent: Row[];

package/dist/esm/output.js

@@ -154,12 +154,16 @@ export function featuresLine(features) {
     const off = features.off.length > 0 ? ` · off: ${features.off.map((name)=>`${name} (${featureOffHint(name)})`).join(', ')}` : '';
     return `features: ${features.on.join(', ')}${off}`;
 }
+// name, or name:weight when the weight is not the default 1.
+function signalLabels(signals) {
+    return Object.entries(signals).map(([name, weight])=>weight === 1 ? name : `${name}:${weight}`).join(', ');
+}
 export function presetsLines(presets) {
     // "0 embedded" is ambiguous between a scope that declined vectors and one that has not built
-    // them yet, so a semantic-off preset says which.
+    // them yet, so a preset without the vectors signal says what it uses instead.
     return [
         'presets:',
-        ...presets.map((p)=>`  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.semantic === false ? ' (semantic: false)' : ''}`)
+        ...presets.map((p)=>`  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)
     ];
 }
 export function renderMap(result) {

package/dist/esm/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, and deliberately not cell(): csv is the format for redirecting a result to a\n// file, so it keeps every character, embedded newlines included. What it cannot carry is\n// NULL vs empty string, or a value's type -- that is what json is for.\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// Header names come from a row wherever there is one: a statement's column list keeps\n// duplicates that the row object has already collapsed, so `SELECT a AS x, b AS x` would print\n// b's value under both headers. `columns` is the fallback that labels a 0-row result, and with\n// no rows and no statement (a bounded command that found nothing) csv writes nothing at all --\n// a bare newline would read to a csv parser as one empty record.\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Collapsed the same way a row object collapses them, so a statement naming a column\n    // twice describes itself identically whether it matched anything or not.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: rows arrive from an\n// iterator, so a large result is never held here as rows and again as a rendered string.\n// Table buffers by necessity (a column is as wide as its widest cell, which the last row can\n// change); json and csv write row by row. The writes are synchronous, so when stdout is a pipe\n// Node buffers what the reader has not taken yet -- the saving is this module's copies, not\n// the operating system's.\nexport function printRowStream(rows: Iterable<Row>, format: RowFormat, columns: string[]): void {\n  const iterator = rows[Symbol.iterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = iterator.next();\n  const rest: Iterable<Row> = { [Symbol.iterator]: () => iterator };\n\n  if (format === 'table') {\n    printRows(first.done ? [] : [first.value, ...rest], format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    // JSON.stringify throws on BigInt, so a row with an int64 past 2^53 (setReadBigInts) is\n    // converted via the replacer: a value that fits a safe integer stays a number, a larger one\n    // becomes its decimal string, since a JSON number cannot carry it losslessly.\n    const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\n    const bigintSafe = (_key: string, value: unknown) => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n    const indent = (row: Row) =>\n      JSON.stringify(row, bigintSafe, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; semantic?: boolean }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a semantic-off preset says which.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.semantic === false ? ' (semantic: false)' : ''}`)];\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; presets: Array<{ name: string; files: number; embedded: number; semantic?: boolean }>; hubs: Row[]; recent: Row[]; recentCaveat: string | null }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","test","csvLine","values","map","join","warnOversize","bytes","rowCount","console","warn","Math","round","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","first","next","rest","done","write","process","stdout","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","indent","JSON","stringify","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","cells","col","natural","i","clip","w","padEnd","slice","formatRow","trimEnd","repeat","featureOffHint","featuresLine","features","off","on","presetsLines","presets","p","files","embedded","semantic","renderMap","result","parts","docs","count","fields","fieldsTotal","push","hubs","includes","recentCaveat","recent","renderPeek","lines","path","tokens","parseError","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,yFAAyF;AACzF,yFAAyF;AACzF,uEAAuE;AACvE,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKJ,OAAOD;IACjE,OAAO,WAAWM,IAAI,CAACF,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASG,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACN,UAAUO,IAAI,CAAC;AACnC;AAEA,SAASC,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAASd,gBAAgB;IAC7BgB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,sFAAsF;AACtF,+FAA+F;AAC/F,+FAA+F;AAC/F,+FAA+F;AAC/F,iEAAiE;AACjE,OAAO,SAASK,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,qFAAqF;QACrF,yEAAyE;QACzE,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAACpB,QAAQe;eAAWH,KAAKV,GAAG,CAAC,CAACmB,MAAQrB,QAAQe,MAAMb,GAAG,CAAC,CAACoB,OAASD,GAAG,CAACC,KAAK;SAAI,CAACnB,IAAI,CAAC;QACtGI,QAAQgB,GAAG,CAACH;QACZ,yDAAyD;QACzDhB,aAAaoB,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCN,QAAQgB,GAAG,CAACH;IACZhB,aAAaoB,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,uFAAuF;AACvF,yFAAyF;AACzF,6FAA6F;AAC7F,+FAA+F;AAC/F,4FAA4F;AAC5F,0BAA0B;AAC1B,OAAO,SAASW,eAAef,IAAmB,EAAEC,MAAiB,EAAEC,OAAiB;IACtF,MAAMc,WAAWhB,IAAI,CAACiB,OAAOD,QAAQ,CAAC;IACtC,wFAAwF;IACxF,2FAA2F;IAC3F,MAAME,QAAQF,SAASG,IAAI;IAC3B,MAAMC,OAAsB;QAAE,CAACH,OAAOD,QAAQ,CAAC,EAAE,IAAMA;IAAS;IAEhE,IAAIf,WAAW,SAAS;QACtBF,UAAUmB,MAAMG,IAAI,GAAG,EAAE,GAAG;YAACH,MAAMrC,KAAK;eAAKuC;SAAK,EAAEnB,QAAQC;QAC5D;IACF;IAEA,IAAIT,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM4B,QAAQ,CAACrC;QACb,yDAAyD;QACzDQ,SAASmB,OAAOC,UAAU,CAAC5B;QAC3BsC,QAAQC,MAAM,CAACF,KAAK,CAACrC;IACvB;IAEA,IAAIgB,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,wFAAwF;QACxF,4FAA4F;QAC5F,8EAA8E;QAC9E,MAAMwB,WAAWC,OAAOC,OAAOC,gBAAgB;QAC/C,MAAMC,aAAa,CAACC,MAAcjD,QAAoB,OAAOA,UAAU,WAAYA,SAAS,CAAC4C,YAAY5C,SAAS4C,WAAWE,OAAO9C,SAASA,MAAMkD,QAAQ,KAAMlD;QACjK,MAAMmD,SAAS,CAACvB,MACdwB,KAAKC,SAAS,CAACzB,KAAKoB,YAAY,GAC7BM,KAAK,CAAC,MACN7C,GAAG,CAAC,CAAC8C,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzB7C,IAAI,CAAC;QACV,IAAI2B,MAAMG,IAAI,EAAE;YACdC,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAMU,OAAOd,MAAMrC,KAAK;YACxBa,WAAW;YACX,KAAK,MAAMe,OAAOW,KAAM;gBACtBE,MAAM;gBACNA,MAAMU,OAAOvB;gBACbf;YACF;YACA4B,MAAM;QACR;IACF,OAAO,IAAIJ,MAAMG,IAAI,EAAE;QACrB,MAAMlB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGkB,MAAM,GAAGlC,QAAQe,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACY,MAAMrC,KAAK;QACrCyC,MAAM,GAAGlC,QAAQe,OAAO,EAAE,CAAC;QAC3BmB,MAAM,GAAGlC,QAAQe,MAAMb,GAAG,CAAC,CAACoB,OAASQ,MAAMrC,KAAK,CAAC6B,KAAK,GAAG,EAAE,CAAC;QAC5DhB,WAAW;QACX,KAAK,MAAMe,OAAOW,KAAM;YACtBE,MAAM,GAAGlC,QAAQe,MAAMb,GAAG,CAAC,CAACoB,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpDhB;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM2C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAOpC,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMuC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACpD,KAAKqD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGnD,KAAKqD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGnD,KAAKqD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAAS3B,WAAWd,IAAW,EAAEC,MAAc,EAAEkD,QAAQ5B,QAAQC,MAAM,CAACtB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAOgC,KAAKC,SAAS,CAAClC,MAAM,MAAM;IACzD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAMoD,QAAQpD,KAAKV,GAAG,CAAC,CAACmB,MAAQP,QAAQZ,GAAG,CAAC,CAAC+D,MAAQzE,KAAK6B,GAAG,CAAC4C,IAAI;IAClE,MAAMC,UAAUpD,QAAQZ,GAAG,CAAC,CAAC+D,KAAKE,IAAM1D,KAAKqD,GAAG,CAACG,IAAIjD,MAAM,KAAKgD,MAAM9D,GAAG,CAAC,CAACmB,MAAQA,GAAG,CAAC8C,EAAE,CAACnD,MAAM;IAChG,MAAMoC,SAASW,SAASA,QAAQd,aAAaE,UAAUe,SAASH,SAASG;IAEzE,MAAME,OAAO,CAAC3E,OAAe4E,IAAe5E,MAAMuB,MAAM,IAAIqD,IAAI5E,MAAM6E,MAAM,CAACD,KAAK,GAAG5E,MAAM8E,KAAK,CAAC,GAAGF,IAAI,KAAKnB,WAAW;IACxH,MAAMsB,YAAY,CAACvE,SACjBA,OACGC,GAAG,CAAC,CAACT,OAAO0E,IAAMC,KAAK3E,OAAO2D,MAAM,CAACe,EAAE,GACvChE,IAAI,CAAC,MACLsE,OAAO;IAEZ,OAAO;QAACD,UAAU1D;QAAUsC,OAAOlD,GAAG,CAAC,CAACmE,IAAM,IAAIK,MAAM,CAACL,IAAIlE,IAAI,CAAC;WAAU6D,MAAM9D,GAAG,CAACsE;KAAW,CAACrE,IAAI,CAAC;AACzG;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAASwE,eAAerD,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAASsD,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAAC9D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE6D,SAASC,GAAG,CAAC5E,GAAG,CAAC,CAACoB,OAAS,GAAGA,KAAK,EAAE,EAAEqD,eAAerD,MAAM,CAAC,CAAC,EAAEnB,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE0E,SAASE,EAAE,CAAC5E,IAAI,CAAC,QAAQ2E,KAAK;AACpD;AAEA,OAAO,SAASE,aAAaC,OAAqF;IAChH,6FAA6F;IAC7F,iDAAiD;IACjD,OAAO;QAAC;WAAeA,QAAQ/E,GAAG,CAAC,CAACgF,IAAM,CAAC,EAAE,EAAEA,EAAE5D,IAAI,CAAC,EAAE,EAAE4D,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEG,QAAQ,KAAK,QAAQ,uBAAuB,IAAI;KAAE;AAC1J;AAEA,OAAO,SAASC,UAAUC,MAAiR;IACzS,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEjF,KAAKC,KAAK,CAAC6E,OAAOE,IAAI,CAACpF,KAAK,GAAG,MAAM,IAAI,CAAC;QAAEuE,aAAaW,OAAOV,QAAQ;QAAG;WAAOG,aAAaO,OAAON,OAAO;QAAG;QAAIvD,WAAW6D,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAAC3E,MAAM,EAAEwE,MAAMK,IAAI,CAAC,CAAC,EAAE,EAAEN,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAAC3E,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIuE,OAAOO,IAAI,CAAC9E,MAAM,GAAG,GAAGwE,MAAMK,IAAI,CAAC,0BAA0BnE,WAAW6D,OAAOO,IAAI,EAAE;SACpF,IAAIP,OAAOV,QAAQ,CAACC,GAAG,CAACiB,QAAQ,CAAC,SAASP,MAAMK,IAAI,CAAC,CAAC,aAAa,EAAElB,eAAe,QAAQ,CAAC,CAAC;IACnGa,MAAMK,IAAI,CAAC;IACX,IAAIN,OAAOS,YAAY,EAAER,MAAMK,IAAI,CAACN,OAAOS,YAAY;IACvDR,MAAMK,IAAI,CAACnE,WAAW6D,OAAOU,MAAM,EAAE;IACrC,OAAOT,MAAMrF,IAAI,CAAC;AACpB;AAEA,OAAO,SAAS+F,WAAWX,MAAoR;IAC7S,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAId,OAAOe,UAAU,EAAEH,MAAMN,IAAI,CAAC,CAAC,wDAAwD,EAAEN,OAAOe,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAK9G,MAAM,IAAIwB,OAAOuF,OAAO,CAACjB,OAAOkB,WAAW,EAAGN,MAAMN,IAAI,CAAC,CAAC,EAAE,EAAEU,IAAI,EAAE,EAAE9G,OAAO;IAC9F,IAAI8F,OAAOmB,QAAQ,CAAC1F,MAAM,GAAG,GAAG;QAC9BmF,MAAMN,IAAI,CAAC,IAAI;QACf,KAAK,MAAMc,KAAKpB,OAAOmB,QAAQ,CAAEP,MAAMN,IAAI,CAAC,CAAC,EAAE,EAAE,IAAInB,MAAM,CAACiC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEN,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAId,OAAOyB,aAAa,GAAGzB,OAAOmB,QAAQ,CAAC1F,MAAM,EAAEmF,MAAMN,IAAI,CAAC,CAAC,IAAI,EAAEN,OAAOyB,aAAa,GAAGzB,OAAOmB,QAAQ,CAAC1F,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAIuE,OAAOT,GAAG,CAACiB,QAAQ,CAAC,aAAaI,MAAMN,IAAI,CAAC,IAAI;IAC3D,IAAIN,OAAOT,GAAG,CAACiB,QAAQ,CAAC,UAAU;QAChCI,MAAMN,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMoB,WAAW,CAACC,OAAeC,OAAiB3D;YAChD,IAAIA,QAAQ,GAAG2C,MAAMN,IAAI,CAAC,GAAGqB,MAAM,EAAE,EAAE1D,MAAM,GAAG,EAAE2D,MAAMhH,IAAI,CAAC,QAAQqD,QAAQ2D,MAAMnG,MAAM,GAAG,CAAC,GAAG,EAAEwC,QAAQ2D,MAAMnG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAIuE,OAAO6B,aAAa,GAAG7B,OAAO8B,eAAe,GAAG9B,OAAO+B,cAAc,GAAG,GAAGnB,MAAMN,IAAI,CAAC;QAC1FoB,SAAS,aAAa1B,OAAOgC,QAAQ,EAAEhC,OAAO6B,aAAa;QAC3DH,SAAS,cAAc1B,OAAOiC,UAAU,EAAEjC,OAAO8B,eAAe;QAChEJ,SAAS,aAAa1B,OAAOkC,SAAS,EAAElC,OAAO+B,cAAc;IAC/D;IACA,OAAOnB,MAAMhG,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, and deliberately not cell(): csv is the format for redirecting a result to a\n// file, so it keeps every character, embedded newlines included. What it cannot carry is\n// NULL vs empty string, or a value's type -- that is what json is for.\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// Header names come from a row wherever there is one: a statement's column list keeps\n// duplicates that the row object has already collapsed, so `SELECT a AS x, b AS x` would print\n// b's value under both headers. `columns` is the fallback that labels a 0-row result, and with\n// no rows and no statement (a bounded command that found nothing) csv writes nothing at all --\n// a bare newline would read to a csv parser as one empty record.\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Collapsed the same way a row object collapses them, so a statement naming a column\n    // twice describes itself identically whether it matched anything or not.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: rows arrive from an\n// iterator, so a large result is never held here as rows and again as a rendered string.\n// Table buffers by necessity (a column is as wide as its widest cell, which the last row can\n// change); json and csv write row by row. The writes are synchronous, so when stdout is a pipe\n// Node buffers what the reader has not taken yet -- the saving is this module's copies, not\n// the operating system's.\nexport function printRowStream(rows: Iterable<Row>, format: RowFormat, columns: string[]): void {\n  const iterator = rows[Symbol.iterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = iterator.next();\n  const rest: Iterable<Row> = { [Symbol.iterator]: () => iterator };\n\n  if (format === 'table') {\n    printRows(first.done ? [] : [first.value, ...rest], format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    // JSON.stringify throws on BigInt, so a row with an int64 past 2^53 (setReadBigInts) is\n    // converted via the replacer: a value that fits a safe integer stays a number, a larger one\n    // becomes its decimal string, since a JSON number cannot carry it losslessly.\n    const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\n    const bigintSafe = (_key: string, value: unknown) => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n    const indent = (row: Row) =>\n      JSON.stringify(row, bigintSafe, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\n// name, or name:weight when the weight is not the default 1.\nfunction signalLabels(signals: Record<string, number>): string {\n  return Object.entries(signals)\n    .map(([name, weight]) => (weight === 1 ? name : `${name}:${weight}`))\n    .join(', ');\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a preset without the vectors signal says what it uses instead.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)];\n}\n\nexport function renderMap(result: {\n  docs: { count: number; bytes: number };\n  fields: Row[];\n  fieldsTotal: number;\n  features: { on: string[]; off: string[] };\n  presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>;\n  hubs: Row[];\n  recent: Row[];\n  recentCaveat: string | null;\n}): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","test","csvLine","values","map","join","warnOversize","bytes","rowCount","console","warn","Math","round","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","first","next","rest","done","write","process","stdout","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","indent","JSON","stringify","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","cells","col","natural","i","clip","w","padEnd","slice","formatRow","trimEnd","repeat","featureOffHint","featuresLine","features","off","on","signalLabels","signals","entries","weight","presetsLines","presets","p","files","embedded","vectors","renderMap","result","parts","docs","count","fields","fieldsTotal","push","hubs","includes","recentCaveat","recent","renderPeek","lines","path","tokens","parseError","key","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,yFAAyF;AACzF,yFAAyF;AACzF,uEAAuE;AACvE,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKJ,OAAOD;IACjE,OAAO,WAAWM,IAAI,CAACF,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASG,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACN,UAAUO,IAAI,CAAC;AACnC;AAEA,SAASC,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAASd,gBAAgB;IAC7BgB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,sFAAsF;AACtF,+FAA+F;AAC/F,+FAA+F;AAC/F,+FAA+F;AAC/F,iEAAiE;AACjE,OAAO,SAASK,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,qFAAqF;QACrF,yEAAyE;QACzE,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAACpB,QAAQe;eAAWH,KAAKV,GAAG,CAAC,CAACmB,MAAQrB,QAAQe,MAAMb,GAAG,CAAC,CAACoB,OAASD,GAAG,CAACC,KAAK;SAAI,CAACnB,IAAI,CAAC;QACtGI,QAAQgB,GAAG,CAACH;QACZ,yDAAyD;QACzDhB,aAAaoB,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCN,QAAQgB,GAAG,CAACH;IACZhB,aAAaoB,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,uFAAuF;AACvF,yFAAyF;AACzF,6FAA6F;AAC7F,+FAA+F;AAC/F,4FAA4F;AAC5F,0BAA0B;AAC1B,OAAO,SAASW,eAAef,IAAmB,EAAEC,MAAiB,EAAEC,OAAiB;IACtF,MAAMc,WAAWhB,IAAI,CAACiB,OAAOD,QAAQ,CAAC;IACtC,wFAAwF;IACxF,2FAA2F;IAC3F,MAAME,QAAQF,SAASG,IAAI;IAC3B,MAAMC,OAAsB;QAAE,CAACH,OAAOD,QAAQ,CAAC,EAAE,IAAMA;IAAS;IAEhE,IAAIf,WAAW,SAAS;QACtBF,UAAUmB,MAAMG,IAAI,GAAG,EAAE,GAAG;YAACH,MAAMrC,KAAK;eAAKuC;SAAK,EAAEnB,QAAQC;QAC5D;IACF;IAEA,IAAIT,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM4B,QAAQ,CAACrC;QACb,yDAAyD;QACzDQ,SAASmB,OAAOC,UAAU,CAAC5B;QAC3BsC,QAAQC,MAAM,CAACF,KAAK,CAACrC;IACvB;IAEA,IAAIgB,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,wFAAwF;QACxF,4FAA4F;QAC5F,8EAA8E;QAC9E,MAAMwB,WAAWC,OAAOC,OAAOC,gBAAgB;QAC/C,MAAMC,aAAa,CAACC,MAAcjD,QAAoB,OAAOA,UAAU,WAAYA,SAAS,CAAC4C,YAAY5C,SAAS4C,WAAWE,OAAO9C,SAASA,MAAMkD,QAAQ,KAAMlD;QACjK,MAAMmD,SAAS,CAACvB,MACdwB,KAAKC,SAAS,CAACzB,KAAKoB,YAAY,GAC7BM,KAAK,CAAC,MACN7C,GAAG,CAAC,CAAC8C,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzB7C,IAAI,CAAC;QACV,IAAI2B,MAAMG,IAAI,EAAE;YACdC,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAMU,OAAOd,MAAMrC,KAAK;YACxBa,WAAW;YACX,KAAK,MAAMe,OAAOW,KAAM;gBACtBE,MAAM;gBACNA,MAAMU,OAAOvB;gBACbf;YACF;YACA4B,MAAM;QACR;IACF,OAAO,IAAIJ,MAAMG,IAAI,EAAE;QACrB,MAAMlB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGkB,MAAM,GAAGlC,QAAQe,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACY,MAAMrC,KAAK;QACrCyC,MAAM,GAAGlC,QAAQe,OAAO,EAAE,CAAC;QAC3BmB,MAAM,GAAGlC,QAAQe,MAAMb,GAAG,CAAC,CAACoB,OAASQ,MAAMrC,KAAK,CAAC6B,KAAK,GAAG,EAAE,CAAC;QAC5DhB,WAAW;QACX,KAAK,MAAMe,OAAOW,KAAM;YACtBE,MAAM,GAAGlC,QAAQe,MAAMb,GAAG,CAAC,CAACoB,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpDhB;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM2C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAOpC,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMuC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACpD,KAAKqD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGnD,KAAKqD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGnD,KAAKqD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAAS3B,WAAWd,IAAW,EAAEC,MAAc,EAAEkD,QAAQ5B,QAAQC,MAAM,CAACtB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAOgC,KAAKC,SAAS,CAAClC,MAAM,MAAM;IACzD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAMoD,QAAQpD,KAAKV,GAAG,CAAC,CAACmB,MAAQP,QAAQZ,GAAG,CAAC,CAAC+D,MAAQzE,KAAK6B,GAAG,CAAC4C,IAAI;IAClE,MAAMC,UAAUpD,QAAQZ,GAAG,CAAC,CAAC+D,KAAKE,IAAM1D,KAAKqD,GAAG,CAACG,IAAIjD,MAAM,KAAKgD,MAAM9D,GAAG,CAAC,CAACmB,MAAQA,GAAG,CAAC8C,EAAE,CAACnD,MAAM;IAChG,MAAMoC,SAASW,SAASA,QAAQd,aAAaE,UAAUe,SAASH,SAASG;IAEzE,MAAME,OAAO,CAAC3E,OAAe4E,IAAe5E,MAAMuB,MAAM,IAAIqD,IAAI5E,MAAM6E,MAAM,CAACD,KAAK,GAAG5E,MAAM8E,KAAK,CAAC,GAAGF,IAAI,KAAKnB,WAAW;IACxH,MAAMsB,YAAY,CAACvE,SACjBA,OACGC,GAAG,CAAC,CAACT,OAAO0E,IAAMC,KAAK3E,OAAO2D,MAAM,CAACe,EAAE,GACvChE,IAAI,CAAC,MACLsE,OAAO;IAEZ,OAAO;QAACD,UAAU1D;QAAUsC,OAAOlD,GAAG,CAAC,CAACmE,IAAM,IAAIK,MAAM,CAACL,IAAIlE,IAAI,CAAC;WAAU6D,MAAM9D,GAAG,CAACsE;KAAW,CAACrE,IAAI,CAAC;AACzG;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAASwE,eAAerD,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAASsD,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAAC9D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE6D,SAASC,GAAG,CAAC5E,GAAG,CAAC,CAACoB,OAAS,GAAGA,KAAK,EAAE,EAAEqD,eAAerD,MAAM,CAAC,CAAC,EAAEnB,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE0E,SAASE,EAAE,CAAC5E,IAAI,CAAC,QAAQ2E,KAAK;AACpD;AAEA,6DAA6D;AAC7D,SAASE,aAAaC,OAA+B;IACnD,OAAOhE,OAAOiE,OAAO,CAACD,SACnB/E,GAAG,CAAC,CAAC,CAACoB,MAAM6D,OAAO,GAAMA,WAAW,IAAI7D,OAAO,GAAGA,KAAK,CAAC,EAAE6D,QAAQ,EAClEhF,IAAI,CAAC;AACV;AAEA,OAAO,SAASiF,aAAaC,OAAkG;IAC7H,6FAA6F;IAC7F,8EAA8E;IAC9E,OAAO;QAAC;WAAeA,QAAQnF,GAAG,CAAC,CAACoF,IAAM,CAAC,EAAE,EAAEA,EAAEhE,IAAI,CAAC,EAAE,EAAEgE,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEL,OAAO,CAACQ,OAAO,KAAK3F,YAAY,KAAK,CAAC,WAAW,EAAEkF,aAAaM,EAAEL,OAAO,EAAE,CAAC,CAAC,EAAE;KAAE;AACzL;AAEA,OAAO,SAASS,UAAUC,MASzB;IACC,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAErF,KAAKC,KAAK,CAACiF,OAAOE,IAAI,CAACxF,KAAK,GAAG,MAAM,IAAI,CAAC;QAAEuE,aAAae,OAAOd,QAAQ;QAAG;WAAOO,aAAaO,OAAON,OAAO;QAAG;QAAI3D,WAAWiE,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAAC/E,MAAM,EAAE4E,MAAMK,IAAI,CAAC,CAAC,EAAE,EAAEN,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAAC/E,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI2E,OAAOO,IAAI,CAAClF,MAAM,GAAG,GAAG4E,MAAMK,IAAI,CAAC,0BAA0BvE,WAAWiE,OAAOO,IAAI,EAAE;SACpF,IAAIP,OAAOd,QAAQ,CAACC,GAAG,CAACqB,QAAQ,CAAC,SAASP,MAAMK,IAAI,CAAC,CAAC,aAAa,EAAEtB,eAAe,QAAQ,CAAC,CAAC;IACnGiB,MAAMK,IAAI,CAAC;IACX,IAAIN,OAAOS,YAAY,EAAER,MAAMK,IAAI,CAACN,OAAOS,YAAY;IACvDR,MAAMK,IAAI,CAACvE,WAAWiE,OAAOU,MAAM,EAAE;IACrC,OAAOT,MAAMzF,IAAI,CAAC;AACpB;AAEA,OAAO,SAASmG,WAAWX,MAAoR;IAC7S,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAId,OAAOe,UAAU,EAAEH,MAAMN,IAAI,CAAC,CAAC,wDAAwD,EAAEN,OAAOe,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAKlH,MAAM,IAAIwB,OAAOiE,OAAO,CAACS,OAAOiB,WAAW,EAAGL,MAAMN,IAAI,CAAC,CAAC,EAAE,EAAEU,IAAI,EAAE,EAAElH,OAAO;IAC9F,IAAIkG,OAAOkB,QAAQ,CAAC7F,MAAM,GAAG,GAAG;QAC9BuF,MAAMN,IAAI,CAAC,IAAI;QACf,KAAK,MAAMa,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAMN,IAAI,CAAC,CAAC,EAAE,EAAE,IAAIvB,MAAM,CAACoC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAId,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAAC7F,MAAM,EAAEuF,MAAMN,IAAI,CAAC,CAAC,IAAI,EAAEN,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAAC7F,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAI2E,OAAOb,GAAG,CAACqB,QAAQ,CAAC,aAAaI,MAAMN,IAAI,CAAC,IAAI;IAC3D,IAAIN,OAAOb,GAAG,CAACqB,QAAQ,CAAC,UAAU;QAChCI,MAAMN,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMmB,WAAW,CAACC,OAAeC,OAAiB9D;YAChD,IAAIA,QAAQ,GAAG+C,MAAMN,IAAI,CAAC,GAAGoB,MAAM,EAAE,EAAE7D,MAAM,GAAG,EAAE8D,MAAMnH,IAAI,CAAC,QAAQqD,QAAQ8D,MAAMtG,MAAM,GAAG,CAAC,GAAG,EAAEwC,QAAQ8D,MAAMtG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAI2E,OAAO4B,aAAa,GAAG5B,OAAO6B,eAAe,GAAG7B,OAAO8B,cAAc,GAAG,GAAGlB,MAAMN,IAAI,CAAC;QAC1FmB,SAAS,aAAazB,OAAO+B,QAAQ,EAAE/B,OAAO4B,aAAa;QAC3DH,SAAS,cAAczB,OAAOgC,UAAU,EAAEhC,OAAO6B,eAAe;QAChEJ,SAAS,aAAazB,OAAOiC,SAAS,EAAEjC,OAAO8B,cAAc;IAC/D;IACA,OAAOlB,MAAMpG,IAAI,CAAC;AACpB"}

package/dist/esm/scan.js

@@ -2,7 +2,7 @@ import { globSync, readFileSync, statSync } from 'node:fs';
 import { join, sep } from 'node:path';
 import removeMarkdown from 'remove-markdown';
 import { isCollection, parseDocument, visit } from 'yaml';
-import { embedEnabled, presetNames, presetSemanticEnabled } from './config/index.js';
+import { embedEnabled, presetHasSignal, presetNames } from './config/index.js';
 // Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
 // Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell
 // a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.
@@ -34,7 +34,10 @@ function normalizeText(value) {
 // carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.
 function stripText(value) {
     const withoutWikilinks = value.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2').replace(/\[\[([^\]]+)\]\]/g, '$1');
-    const withoutMarkdown = removeMarkdown(withoutWikilinks);
+    // remove-markdown deletes footnote definitions wholesale ("[^1]: text" -> ""), so unwrap
+    // them (and drop inline refs) before it runs; the definition text stays indexed.
+    const withoutFootnotes = withoutWikilinks.replace(/^\[\^[^\]]+\]:[ \t]?/gm, '').replace(/\[\^[^\]]+\]/g, '');
+    const withoutMarkdown = removeMarkdown(withoutFootnotes);
     const withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
     return normalizeText(withoutTables);
 }
@@ -67,7 +70,7 @@ export function listFiles(cfg, baseDir) {
     }
     // Which presets want vectors is a property of the config, not of any file.
     const embedding = embedEnabled(cfg);
-    const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name)=>presetSemanticEnabled(cfg, name))) : null;
+    const vectorPresets = embedding ? new Set(presetNames(cfg).filter((name)=>presetHasSignal(cfg, name, 'vectors'))) : null;
     const files = [];
     for (const relPath of [
         ...coverage.keys()
@@ -80,7 +83,7 @@ export function listFiles(cfg, baseDir) {
         const presets = [
             ...coverage.get(relPath)
         ].sort();
-        const embed = semanticPresets !== null && presets.some((name)=>semanticPresets.has(name));
+        const embed = vectorPresets !== null && presets.some((name)=>vectorPresets.has(name));
         files.push({
             relPath,
             absPath,

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config/index.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config/index.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  ctimeMs: number; // filesystem birthtime; a clone or copy resets it, like _mtime\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, ctimeMs: st.birthtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data)])),\n    },\n    warnings,\n  };\n}\n"],"names":["globSync","readFileSync","statSync","join","sep","removeMarkdown","isCollection","parseDocument","visit","embedEnabled","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","add","embedding","semanticPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","ctimeMs","birthtimeMs","size","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","doc","warnings","found","Pair","_key","pair","key","BREAK","firstLine","message","parseFrontmatter","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","parseFile","file","extractors","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AAC3D,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AACtC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,SAASC,YAAY,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,oBAAoB;AAGrF,6EAA6E;AAE7E,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4DAA4D;AAC5D,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBlB,eAAeiB;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAYA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBvB,GAAG;IAClE,OAAOuB,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWxB,IAAI,CAAC,OAAOuB;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAchC,QAAQ;IAC5B,KAAK,MAAMiC,QAAQ3B,YAAYsB,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWxC,SAASsC,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIrB;YACzC+B,IAAIE,GAAG,CAACT;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMG,YAAYtC,aAAauB;IAC/B,MAAMgB,kBAAkBD,YAAY,IAAIlC,IAAIH,YAAYsB,KAAKiB,MAAM,CAAC,CAACZ,OAAS1B,sBAAsBqB,KAAKK,UAAU;IAEnH,MAAMa,QAAoB,EAAE;IAC5B,KAAK,MAAMxB,WAAW;WAAIQ,SAASiB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlD,KAAK8B,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM4B,KAAKpD,SAASmD,SAASxB;QAC7B,IAAI,EAACyB,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMhB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC0B,IAAI;QAChE,MAAMI,QAAQR,oBAAoB,QAAQT,QAAQkB,IAAI,CAAC,CAACpB,OAASW,gBAAgBU,GAAG,CAACrB;QACrFa,MAAMS,IAAI,CAAC;YAAEjC;YAAS2B;YAASO,SAASN,GAAGM,OAAO;YAAEC,SAASP,GAAGQ,WAAW;YAAEC,MAAMT,GAAGS,IAAI;YAAExB;YAASiB;QAAM;IAC7G;IACA,OAAON;AACT;AAkBA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMc,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBlD,KAAa;IAC7C,OAAOiD,kBAAkBE,IAAI,CAACnD;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASoD,cAAcpD,KAAa;IACzC,MAAMqD,IAAIL,aAAaM,IAAI,CAACtD;IAC5B,IAAIqD,MAAM,MAAM,OAAOrD;IACvB,MAAM,GAAGuD,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASxD,aAAawD,SAAS,MAAM,KAAKA,KAAKtD,OAAO,CAAC,KAAK;IAC3E,MAAMwD,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAe9D,QAAQ8D;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASK,SAASnE,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOoE,OAAOpE,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO+D,OAAOM,aAAa,CAACrE,SAASoE,OAAOpE,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOoD,cAAcpD;IACpD,OAAOsE,KAAKC,SAAS,CAACvE;AACxB;AAEA,gEAAgE;AAChE,SAASwE,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,MAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAK/E,WAAW,OAAO;QAAE2E,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBvE,OAAe,EAAEwE,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC5F,MAAM0F,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACjG,aAAaiG,KAAKC,GAAG,GAAG,OAAOvF;YACpCmF,QAAQ;YACR,OAAO5F,MAAMiG,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASgF,UAAUC,OAAe;IAChC,OAAOA,QAAQ/E,KAAK,CAAC,KAAK,CAAC,EAAE,CAACT,OAAO,CAAC,SAAS;AACjD;AAEA,SAASyF,iBAAiBlF,OAAe,EAAEkE,EAAU,EAAEO,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAM3F,cAAcqF,IAAI;QAAEiB,UAAU;IAAS;IACnD,MAAMC,UAAUZ,IAAIa,MAAM,CAAC9D,MAAM,CAAC,CAAC+D,MAAQ,CAAClG,oBAAoB4C,GAAG,CAACsD,IAAIC,IAAI;IAC5E,IAAIH,QAAQlC,MAAM,GAAG,GAAG;QACtB,MAAMsC,SAASJ,QAAQlC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEkC,QAAQlC,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMuC,aAAa,GAAGT,UAAUI,OAAO,CAAC,EAAE,CAACH,OAAO,IAAIO,QAAQ;QAC9Df,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEyF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOlB,IAAImB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMG,aAAaT,UAAU,AAACM,IAAcL,OAAO;QACnDR,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEyF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASnG,WAAW,OAAO;QAAEmG,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBhB,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,CAAC,EAAEyF,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAlB,oBAAoBvE,SAASwE,KAAKC;IAClC,OAAO;QAAEiB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMjC,MAAMxF,aAAawH,KAAKpE,OAAO,EAAE;IACvC,MAAM8C,WAAqB,EAAE;IAE7B,MAAM,EAAEP,EAAE,EAAEC,MAAM8B,OAAO,EAAE,GAAGnC,iBAAiBC;IAC/C,MAAM,EAAE2B,IAAI,EAAED,UAAU,EAAE,GAAGvB,OAAO,OAAO;QAAEwB,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIP,iBAAiBa,KAAK/F,OAAO,EAAEkE,IAAIO;IAC1I,MAAMyB,SAA0D,CAAC;IAEjE,KAAK,MAAMpB,OAAOqB,OAAO1E,IAAI,CAACiE,MAAO;QACnC,IAAIxG,iBAAiB8C,GAAG,CAAC8C,MAAM;YAC7BL,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAE8D,KAAK/F,OAAO,CAAC,8BAA8B,EAAE8E,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMxF,QAAQmE,SAASiC,IAAI,CAACZ,IAAI;QAChC,IAAI,OAAOxF,UAAU,YAAYkD,kBAAkBlD,UAAU+D,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAAClE,SAAS;YAC5FmF,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAE8D,KAAK/F,OAAO,CAAC,EAAE,EAAE8E,IAAI,sBAAsB,EAAExF,MAAM,8CAA8C,CAAC;QAC9H;QACA4G,MAAM,CAACpB,IAAI,GAAGxF;IAChB;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAM8G,SAAS;QAAEC,OAAOhH,cAAcqG,KAAKW,KAAK;QAAGC,SAASjH,cAAcqG,KAAKY,OAAO;QAAGC,MAAM5G,UAAUsG;IAAS;IAElH,OAAO;QACLzB,KAAK;YACHxE,SAAS+F,KAAK/F,OAAO;YACrBkC,SAAS6D,KAAK7D,OAAO;YACrBC,SAAS4D,KAAK5D,OAAO;YACrBE,MAAM0D,KAAK1D,IAAI;YACfxB,SAASkF,KAAKlF,OAAO;YACrB6E,MAAMQ;YACNT;YACAW;YACAI,WAAWL,OAAOM,WAAW,CAACT,WAAWzE,MAAM,CAAC,CAACmF,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAE/F,IAAI;qBAAE+F,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY3C,KAAKkC,SAASG,QAAQV;iBAAM;;QAChI;QACAjB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config/index.ts';\nimport { embedEnabled, presetHasSignal, presetNames } from './config/index.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  // remove-markdown deletes footnote definitions wholesale (\"[^1]: text\" -> \"\"), so unwrap\n  // them (and drop inline refs) before it runs; the definition text stays indexed.\n  const withoutFootnotes = withoutWikilinks.replace(/^\\[\\^[^\\]]+\\]:[ \\t]?/gm, '').replace(/\\[\\^[^\\]]+\\]/g, '');\n  const withoutMarkdown = removeMarkdown(withoutFootnotes);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  ctimeMs: number; // filesystem birthtime; a clone or copy resets it, like _mtime\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset's signals include vectors\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const vectorPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetHasSignal(cfg, name, 'vectors'))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = vectorPresets !== null && presets.some((name) => vectorPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, ctimeMs: st.birthtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data)])),\n    },\n    warnings,\n  };\n}\n"],"names":["globSync","readFileSync","statSync","join","sep","removeMarkdown","isCollection","parseDocument","visit","embedEnabled","presetHasSignal","presetNames","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutFootnotes","withoutMarkdown","withoutTables","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","add","embedding","vectorPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","ctimeMs","birthtimeMs","size","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","doc","warnings","found","Pair","_key","pair","key","BREAK","firstLine","message","parseFrontmatter","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","parseFile","file","extractors","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AAC3D,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AACtC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,SAASC,YAAY,EAAEC,eAAe,EAAEC,WAAW,QAAQ,oBAAoB;AAG/E,6EAA6E;AAE7E,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4DAA4D;AAC5D,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,yFAAyF;IACzF,iFAAiF;IACjF,MAAMI,mBAAmBD,iBAAiBH,OAAO,CAAC,0BAA0B,IAAIA,OAAO,CAAC,iBAAiB;IACzG,MAAMK,kBAAkBnB,eAAekB;IACvC,MAAME,gBAAgBD,gBAAgBL,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AAYA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBxB,GAAG;IAClE,OAAOwB,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWzB,IAAI,CAAC,OAAOwB;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAcjC,QAAQ;IAC5B,KAAK,MAAMkC,QAAQ3B,YAAYsB,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWzC,SAASuC,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAItB;YACzCgC,IAAIE,GAAG,CAACT;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMG,YAAYvC,aAAawB;IAC/B,MAAMgB,gBAAgBD,YAAY,IAAInC,IAAIF,YAAYsB,KAAKiB,MAAM,CAAC,CAACZ,OAAS5B,gBAAgBuB,KAAKK,MAAM,eAAe;IAEtH,MAAMa,QAAoB,EAAE;IAC5B,KAAK,MAAMxB,WAAW;WAAIQ,SAASiB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUnD,KAAK+B,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM4B,KAAKrD,SAASoD,SAASxB;QAC7B,IAAI,EAACyB,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMhB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC0B,IAAI;QAChE,MAAMI,QAAQR,kBAAkB,QAAQT,QAAQkB,IAAI,CAAC,CAACpB,OAASW,cAAcU,GAAG,CAACrB;QACjFa,MAAMS,IAAI,CAAC;YAAEjC;YAAS2B;YAASO,SAASN,GAAGM,OAAO;YAAEC,SAASP,GAAGQ,WAAW;YAAEC,MAAMT,GAAGS,IAAI;YAAExB;YAASiB;QAAM;IAC7G;IACA,OAAON;AACT;AAkBA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMc,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBnD,KAAa;IAC7C,OAAOkD,kBAAkBE,IAAI,CAACpD;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASqD,cAAcrD,KAAa;IACzC,MAAMsD,IAAIL,aAAaM,IAAI,CAACvD;IAC5B,IAAIsD,MAAM,MAAM,OAAOtD;IACvB,MAAM,GAAGwD,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASzD,aAAayD,SAAS,MAAM,KAAKA,KAAKvD,OAAO,CAAC,KAAK;IAC3E,MAAMyD,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAe/D,QAAQ+D;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASK,SAASpE,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOqE,OAAOrE,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOgE,OAAOM,aAAa,CAACtE,SAASqE,OAAOrE,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOqD,cAAcrD;IACpD,OAAOuE,KAAKC,SAAS,CAACxE;AACxB;AAEA,gEAAgE;AAChE,SAASyE,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,MAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKhF,WAAW,OAAO;QAAE4E,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBvE,OAAe,EAAEwE,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC7F,MAAM2F,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAAClG,aAAakG,KAAKC,GAAG,GAAG,OAAOxF;YACpCoF,QAAQ;YACR,OAAO7F,MAAMkG,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASgF,UAAUC,OAAe;IAChC,OAAOA,QAAQ/E,KAAK,CAAC,KAAK,CAAC,EAAE,CAACV,OAAO,CAAC,SAAS;AACjD;AAEA,SAAS0F,iBAAiBlF,OAAe,EAAEkE,EAAU,EAAEO,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAM5F,cAAcsF,IAAI;QAAEiB,UAAU;IAAS;IACnD,MAAMC,UAAUZ,IAAIa,MAAM,CAAC9D,MAAM,CAAC,CAAC+D,MAAQ,CAACnG,oBAAoB6C,GAAG,CAACsD,IAAIC,IAAI;IAC5E,IAAIH,QAAQlC,MAAM,GAAG,GAAG;QACtB,MAAMsC,SAASJ,QAAQlC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEkC,QAAQlC,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMuC,aAAa,GAAGT,UAAUI,OAAO,CAAC,EAAE,CAACH,OAAO,IAAIO,QAAQ;QAC9Df,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEyF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOlB,IAAImB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMG,aAAaT,UAAU,AAACM,IAAcL,OAAO;QACnDR,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEyF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASpG,WAAW,OAAO;QAAEoG,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBhB,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,CAAC,EAAEyF,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAlB,oBAAoBvE,SAASwE,KAAKC;IAClC,OAAO;QAAEiB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMjC,MAAMzF,aAAayH,KAAKpE,OAAO,EAAE;IACvC,MAAM8C,WAAqB,EAAE;IAE7B,MAAM,EAAEP,EAAE,EAAEC,MAAM8B,OAAO,EAAE,GAAGnC,iBAAiBC;IAC/C,MAAM,EAAE2B,IAAI,EAAED,UAAU,EAAE,GAAGvB,OAAO,OAAO;QAAEwB,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIP,iBAAiBa,KAAK/F,OAAO,EAAEkE,IAAIO;IAC1I,MAAMyB,SAA0D,CAAC;IAEjE,KAAK,MAAMpB,OAAOqB,OAAO1E,IAAI,CAACiE,MAAO;QACnC,IAAIzG,iBAAiB+C,GAAG,CAAC8C,MAAM;YAC7BL,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAE8D,KAAK/F,OAAO,CAAC,8BAA8B,EAAE8E,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMzF,QAAQoE,SAASiC,IAAI,CAACZ,IAAI;QAChC,IAAI,OAAOzF,UAAU,YAAYmD,kBAAkBnD,UAAUgE,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACnE,SAAS;YAC5FoF,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAE8D,KAAK/F,OAAO,CAAC,EAAE,EAAE8E,IAAI,sBAAsB,EAAEzF,MAAM,8CAA8C,CAAC;QAC9H;QACA6G,MAAM,CAACpB,IAAI,GAAGzF;IAChB;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAM+G,SAAS;QAAEC,OAAOjH,cAAcsG,KAAKW,KAAK;QAAGC,SAASlH,cAAcsG,KAAKY,OAAO;QAAGC,MAAM7G,UAAUuG;IAAS;IAElH,OAAO;QACLzB,KAAK;YACHxE,SAAS+F,KAAK/F,OAAO;YACrBkC,SAAS6D,KAAK7D,OAAO;YACrBC,SAAS4D,KAAK5D,OAAO;YACrBE,MAAM0D,KAAK1D,IAAI;YACfxB,SAASkF,KAAKlF,OAAO;YACrB6E,MAAMQ;YACNT;YACAW;YACAI,WAAWL,OAAOM,WAAW,CAACT,WAAWzE,MAAM,CAAC,CAACmF,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAE/F,IAAI;qBAAE+F,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY3C,KAAKkC,SAASG,QAAQV;iBAAM;;QAChI;QACAjB;IACF;AACF"}

package/dist/esm/segment.js

@@ -3,9 +3,10 @@
 // Segmentation is context-free: seg(query) appears inside seg(document) whenever the query is
 // a substring of the document. That is the contract -- substring semantics, what `grep` and
 // `LIKE '%..%'` give -- and it is what makes index and query agree on every input, always.
-// Earlier attempts used ICU word segmentation (Intl.Segmenter), which is context-dependent:
-// `东京都政府` splits as `东 | 京都 | 政府` while the query `东京` splits as one word, so the
-// index and the query disagree about identical text. No ICU here. No dictionary.
+// Intl.Segmenter's word mode is context-dependent (`东京都政府` splits as `东 | 京都 | 政府`
+// while the query `东京` splits as one word) and is rejected for that reason. Its grapheme
+// mode has no such dependency -- UAX #29 cluster boundaries never look past adjacent
+// characters -- so grapheme mode is used below; no dictionary, no word mode.
 // Scripts written without word spaces: a closed set of writing systems.
 // Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.
 const SCRIPTS = '\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}';
@@ -13,30 +14,61 @@ const SCRIPTS = '\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{
 // Latin letter (decomposed é) never starts one.
 const RUN = new RegExp(`((?:[${SCRIPTS}]\\p{M}*)+)`, 'gu');
 const HAS_RUN = new RegExp(`[${SCRIPTS}]`, 'u');
-// Grapheme = base char plus its marks. Context-free: index and query can never disagree.
-const GRAPHEME = /(\P{M}\p{M}*)/gu;
-// Token barrier between separate runs so their graphemes are never phrase-adjacent.
-// U+A7F7: a letter (so FTS5 keeps it as a token) that no one types into a search.
+// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.
+// Built once (construction cost amortizes) and used for both index and query splitting.
+const GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, {
+    granularity: 'grapheme'
+});
+// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions
+// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.
+const PUNCTUATION = /\p{P}/u;
+// Token barrier between separate runs (or a punctuation split within one) so their graphemes
+// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.
 const BARRIER = 'ꟷ';
 function graphemes(run) {
-    var _run_match;
-    return (_run_match = run.match(GRAPHEME)) !== null && _run_match !== void 0 ? _run_match : [];
+    return Array.from(GRAPHEME_SEGMENTER.segment(run), (s)=>s.segment);
+}
+// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,
+// matching unicode61). The one place punctuation is classified, so index and query agree.
+function splitOnPunctuation(run) {
+    const groups = [
+        []
+    ];
+    for (const g of graphemes(run)){
+        if (PUNCTUATION.test(g)) groups.push([]);
+        else groups[groups.length - 1].push(g);
+    }
+    return groups.filter((g)=>g.length > 0);
 }
 // Index side: '' when the field has no unspaced-script run (the common case, paid for by
 // nothing). Otherwise each run explodes into its graphemes, barrier-delimited from its
-// neighbors so graphemes from separate runs are never phrase-adjacent, and whitespace collapses.
+// neighbors and from a punctuation split within itself.
 export function segmentField(text) {
     if (!HAS_RUN.test(text)) return '';
-    const out = text.replace(RUN, (run)=>` ${BARRIER} ${graphemes(run).join(' ')} ${BARRIER} `);
+    const out = text.replace(RUN, (run)=>{
+        const body = splitOnPunctuation(run).map((g)=>g.join(' ')).join(` ${BARRIER} `);
+        return ` ${BARRIER} ${body} ${BARRIER} `;
+    });
     return out.replace(/\s+/g, ' ').trim();
 }
 // A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a
 // quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.
 const QUALIFIER = /(^|[\s(])(-?)(title|summary|text)\s*:\s*$/;
-// Query side: each unspaced run becomes a quoted phrase of its graphemes, matching how
-// segmentField indexed it. A qualifier ahead of such a run maps to its `_seg` column, since
-// segmented text lives only there. An author's own quoted phrase, and everything else, pass
-// through byte-identical.
+// An unqualified run's target: raw title/summary/text drop punctuation as unicode61's token
+// separator, so two adjacent single-grapheme phrase positions match across punctuation the same
+// as across a real gap (`数数` vs `数。数`) -- a false positive only the barriered `_seg` columns
+// are safe from. FTS5's column-set filter, not parens+OR, so the group still composes under
+// AND/OR/NOT/juxtaposition exactly like the single-column qualifier form below.
+const SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';
+// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),
+// space-joined -- the same split points segmentField barriers, so query and index agree.
+function runQuery(run, columnPrefix) {
+    return splitOnPunctuation(run).map((g)=>`${columnPrefix}${g.length > 1 ? `"${g.join(' ')}"` : g[0]}`).join(' ');
+}
+// Query side: each unspaced run becomes phrases of its graphemes, matching how segmentField
+// indexed it. A qualifier ahead of such a run maps to its `_seg` column; unqualified maps to all
+// three (SIDECAR_COLUMNS), since raw columns cannot express the contract. An author's own quoted
+// phrase is their explicit escape hatch to FTS5's native syntax, and passes through byte-identical.
 export function segmentMatch(terms) {
     if (!HAS_RUN.test(terms)) return terms;
     let out = '';
@@ -52,13 +84,11 @@ export function segmentMatch(terms) {
             out += pieces[i]; // an author's phrase is matched as written
             continue;
         }
-        const g = graphemes(pieces[i]);
-        const phrase = g.length > 1 ? `"${g.join(' ')}"` : g[0];
         const m = out.match(QUALIFIER);
         if (m) {
-            out = `${out.slice(0, m.index)}${m[1]}${m[2]}${m[3]}_seg:${phrase}`;
+            out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;
         } else {
-            out += phrase;
+            out += runQuery(pieces[i], SIDECAR_COLUMNS);
         }
     }
     return out;

package/dist/esm/segment.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own.\n//\n// Segmentation is context-free: seg(query) appears inside seg(document) whenever the query is\n// a substring of the document. That is the contract -- substring semantics, what `grep` and\n// `LIKE '%..%'` give -- and it is what makes index and query agree on every input, always.\n// Earlier attempts used ICU word segmentation (Intl.Segmenter), which is context-dependent:\n// `东京都政府` splits as `东 | 京都 | 政府` while the query `东京` splits as one word, so the\n// index and the query disagree about identical text. No ICU here. No dictionary.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nconst SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${SCRIPTS}]`, 'u');\n// Grapheme = base char plus its marks. Context-free: index and query can never disagree.\nconst GRAPHEME = /(\\P{M}\\p{M}*)/gu;\n// Token barrier between separate runs so their graphemes are never phrase-adjacent.\n// U+A7F7: a letter (so FTS5 keeps it as a token) that no one types into a search.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  return run.match(GRAPHEME) ?? [];\n}\n\n// Index side: '' when the field has no unspaced-script run (the common case, paid for by\n// nothing). Otherwise each run explodes into its graphemes, barrier-delimited from its\n// neighbors so graphemes from separate runs are never phrase-adjacent, and whitespace collapses.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => ` ${BARRIER} ${graphemes(run).join(' ')} ${BARRIER} `);\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Query side: each unspaced run becomes a quoted phrase of its graphemes, matching how\n// segmentField indexed it. A qualifier ahead of such a run maps to its `_seg` column, since\n// segmented text lives only there. An author's own quoted phrase, and everything else, pass\n// through byte-identical.\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const g = graphemes(pieces[i]);\n    const phrase = g.length > 1 ? `\"${g.join(' ')}\"` : g[0];\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${m[2]}${m[3]}_seg:${phrase}`;\n    } else {\n      out += phrase;\n    }\n  }\n  return out;\n}\n"],"names":["SCRIPTS","RUN","RegExp","HAS_RUN","GRAPHEME","BARRIER","graphemes","run","match","segmentField","text","test","out","replace","join","trim","QUALIFIER","segmentMatch","terms","quoted","pieces","split","i","length","ch","g","phrase","m","slice","index"],"mappings":"AAAA,8DAA8D;AAC9D,EAAE;AACF,8FAA8F;AAC9F,4FAA4F;AAC5F,2FAA2F;AAC3F,4FAA4F;AAC5F,kFAAkF;AAClF,iFAAiF;AAEjF,wEAAwE;AACxE,kFAAkF;AAClF,MAAMA,UAAU;AAChB,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,MAAM,IAAIC,OAAO,CAAC,KAAK,EAAEF,QAAQ,WAAW,CAAC,EAAE;AACrD,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEF,QAAQ,CAAC,CAAC,EAAE;AAC3C,yFAAyF;AACzF,MAAMI,WAAW;AACjB,oFAAoF;AACpF,kFAAkF;AAClF,MAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;QACrBA;IAAP,QAAOA,aAAAA,IAAIC,KAAK,CAACJ,uBAAVG,wBAAAA,aAAuB,EAAE;AAClC;AAEA,yFAAyF;AACzF,uFAAuF;AACvF,iGAAiG;AACjG,OAAO,SAASE,aAAaC,IAAY;IACvC,IAAI,CAACP,QAAQQ,IAAI,CAACD,OAAO,OAAO;IAChC,MAAME,MAAMF,KAAKG,OAAO,CAACZ,KAAK,CAACM,MAAQ,CAAC,CAAC,EAAEF,QAAQ,CAAC,EAAEC,UAAUC,KAAKO,IAAI,CAAC,KAAK,CAAC,EAAET,QAAQ,CAAC,CAAC;IAC5F,OAAOO,IAAIC,OAAO,CAAC,QAAQ,KAAKE,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,uFAAuF;AACvF,4FAA4F;AAC5F,4FAA4F;AAC5F,0BAA0B;AAC1B,OAAO,SAASC,aAAaC,KAAa;IACxC,IAAI,CAACf,QAAQQ,IAAI,CAACO,QAAQ,OAAOA;IACjC,IAAIN,MAAM;IACV,IAAIO,SAAS;IACb,MAAMC,SAASF,MAAMG,KAAK,CAACpB,MAAM,2CAA2C;IAC5E,IAAK,IAAIqB,IAAI,GAAGA,IAAIF,OAAOG,MAAM,EAAED,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAME,MAAMJ,MAAM,CAACE,EAAE,CAAE,IAAIE,OAAO,KAAKL,SAAS,CAACA;YACtDP,OAAOQ,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVP,OAAOQ,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,MAAMG,IAAInB,UAAUc,MAAM,CAACE,EAAE;QAC7B,MAAMI,SAASD,EAAEF,MAAM,GAAG,IAAI,CAAC,CAAC,EAAEE,EAAEX,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGW,CAAC,CAAC,EAAE;QACvD,MAAME,IAAIf,IAAIJ,KAAK,CAACQ;QACpB,IAAIW,GAAG;YACLf,MAAM,GAAGA,IAAIgB,KAAK,CAAC,GAAGD,EAAEE,KAAK,IAAIF,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,EAAED,QAAQ;QACrE,OAAO;YACLd,OAAOc;QACT;IACF;IACA,OAAOd;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own.\n//\n// Segmentation is context-free: seg(query) appears inside seg(document) whenever the query is\n// a substring of the document. That is the contract -- substring semantics, what `grep` and\n// `LIKE '%..%'` give -- and it is what makes index and query agree on every input, always.\n// Intl.Segmenter's word mode is context-dependent (`东京都政府` splits as `东 | 京都 | 政府`\n// while the query `东京` splits as one word) and is rejected for that reason. Its grapheme\n// mode has no such dependency -- UAX #29 cluster boundaries never look past adjacent\n// characters -- so grapheme mode is used below; no dictionary, no word mode.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nconst SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${SCRIPTS}]`, 'u');\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built once (construction cost amortizes) and used for both index and query splitting.\nconst GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  return Array.from(GRAPHEME_SEGMENTER.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run (the common case, paid for by\n// nothing). Otherwise each run explodes into its graphemes, barrier-delimited from its\n// neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// An unqualified run's target: raw title/summary/text drop punctuation as unicode61's token\n// separator, so two adjacent single-grapheme phrase positions match across punctuation the same\n// as across a real gap (`数数` vs `数。数`) -- a false positive only the barriered `_seg` columns\n// are safe from. FTS5's column-set filter, not parens+OR, so the group still composes under\n// AND/OR/NOT/juxtaposition exactly like the single-column qualifier form below.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching how segmentField\n// indexed it. A qualifier ahead of such a run maps to its `_seg` column; unqualified maps to all\n// three (SIDECAR_COLUMNS), since raw columns cannot express the contract. An author's own quoted\n// phrase is their explicit escape hatch to FTS5's native syntax, and passes through byte-identical.\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["SCRIPTS","RUN","RegExp","HAS_RUN","GRAPHEME_SEGMENTER","Intl","Segmenter","undefined","granularity","PUNCTUATION","BARRIER","graphemes","run","Array","from","segment","s","splitOnPunctuation","groups","g","test","push","length","filter","segmentField","text","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","segmentMatch","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,8DAA8D;AAC9D,EAAE;AACF,8FAA8F;AAC9F,4FAA4F;AAC5F,2FAA2F;AAC3F,mFAAmF;AACnF,yFAAyF;AACzF,qFAAqF;AACrF,6EAA6E;AAE7E,wEAAwE;AACxE,kFAAkF;AAClF,MAAMA,UAAU;AAChB,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,MAAM,IAAIC,OAAO,CAAC,KAAK,EAAEF,QAAQ,WAAW,CAAC,EAAE;AACrD,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEF,QAAQ,CAAC,CAAC,EAAE;AAC3C,6FAA6F;AAC7F,wFAAwF;AACxF,MAAMI,qBAAqB,IAAIC,KAAKC,SAAS,CAACC,WAAW;IAAEC,aAAa;AAAW;AACnF,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,MAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5B,OAAOC,MAAMC,IAAI,CAACV,mBAAmBW,OAAO,CAACH,MAAM,CAACI,IAAMA,EAAED,OAAO;AACrE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBL,GAAW;IACrC,MAAMM,SAAqB;QAAC,EAAE;KAAC;IAC/B,KAAK,MAAMC,KAAKR,UAAUC,KAAM;QAC9B,IAAIH,YAAYW,IAAI,CAACD,IAAID,OAAOG,IAAI,CAAC,EAAE;aAClCH,MAAM,CAACA,OAAOI,MAAM,GAAG,EAAE,CAACD,IAAI,CAACF;IACtC;IACA,OAAOD,OAAOK,MAAM,CAAC,CAACJ,IAAMA,EAAEG,MAAM,GAAG;AACzC;AAEA,yFAAyF;AACzF,uFAAuF;AACvF,wDAAwD;AACxD,OAAO,SAASE,aAAaC,IAAY;IACvC,IAAI,CAACtB,QAAQiB,IAAI,CAACK,OAAO,OAAO;IAChC,MAAMC,MAAMD,KAAKE,OAAO,CAAC1B,KAAK,CAACW;QAC7B,MAAMgB,OAAOX,mBAAmBL,KAC7BiB,GAAG,CAAC,CAACV,IAAMA,EAAEW,IAAI,CAAC,MAClBA,IAAI,CAAC,CAAC,CAAC,EAAEpB,QAAQ,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,EAAEA,QAAQ,CAAC,EAAEkB,KAAK,CAAC,EAAElB,QAAQ,CAAC,CAAC;IAC1C;IACA,OAAOgB,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,gGAAgG;AAChG,6FAA6F;AAC7F,4FAA4F;AAC5F,gFAAgF;AAChF,MAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAAStB,GAAW,EAAEuB,YAAoB;IACjD,OAAOlB,mBAAmBL,KACvBiB,GAAG,CAAC,CAACV,IAAM,GAAGgB,eAAehB,EAAEG,MAAM,GAAG,IAAI,CAAC,CAAC,EAAEH,EAAEW,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGX,CAAC,CAAC,EAAE,EAAE,EACvEW,IAAI,CAAC;AACV;AAEA,4FAA4F;AAC5F,iGAAiG;AACjG,iGAAiG;AACjG,oGAAoG;AACpG,OAAO,SAASM,aAAaC,KAAa;IACxC,IAAI,CAAClC,QAAQiB,IAAI,CAACiB,QAAQ,OAAOA;IACjC,IAAIX,MAAM;IACV,IAAIY,SAAS;IACb,MAAMC,SAASF,MAAMG,KAAK,CAACvC,MAAM,2CAA2C;IAC5E,IAAK,IAAIwC,IAAI,GAAGA,IAAIF,OAAOjB,MAAM,EAAEmB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAMC,MAAMH,MAAM,CAACE,EAAE,CAAE,IAAIC,OAAO,KAAKJ,SAAS,CAACA;YACtDZ,OAAOa,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVZ,OAAOa,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,MAAME,IAAIjB,IAAIkB,KAAK,CAACZ;QACpB,IAAIW,GAAG;YACLjB,MAAM,GAAGA,IAAImB,KAAK,CAAC,GAAGF,EAAEG,KAAK,IAAIH,CAAC,CAAC,EAAE,GAAGT,SAASK,MAAM,CAACE,EAAE,EAAE,GAAGE,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG;QACtF,OAAO;YACLjB,OAAOQ,SAASK,MAAM,CAACE,EAAE,EAAER;QAC7B;IACF;IACA,OAAOP;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.15.3",
+  "version": "0.16.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -77,17 +77,25 @@
   },
   "dependencies": {
     "@huggingface/tokenizers": "^0.1.3",
+    "franc-min": "^6.2.0",
     "remove-markdown": "^0.6.4",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
     "@types/mocha": "*",
     "@types/node": "*",
+    "cr": "^0.1.0",
     "node-version-use": "*",
     "ts-dev-stack": "*",
     "tsds-config": "*"
   },
   "engines": {
     "node": ">=22.20"
+  },
+  "allowScripts": {
+    "node-semvers": true,
+    "node-filename-to-dist-paths": true,
+    "node-version-use": true,
+    "thread-sleep-compat": true
   }
 }

package/schema.json

@@ -12,12 +12,12 @@
     },
     "version": {
       "type": "integer",
-      "enum": [1, 2, 3, 4],
+      "enum": [1, 2, 3, 4, 5],
       "description": "Config format version. Older versions are auto-migrated (and the file rewritten) on load; a version newer than this sense build supports makes it exit with an error rather than misinterpret the file. Omit to default to 1."
     },
     "presets": {
       "type": "object",
-      "description": "File selection, index-time, and named search defaults. A preset is self-contained: no inheritance between presets, and no ordering -- presets may overlap freely (they are views, not partitions), and a preset whose globs match nothing is valid. A file is indexed iff any preset's include/exclude covers it (union across presets); `default` is used when a command names no preset, and must be declared. A file is embedded when the top-level `embed` block names a model and some covering preset has `semantic` on. `sense init` generates `default` plus one `large` example.",
+      "description": "File selection, index-time, and named search defaults. A preset is self-contained: no inheritance between presets, and no ordering -- presets may overlap freely (they are views, not partitions), and a preset whose globs match nothing is valid. A file is indexed iff any preset's include/exclude covers it (union across presets); `default` is used when a command names no preset, and must be declared. A file is embedded when the top-level `embed` block names a model and some covering preset's `signals` include `vectors`. `sense init` generates `default` plus one `large` example.",
       "minProperties": 1,
       "additionalProperties": {
         "type": "object",
@@ -36,9 +36,16 @@
             "minItems": 1,
             "description": "Glob patterns excluded from this preset's `include`, tsconfig-standard semantics."
           },
-          "semantic": {
-            "type": "boolean",
-            "description": "Vectors for the files this preset covers, and for searches scoped to it. Absent or true means on; only ever meaningfully written as false. A layer searched for exact wording (ingested sources, archives, generated output) sets it false: its files cost no embedding, and searches under it run on words and links. Requires the top-level `embed` block to name a model before any preset can use vectors."
+          "signals": {
+            "type": "object",
+            "additionalProperties": false,
+            "minProperties": 1,
+            "properties": {
+              "words": { "type": "number", "exclusiveMinimum": 0 },
+              "links": { "type": "number", "exclusiveMinimum": 0 },
+              "vectors": { "type": "number", "exclusiveMinimum": 0 }
+            },
+            "description": "Exhaustive when present: every signal this preset's searches compose, each mapped to its RRF weight (a finite number > 0; presence is enablement -- words = BM25 match, links = link-graph expansion, vectors = embedding similarity). A signal's RRF contribution is `weight / (RRF_K + rank)`; weight 1 (the default for every signal) reproduces unweighted scores exactly. Absent means every signal whose prerequisite holds, each at weight 1 -- words always, links when the `links` feature is on, vectors when the top-level `embed` block names a model. `links` requires `words` (link expansion is seeded by word matches); `vectors` requires the `embed` block. A layer searched for exact wording (ingested sources, archives, generated output) declares `{\"words\": 1, \"links\": 1}`: its files cost no embedding, and searches under it run on words and links only."
           },
           "k": {
             "type": "integer",
@@ -78,11 +85,11 @@
     "embed": {
       "type": "object",
       "additionalProperties": false,
-      "description": "The vector switch and the model it uses. Present with a `model` means every indexed file gets vectors; omit the block entirely to index without vectors. `model` (required: Hugging Face id or local path), `type` (`static` = built-in pure-JS Model2Vec loader, default; `api` = OpenAI-compatible POST /embeddings), `url` (api base, e.g. http://localhost:11434/v1), `key` (name of the env var holding the bearer token). Nothing downloads the model implicitly -- run `sense download` once per machine. Changing model or type rebuilds the cache like a feature toggle.",
+      "description": "The vector switch and the model it uses. Present with a `model` means every indexed file gets vectors; omit the block entirely to index without vectors. `model` (required: Hugging Face id or local path), `provider` (`static` = built-in pure-JS Model2Vec loader, default; `openai` = OpenAI-compatible POST /embeddings, covering Ollama, LM Studio, and other compatible endpoints; `cohere` = native Cohere `/v2/embed`), `url` (openai provider base, e.g. http://localhost:11434/v1), `key` (name of the env var holding the bearer token). Nothing downloads the model implicitly -- run `sense download` once per machine. Changing model or provider rebuilds the cache like a feature toggle.",
       "required": ["model"],
       "properties": {
         "model": { "type": "string" },
-        "type": { "type": "string", "enum": ["static", "api"] },
+        "provider": { "type": "string", "enum": ["static", "openai", "cohere"] },
         "url": { "type": "string" },
         "key": { "type": "string" }
       }