sensemaking 0.23.1 → 0.24.0

package/dist/esm/output/output.js

@@ -7,8 +7,10 @@ function cell(value) {
 }
 // RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every
 // character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.
+// An array (e.g. snippets) joins with the one delimiter our content provably cannot contain,
+// inside the quoted field newlines already trigger (DESIGN.md "Rendering a list of snippets").
 function csvField(value) {
-    const text = value === null || value === undefined ? '' : String(value);
+    const text = value === null || value === undefined ? '' : Array.isArray(value) ? value.join('\n') : String(value);
     return /["\r\n,]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
 }
 function csvLine(values) {
@@ -122,19 +124,40 @@ function fitWidths(widths, limit) {
         fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));
     }
 }
+// An array cell (snippets) becomes one physical line per element; every other value stays the
+// single line cell() already produced. No cell in any other command holds one today, so a row
+// with no array-valued cell yields exactly one physical line, unchanged from before this existed.
+function cellLines(value) {
+    if (Array.isArray(value)) return value.length === 0 ? [
+        ''
+    ] : value.map((v)=>cell(v));
+    return [
+        cell(value)
+    ];
+}
 function renderRows(rows, format, width = process.stdout.columns) {
     if (format === 'json') return stringifyJson(rows, 2);
     if (rows.length === 0) return '(0 rows)';
     const columns = Object.keys(rows[0]);
-    const cells = rows.map((row)=>columns.map((col)=>cell(row[col])));
-    const natural = columns.map((col, i)=>Math.max(col.length, ...cells.map((row)=>row[i].length)));
+    const linesByCell = rows.map((row)=>columns.map((col)=>cellLines(row[col])));
+    const heights = linesByCell.map((cellsInRow)=>Math.max(1, ...cellsInRow.map((lines)=>lines.length)));
+    const natural = columns.map((col, i)=>Math.max(col.length, ...linesByCell.flatMap((cellsInRow)=>cellsInRow[i]).map((line)=>line.length)));
     const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;
     const clip = (value, w)=>value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`;
     const formatRow = (values)=>values.map((value, i)=>clip(value, widths[i])).join('  ').trimEnd();
+    // A row becomes as many physical lines as its tallest cell; other columns blank-pad on
+    // continuation lines (DESIGN.md "Rendering a list of snippets").
+    const body = [];
+    for(let r = 0; r < rows.length; r++){
+        for(let li = 0; li < heights[r]; li++)body.push(formatRow(columns.map((_, c)=>{
+            var _linesByCell_r_c_li;
+            return (_linesByCell_r_c_li = linesByCell[r][c][li]) !== null && _linesByCell_r_c_li !== void 0 ? _linesByCell_r_c_li : '';
+        })));
+    }
     return [
         formatRow(columns),
         widths.map((w)=>'-'.repeat(w)).join('  '),
-        ...cells.map(formatRow)
+        ...body
     ].join('\n');
 }
 // Text renderers for the top-level commands; cli.ts prints what these return.

package/dist/esm/output/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output keeps them as-is.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every\n// character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// JSON.stringify throws on BigInt, and stores return int64 as BigInt (DuckDB does): a value\n// that fits a safe integer stays a number, a larger one becomes its decimal string.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nconst bigintSafe = (_key: string, value: unknown): unknown => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n\nexport function stringifyJson(value: unknown, indent?: number): string {\n  return JSON.stringify(value, bigintSafe, indent);\n}\n\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Names from a row wherever there is one: the statement's column list keeps duplicates the\n    // row collapsed. Nothing at all with no row and no columns -- a bare newline is an empty record.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: the result is never held as\n// rows and again as a rendered string. Table buffers (column width depends on the last row); json/csv write row by row.\nexport async function printRowStream(rows: AsyncIterable<Row>, format: RowFormat, columns: string[]): Promise<void> {\n  const iterator = rows[Symbol.asyncIterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = await iterator.next();\n  const rest: AsyncIterable<Row> = { [Symbol.asyncIterator]: () => iterator };\n\n  if (format === 'table') {\n    const collected: Row[] = first.done ? [] : [first.value];\n    for await (const row of rest) collected.push(row);\n    printRows(collected, format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    const indent = (row: Row) =>\n      stringifyJson(row, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for await (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for await (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return stringifyJson(rows, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\n// name, or name:weight when the weight is not the default 1.\nfunction signalLabels(signals: Record<string, number>): string {\n  return Object.entries(signals)\n    .map(([name, weight]) => (weight === 1 ? name : `${name}:${weight}`))\n    .join(', ');\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a preset without the vectors signal says what it uses instead.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)];\n}\n\nexport function renderMap(result: {\n  docs: { count: number; bytes: number };\n  fields: Row[];\n  fieldsTotal: number;\n  features: { on: string[]; off: string[] };\n  presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>;\n  hubs: Row[];\n  recent: Row[];\n  recentCaveat: string | null;\n}): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","test","csvLine","values","map","join","warnOversize","bytes","rowCount","console","warn","Math","round","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","stringifyJson","indent","JSON","stringify","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","asyncIterator","first","next","rest","collected","done","push","write","process","stdout","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","cells","col","natural","i","clip","w","padEnd","slice","formatRow","trimEnd","repeat","featureOffHint","featuresLine","features","off","on","signalLabels","signals","entries","weight","presetsLines","presets","p","files","embedded","vectors","renderMap","result","parts","docs","count","fields","fieldsTotal","hubs","includes","recentCaveat","recent","renderPeek","lines","path","tokens","parseError","key","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,sGAAsG;AACtG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKJ,OAAOD;IACjE,OAAO,WAAWM,IAAI,CAACF,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASG,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACN,UAAUO,IAAI,CAAC;AACnC;AAEA,SAASC,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAASd,gBAAgB;IAC7BgB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,4FAA4F;AAC5F,oFAAoF;AACpF,MAAMK,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,MAAMC,aAAa,CAACC,MAAcvB,QAA6B,OAAOA,UAAU,WAAYA,SAAS,CAACkB,YAAYlB,SAASkB,WAAWE,OAAOpB,SAASA,MAAMwB,QAAQ,KAAMxB;AAE1K,OAAO,SAASyB,cAAczB,KAAc,EAAE0B,MAAe;IAC3D,OAAOC,KAAKC,SAAS,CAAC5B,OAAOsB,YAAYI;AAC3C;AAEA,OAAO,SAASG,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,2FAA2F;QAC3F,iGAAiG;QACjG,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAAC/B,QAAQ0B;eAAWH,KAAKrB,GAAG,CAAC,CAAC8B,MAAQhC,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASD,GAAG,CAACC,KAAK;SAAI,CAAC9B,IAAI,CAAC;QACtGI,QAAQ2B,GAAG,CAACH;QACZ,yDAAyD;QACzD3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCjB,QAAQ2B,GAAG,CAACH;IACZ3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,+FAA+F;AAC/F,wHAAwH;AACxH,OAAO,eAAeW,eAAef,IAAwB,EAAEC,MAAiB,EAAEC,OAAiB;IACjG,MAAMc,WAAWhB,IAAI,CAACiB,OAAOC,aAAa,CAAC;IAC3C,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMC,QAAQ,MAAMH,SAASI,IAAI;IACjC,MAAMC,OAA2B;QAAE,CAACJ,OAAOC,aAAa,CAAC,EAAE,IAAMF;IAAS;IAE1E,IAAIf,WAAW,SAAS;QACtB,MAAMqB,YAAmBH,MAAMI,IAAI,GAAG,EAAE,GAAG;YAACJ,MAAMjD,KAAK;SAAC;QACxD,WAAW,MAAMuC,OAAOY,KAAMC,UAAUE,IAAI,CAACf;QAC7CV,UAAUuB,WAAWrB,QAAQC;QAC7B;IACF;IAEA,IAAIpB,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM0C,QAAQ,CAACnD;QACb,yDAAyD;QACzDQ,SAAS8B,OAAOC,UAAU,CAACvC;QAC3BoD,QAAQC,MAAM,CAACF,KAAK,CAACnD;IACvB;IAEA,IAAI2B,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,MAAML,SAAS,CAACa,MACdd,cAAcc,KAAK,GAChBmB,KAAK,CAAC,MACNjD,GAAG,CAAC,CAACkD,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzBjD,IAAI,CAAC;QACV,IAAIuC,MAAMI,IAAI,EAAE;YACdE,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAM7B,OAAOuB,MAAMjD,KAAK;YACxBa,WAAW;YACX,WAAW,MAAM0B,OAAOY,KAAM;gBAC5BI,MAAM;gBACNA,MAAM7B,OAAOa;gBACb1B;YACF;YACA0C,MAAM;QACR;IACF,OAAO,IAAIN,MAAMI,IAAI,EAAE;QACrB,MAAMpB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGqB,MAAM,GAAGhD,QAAQ0B,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACa,MAAMjD,KAAK;QACrCuD,MAAM,GAAGhD,QAAQ0B,OAAO,EAAE,CAAC;QAC3BsB,MAAM,GAAGhD,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASS,MAAMjD,KAAK,CAACwC,KAAK,GAAG,EAAE,CAAC;QAC5D3B,WAAW;QACX,WAAW,MAAM0B,OAAOY,KAAM;YAC5BI,MAAM,GAAGhD,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpD3B;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM+C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAO7B,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMgC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACxD,KAAKyD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASpB,WAAWd,IAAW,EAAEC,MAAc,EAAE2C,QAAQlB,QAAQC,MAAM,CAACzB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAON,cAAcK,MAAM;IAClD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAM6C,QAAQ7C,KAAKrB,GAAG,CAAC,CAAC8B,MAAQP,QAAQvB,GAAG,CAAC,CAACmE,MAAQ7E,KAAKwC,GAAG,CAACqC,IAAI;IAClE,MAAMC,UAAU7C,QAAQvB,GAAG,CAAC,CAACmE,KAAKE,IAAM9D,KAAKyD,GAAG,CAACG,IAAI1C,MAAM,KAAKyC,MAAMlE,GAAG,CAAC,CAAC8B,MAAQA,GAAG,CAACuC,EAAE,CAAC5C,MAAM;IAChG,MAAM6B,SAASW,SAASA,QAAQd,aAAaE,UAAUe,SAASH,SAASG;IAEzE,MAAME,OAAO,CAAC/E,OAAegF,IAAehF,MAAMkC,MAAM,IAAI8C,IAAIhF,MAAMiF,MAAM,CAACD,KAAK,GAAGhF,MAAMkF,KAAK,CAAC,GAAGF,IAAI,KAAKnB,WAAW;IACxH,MAAMsB,YAAY,CAAC3E,SACjBA,OACGC,GAAG,CAAC,CAACT,OAAO8E,IAAMC,KAAK/E,OAAO+D,MAAM,CAACe,EAAE,GACvCpE,IAAI,CAAC,MACL0E,OAAO;IAEZ,OAAO;QAACD,UAAUnD;QAAU+B,OAAOtD,GAAG,CAAC,CAACuE,IAAM,IAAIK,MAAM,CAACL,IAAItE,IAAI,CAAC;WAAUiE,MAAMlE,GAAG,CAAC0E;KAAW,CAACzE,IAAI,CAAC;AACzG;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAAS4E,eAAe9C,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAAS+C,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAACvD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEsD,SAASC,GAAG,CAAChF,GAAG,CAAC,CAAC+B,OAAS,GAAGA,KAAK,EAAE,EAAE8C,eAAe9C,MAAM,CAAC,CAAC,EAAE9B,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE8E,SAASE,EAAE,CAAChF,IAAI,CAAC,QAAQ+E,KAAK;AACpD;AAEA,6DAA6D;AAC7D,SAASE,aAAaC,OAA+B;IACnD,OAAOzD,OAAO0D,OAAO,CAACD,SACnBnF,GAAG,CAAC,CAAC,CAAC+B,MAAMsD,OAAO,GAAMA,WAAW,IAAItD,OAAO,GAAGA,KAAK,CAAC,EAAEsD,QAAQ,EAClEpF,IAAI,CAAC;AACV;AAEA,OAAO,SAASqF,aAAaC,OAAkG;IAC7H,6FAA6F;IAC7F,8EAA8E;IAC9E,OAAO;QAAC;WAAeA,QAAQvF,GAAG,CAAC,CAACwF,IAAM,CAAC,EAAE,EAAEA,EAAEzD,IAAI,CAAC,EAAE,EAAEyD,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEL,OAAO,CAACQ,OAAO,KAAK/F,YAAY,KAAK,CAAC,WAAW,EAAEsF,aAAaM,EAAEL,OAAO,EAAE,CAAC,CAAC,EAAE;KAAE;AACzL;AAEA,OAAO,SAASS,UAAUC,MASzB;IACC,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEzF,KAAKC,KAAK,CAACqF,OAAOE,IAAI,CAAC5F,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE2E,aAAae,OAAOd,QAAQ;QAAG;WAAOO,aAAaO,OAAON,OAAO;QAAG;QAAIpD,WAAW0D,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACxE,MAAM,EAAEqE,MAAMjD,IAAI,CAAC,CAAC,EAAE,EAAEgD,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACxE,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIoE,OAAOM,IAAI,CAAC1E,MAAM,GAAG,GAAGqE,MAAMjD,IAAI,CAAC,0BAA0BV,WAAW0D,OAAOM,IAAI,EAAE;SACpF,IAAIN,OAAOd,QAAQ,CAACC,GAAG,CAACoB,QAAQ,CAAC,SAASN,MAAMjD,IAAI,CAAC,CAAC,aAAa,EAAEgC,eAAe,QAAQ,CAAC,CAAC;IACnGiB,MAAMjD,IAAI,CAAC;IACX,IAAIgD,OAAOQ,YAAY,EAAEP,MAAMjD,IAAI,CAACgD,OAAOQ,YAAY;IACvDP,MAAMjD,IAAI,CAACV,WAAW0D,OAAOS,MAAM,EAAE;IACrC,OAAOR,MAAM7F,IAAI,CAAC;AACpB;AAEA,OAAO,SAASsG,WAAWV,MAAoR;IAC7S,MAAMW,QAAQ;QAAC,GAAGX,OAAOY,IAAI,CAAC,IAAI,EAAEZ,OAAOa,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAIb,OAAOc,UAAU,EAAEH,MAAM3D,IAAI,CAAC,CAAC,wDAAwD,EAAEgD,OAAOc,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAKrH,MAAM,IAAImC,OAAO0D,OAAO,CAACS,OAAOgB,WAAW,EAAGL,MAAM3D,IAAI,CAAC,CAAC,EAAE,EAAE+D,IAAI,EAAE,EAAErH,OAAO;IAC9F,IAAIsG,OAAOiB,QAAQ,CAACrF,MAAM,GAAG,GAAG;QAC9B+E,MAAM3D,IAAI,CAAC,IAAI;QACf,KAAK,MAAMkE,KAAKlB,OAAOiB,QAAQ,CAAEN,MAAM3D,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI+B,MAAM,CAACmC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAIb,OAAOuB,aAAa,GAAGvB,OAAOiB,QAAQ,CAACrF,MAAM,EAAE+E,MAAM3D,IAAI,CAAC,CAAC,IAAI,EAAEgD,OAAOuB,aAAa,GAAGvB,OAAOiB,QAAQ,CAACrF,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAIoE,OAAOb,GAAG,CAACoB,QAAQ,CAAC,aAAaI,MAAM3D,IAAI,CAAC,IAAI;IAC3D,IAAIgD,OAAOb,GAAG,CAACoB,QAAQ,CAAC,UAAU;QAChCI,MAAM3D,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMwE,WAAW,CAACC,OAAeC,OAAiB7D;YAChD,IAAIA,QAAQ,GAAG8C,MAAM3D,IAAI,CAAC,GAAGyE,MAAM,EAAE,EAAE5D,MAAM,GAAG,EAAE6D,MAAMtH,IAAI,CAAC,QAAQyD,QAAQ6D,MAAM9F,MAAM,GAAG,CAAC,GAAG,EAAEiC,QAAQ6D,MAAM9F,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAIoE,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGlB,MAAM3D,IAAI,CAAC;QAC1FwE,SAAS,aAAaxB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;QAC3DH,SAAS,cAAcxB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;QAChEJ,SAAS,aAAaxB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC/D;IACA,OAAOlB,MAAMvG,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output keeps them as-is.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every\n// character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.\n// An array (e.g. snippets) joins with the one delimiter our content provably cannot contain,\n// inside the quoted field newlines already trigger (DESIGN.md \"Rendering a list of snippets\").\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : Array.isArray(value) ? value.join('\\n') : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// JSON.stringify throws on BigInt, and stores return int64 as BigInt (DuckDB does): a value\n// that fits a safe integer stays a number, a larger one becomes its decimal string.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nconst bigintSafe = (_key: string, value: unknown): unknown => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n\nexport function stringifyJson(value: unknown, indent?: number): string {\n  return JSON.stringify(value, bigintSafe, indent);\n}\n\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Names from a row wherever there is one: the statement's column list keeps duplicates the\n    // row collapsed. Nothing at all with no row and no columns -- a bare newline is an empty record.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: the result is never held as\n// rows and again as a rendered string. Table buffers (column width depends on the last row); json/csv write row by row.\nexport async function printRowStream(rows: AsyncIterable<Row>, format: RowFormat, columns: string[]): Promise<void> {\n  const iterator = rows[Symbol.asyncIterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = await iterator.next();\n  const rest: AsyncIterable<Row> = { [Symbol.asyncIterator]: () => iterator };\n\n  if (format === 'table') {\n    const collected: Row[] = first.done ? [] : [first.value];\n    for await (const row of rest) collected.push(row);\n    printRows(collected, format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    const indent = (row: Row) =>\n      stringifyJson(row, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for await (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for await (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\n// An array cell (snippets) becomes one physical line per element; every other value stays the\n// single line cell() already produced. No cell in any other command holds one today, so a row\n// with no array-valued cell yields exactly one physical line, unchanged from before this existed.\nfunction cellLines(value: unknown): string[] {\n  if (Array.isArray(value)) return value.length === 0 ? [''] : value.map((v) => cell(v));\n  return [cell(value)];\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return stringifyJson(rows, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const linesByCell = rows.map((row) => columns.map((col) => cellLines(row[col])));\n  const heights = linesByCell.map((cellsInRow) => Math.max(1, ...cellsInRow.map((lines) => lines.length)));\n  const natural = columns.map((col, i) => Math.max(col.length, ...linesByCell.flatMap((cellsInRow) => cellsInRow[i]).map((line) => line.length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  // A row becomes as many physical lines as its tallest cell; other columns blank-pad on\n  // continuation lines (DESIGN.md \"Rendering a list of snippets\").\n  const body: string[] = [];\n  for (let r = 0; r < rows.length; r++) {\n    for (let li = 0; li < heights[r]; li++) body.push(formatRow(columns.map((_, c) => linesByCell[r][c][li] ?? '')));\n  }\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...body].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\n// name, or name:weight when the weight is not the default 1.\nfunction signalLabels(signals: Record<string, number>): string {\n  return Object.entries(signals)\n    .map(([name, weight]) => (weight === 1 ? name : `${name}:${weight}`))\n    .join(', ');\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a preset without the vectors signal says what it uses instead.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)];\n}\n\nexport function renderMap(result: {\n  docs: { count: number; bytes: number };\n  fields: Row[];\n  fieldsTotal: number;\n  features: { on: string[]; off: string[] };\n  presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>;\n  hubs: Row[];\n  recent: Row[];\n  recentCaveat: string | null;\n}): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","Array","isArray","join","test","csvLine","values","map","warnOversize","bytes","rowCount","console","warn","Math","round","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","stringifyJson","indent","JSON","stringify","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","asyncIterator","first","next","rest","collected","done","push","write","process","stdout","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","cellLines","v","width","linesByCell","col","heights","cellsInRow","lines","natural","i","flatMap","clip","w","padEnd","slice","formatRow","trimEnd","body","r","li","_","c","repeat","featureOffHint","featuresLine","features","off","on","signalLabels","signals","entries","weight","presetsLines","presets","p","files","embedded","vectors","renderMap","result","parts","docs","count","fields","fieldsTotal","hubs","includes","recentCaveat","recent","renderPeek","path","tokens","parseError","key","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,sGAAsG;AACtG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,6FAA6F;AAC7F,+FAA+F;AAC/F,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKC,MAAMC,OAAO,CAACP,SAASA,MAAMQ,IAAI,CAAC,QAAQP,OAAOD;IAC3G,OAAO,WAAWS,IAAI,CAACL,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASM,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACT,UAAUK,IAAI,CAAC;AACnC;AAEA,SAASK,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAAShB,gBAAgB;IAC7BkB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,4FAA4F;AAC5F,oFAAoF;AACpF,MAAMK,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,MAAMC,aAAa,CAACC,MAAczB,QAA6B,OAAOA,UAAU,WAAYA,SAAS,CAACoB,YAAYpB,SAASoB,WAAWE,OAAOtB,SAASA,MAAM0B,QAAQ,KAAM1B;AAE1K,OAAO,SAAS2B,cAAc3B,KAAc,EAAE4B,MAAe;IAC3D,OAAOC,KAAKC,SAAS,CAAC9B,OAAOwB,YAAYI;AAC3C;AAEA,OAAO,SAASG,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,2FAA2F;QAC3F,iGAAiG;QACjG,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAAC9B,QAAQyB;eAAWH,KAAKpB,GAAG,CAAC,CAAC6B,MAAQ/B,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASD,GAAG,CAACC,KAAK;SAAI,CAAClC,IAAI,CAAC;QACtGQ,QAAQ2B,GAAG,CAACH;QACZ,yDAAyD;QACzD3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCjB,QAAQ2B,GAAG,CAACH;IACZ3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,+FAA+F;AAC/F,wHAAwH;AACxH,OAAO,eAAeW,eAAef,IAAwB,EAAEC,MAAiB,EAAEC,OAAiB;IACjG,MAAMc,WAAWhB,IAAI,CAACiB,OAAOC,aAAa,CAAC;IAC3C,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMC,QAAQ,MAAMH,SAASI,IAAI;IACjC,MAAMC,OAA2B;QAAE,CAACJ,OAAOC,aAAa,CAAC,EAAE,IAAMF;IAAS;IAE1E,IAAIf,WAAW,SAAS;QACtB,MAAMqB,YAAmBH,MAAMI,IAAI,GAAG,EAAE,GAAG;YAACJ,MAAMnD,KAAK;SAAC;QACxD,WAAW,MAAMyC,OAAOY,KAAMC,UAAUE,IAAI,CAACf;QAC7CV,UAAUuB,WAAWrB,QAAQC;QAC7B;IACF;IAEA,IAAIpB,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM0C,QAAQ,CAACrD;QACb,yDAAyD;QACzDU,SAAS8B,OAAOC,UAAU,CAACzC;QAC3BsD,QAAQC,MAAM,CAACF,KAAK,CAACrD;IACvB;IAEA,IAAI6B,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,MAAML,SAAS,CAACa,MACdd,cAAcc,KAAK,GAChBmB,KAAK,CAAC,MACNhD,GAAG,CAAC,CAACiD,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzBrD,IAAI,CAAC;QACV,IAAI2C,MAAMI,IAAI,EAAE;YACdE,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAM7B,OAAOuB,MAAMnD,KAAK;YACxBe,WAAW;YACX,WAAW,MAAM0B,OAAOY,KAAM;gBAC5BI,MAAM;gBACNA,MAAM7B,OAAOa;gBACb1B;YACF;YACA0C,MAAM;QACR;IACF,OAAO,IAAIN,MAAMI,IAAI,EAAE;QACrB,MAAMpB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGqB,MAAM,GAAG/C,QAAQyB,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACa,MAAMnD,KAAK;QACrCyD,MAAM,GAAG/C,QAAQyB,OAAO,EAAE,CAAC;QAC3BsB,MAAM,GAAG/C,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASS,MAAMnD,KAAK,CAAC0C,KAAK,GAAG,EAAE,CAAC;QAC5D3B,WAAW;QACX,WAAW,MAAM0B,OAAOY,KAAM;YAC5BI,MAAM,GAAG/C,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpD3B;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM+C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAO7B,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMgC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACxD,KAAKyD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,8FAA8F;AAC9F,8FAA8F;AAC9F,kGAAkG;AAClG,SAASU,UAAU5E,KAAc;IAC/B,IAAIM,MAAMC,OAAO,CAACP,QAAQ,OAAOA,MAAMoC,MAAM,KAAK,IAAI;QAAC;KAAG,GAAGpC,MAAMY,GAAG,CAAC,CAACiE,IAAM9E,KAAK8E;IACnF,OAAO;QAAC9E,KAAKC;KAAO;AACtB;AAEA,SAAS8C,WAAWd,IAAW,EAAEC,MAAc,EAAE6C,QAAQpB,QAAQC,MAAM,CAACzB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAON,cAAcK,MAAM;IAClD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAM+C,cAAc/C,KAAKpB,GAAG,CAAC,CAAC6B,MAAQP,QAAQtB,GAAG,CAAC,CAACoE,MAAQJ,UAAUnC,GAAG,CAACuC,IAAI;IAC7E,MAAMC,UAAUF,YAAYnE,GAAG,CAAC,CAACsE,aAAehE,KAAKyD,GAAG,CAAC,MAAMO,WAAWtE,GAAG,CAAC,CAACuE,QAAUA,MAAM/C,MAAM;IACrG,MAAMgD,UAAUlD,QAAQtB,GAAG,CAAC,CAACoE,KAAKK,IAAMnE,KAAKyD,GAAG,CAACK,IAAI5C,MAAM,KAAK2C,YAAYO,OAAO,CAAC,CAACJ,aAAeA,UAAU,CAACG,EAAE,EAAEzE,GAAG,CAAC,CAACiD,OAASA,KAAKzB,MAAM;IAC5I,MAAM6B,SAASa,SAASA,QAAQhB,aAAaE,UAAUoB,SAASN,SAASM;IAEzE,MAAMG,OAAO,CAACvF,OAAewF,IAAexF,MAAMoC,MAAM,IAAIoD,IAAIxF,MAAMyF,MAAM,CAACD,KAAK,GAAGxF,MAAM0F,KAAK,CAAC,GAAGF,IAAI,KAAKzB,WAAW;IACxH,MAAM4B,YAAY,CAAChF,SACjBA,OACGC,GAAG,CAAC,CAACZ,OAAOqF,IAAME,KAAKvF,OAAOiE,MAAM,CAACoB,EAAE,GACvC7E,IAAI,CAAC,MACLoF,OAAO;IAEZ,uFAAuF;IACvF,iEAAiE;IACjE,MAAMC,OAAiB,EAAE;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAI9D,KAAKI,MAAM,EAAE0D,IAAK;QACpC,IAAK,IAAIC,KAAK,GAAGA,KAAKd,OAAO,CAACa,EAAE,EAAEC,KAAMF,KAAKrC,IAAI,CAACmC,UAAUzD,QAAQtB,GAAG,CAAC,CAACoF,GAAGC;gBAAMlB;oBAAAA,sBAAAA,WAAW,CAACe,EAAE,CAACG,EAAE,CAACF,GAAG,cAArBhB,iCAAAA,sBAAyB;;IAC7G;IAEA,OAAO;QAACY,UAAUzD;QAAU+B,OAAOrD,GAAG,CAAC,CAAC4E,IAAM,IAAIU,MAAM,CAACV,IAAIhF,IAAI,CAAC;WAAUqF;KAAK,CAACrF,IAAI,CAAC;AACzF;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAAS2F,eAAezD,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAAS0D,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAAClE,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEiE,SAASC,GAAG,CAAC1F,GAAG,CAAC,CAAC8B,OAAS,GAAGA,KAAK,EAAE,EAAEyD,eAAezD,MAAM,CAAC,CAAC,EAAElC,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE6F,SAASE,EAAE,CAAC/F,IAAI,CAAC,QAAQ8F,KAAK;AACpD;AAEA,6DAA6D;AAC7D,SAASE,aAAaC,OAA+B;IACnD,OAAOpE,OAAOqE,OAAO,CAACD,SACnB7F,GAAG,CAAC,CAAC,CAAC8B,MAAMiE,OAAO,GAAMA,WAAW,IAAIjE,OAAO,GAAGA,KAAK,CAAC,EAAEiE,QAAQ,EAClEnG,IAAI,CAAC;AACV;AAEA,OAAO,SAASoG,aAAaC,OAAkG;IAC7H,6FAA6F;IAC7F,8EAA8E;IAC9E,OAAO;QAAC;WAAeA,QAAQjG,GAAG,CAAC,CAACkG,IAAM,CAAC,EAAE,EAAEA,EAAEpE,IAAI,CAAC,EAAE,EAAEoE,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEL,OAAO,CAACQ,OAAO,KAAK5G,YAAY,KAAK,CAAC,WAAW,EAAEmG,aAAaM,EAAEL,OAAO,EAAE,CAAC,CAAC,EAAE;KAAE;AACzL;AAEA,OAAO,SAASS,UAAUC,MASzB;IACC,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEpG,KAAKC,KAAK,CAACgG,OAAOE,IAAI,CAACvG,KAAK,GAAG,MAAM,IAAI,CAAC;QAAEsF,aAAae,OAAOd,QAAQ;QAAG;WAAOO,aAAaO,OAAON,OAAO;QAAG;QAAI/D,WAAWqE,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACnF,MAAM,EAAEgF,MAAM5D,IAAI,CAAC,CAAC,EAAE,EAAE2D,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACnF,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI+E,OAAOM,IAAI,CAACrF,MAAM,GAAG,GAAGgF,MAAM5D,IAAI,CAAC,0BAA0BV,WAAWqE,OAAOM,IAAI,EAAE;SACpF,IAAIN,OAAOd,QAAQ,CAACC,GAAG,CAACoB,QAAQ,CAAC,SAASN,MAAM5D,IAAI,CAAC,CAAC,aAAa,EAAE2C,eAAe,QAAQ,CAAC,CAAC;IACnGiB,MAAM5D,IAAI,CAAC;IACX,IAAI2D,OAAOQ,YAAY,EAAEP,MAAM5D,IAAI,CAAC2D,OAAOQ,YAAY;IACvDP,MAAM5D,IAAI,CAACV,WAAWqE,OAAOS,MAAM,EAAE;IACrC,OAAOR,MAAM5G,IAAI,CAAC;AACpB;AAEA,OAAO,SAASqH,WAAWV,MAAoR;IAC7S,MAAMhC,QAAQ;QAAC,GAAGgC,OAAOW,IAAI,CAAC,IAAI,EAAEX,OAAOY,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAIZ,OAAOa,UAAU,EAAE7C,MAAM3B,IAAI,CAAC,CAAC,wDAAwD,EAAE2D,OAAOa,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAKjI,MAAM,IAAIqC,OAAOqE,OAAO,CAACS,OAAOe,WAAW,EAAG/C,MAAM3B,IAAI,CAAC,CAAC,EAAE,EAAEyE,IAAI,EAAE,EAAEjI,OAAO;IAC9F,IAAImH,OAAOgB,QAAQ,CAAC/F,MAAM,GAAG,GAAG;QAC9B+C,MAAM3B,IAAI,CAAC,IAAI;QACf,KAAK,MAAM4E,KAAKjB,OAAOgB,QAAQ,CAAEhD,MAAM3B,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI0C,MAAM,CAACkC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAIZ,OAAOsB,aAAa,GAAGtB,OAAOgB,QAAQ,CAAC/F,MAAM,EAAE+C,MAAM3B,IAAI,CAAC,CAAC,IAAI,EAAE2D,OAAOsB,aAAa,GAAGtB,OAAOgB,QAAQ,CAAC/F,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAI+E,OAAOb,GAAG,CAACoB,QAAQ,CAAC,aAAavC,MAAM3B,IAAI,CAAC,IAAI;IAC3D,IAAI2D,OAAOb,GAAG,CAACoB,QAAQ,CAAC,UAAU;QAChCvC,MAAM3B,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMkF,WAAW,CAACC,OAAeC,OAAiBvE;YAChD,IAAIA,QAAQ,GAAGc,MAAM3B,IAAI,CAAC,GAAGmF,MAAM,EAAE,EAAEtE,MAAM,GAAG,EAAEuE,MAAMpI,IAAI,CAAC,QAAQ6D,QAAQuE,MAAMxG,MAAM,GAAG,CAAC,GAAG,EAAEiC,QAAQuE,MAAMxG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAI+E,OAAO0B,aAAa,GAAG1B,OAAO2B,eAAe,GAAG3B,OAAO4B,cAAc,GAAG,GAAG5D,MAAM3B,IAAI,CAAC;QAC1FkF,SAAS,aAAavB,OAAO6B,QAAQ,EAAE7B,OAAO0B,aAAa;QAC3DH,SAAS,cAAcvB,OAAO8B,UAAU,EAAE9B,OAAO2B,eAAe;QAChEJ,SAAS,aAAavB,OAAO+B,SAAS,EAAE/B,OAAO4B,cAAc;IAC/D;IACA,OAAO5D,MAAM3E,IAAI,CAAC;AACpB"}

package/dist/esm/output/search-error.d.ts

@@ -1 +1,2 @@
+export declare function bareTermSyntaxError(terms: string): Error | null;
 export declare function searchError(err: Error, terms: string, scope?: string): Error;

package/dist/esm/output/search-error.js

@@ -1,7 +1,23 @@
 import { SenseError } from '../errors.js';
+import { hasUnspacedRun } from '../text/segment.js';
 // FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and
 // errors `no such column: to` -- true about the parse, misleading about the input.
-const FTS5_PUNCTUATION = /[^\p{L}\p{N}_\s]/u;
+const FTS5_PUNCTUATION = /[^\p{L}\p{M}\p{N}_\s]/u;
+// SQLite treats punctuation in an unquoted spaced-script token as MATCH syntax. Validate that
+// boundary before dispatch so DuckDB and Tantivy do not silently interpret the same input as text.
+export function bareTermSyntaxError(terms) {
+    var _outsideQuotes_match;
+    const outsideQuotes = terms.replace(/"[^"]*"/g, ' ');
+    const suspects = ((_outsideQuotes_match = outsideQuotes.match(/\S+/g)) !== null && _outsideQuotes_match !== void 0 ? _outsideQuotes_match : []).filter((token)=>{
+        if (!FTS5_PUNCTUATION.test(token) || hasUnspacedRun(token)) return false;
+        if (/^[\p{L}_]\w*\s*:/u.test(token)) return false;
+        if (/^[\p{L}\p{N}_]+\*(?:\s|$)/u.test(token)) return false;
+        if (/^(?:AND|OR|NOT)$/u.test(token)) return false;
+        return true;
+    });
+    if (suspects.length === 0) return null;
+    return new SenseError('SEARCH_SYNTAX', `the punctuation in ${suspects.map((term)=>`\`${term}\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '"${suspects[0]}"'. Searchable columns are title, summary, text.`);
+}
 export function searchError(err, terms, scope) {
     var _terms_match;
     var _exec;

package/dist/esm/output/search-error.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/search-error.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\n\n// FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and\n// errors `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_PUNCTUATION = /[^\\p{L}\\p{N}_\\s]/u;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_PUNCTUATION.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where can raise \"no such column\" through this same statement.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","FTS5_PUNCTUATION","searchError","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","map","join"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAE1C,0FAA0F;AAC1F,mFAAmF;AACnF,MAAMC,mBAAmB;AAEzB,OAAO,SAASC,YAAYC,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAGN;IALZ,MAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,MAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQX,iBAAiBM,IAAI,CAACI;IACvG,wFAAwF;IACxF,qEAAqE;IACrE,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,CAACP,IAAMA,EAAEQ,KAAK,CAAC,mBAAmBC,QAAQ,CAACP;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIjB,WAAW,iBAAiB,GAAGM,QAAQ,uBAAuB,EAAEE,SAASa,GAAG,CAAC,CAACV,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEW,IAAI,CAAC,MAAM,gFAAgF,EAAEd,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIhB,WAAW,iBAAiB,GAAGM,QAAQ,yBAAyB,EAAED,MAAM,iHAAiH,CAAC;IACvM;IACA,OAAO,IAAIL,WAAW,iBAAiB,GAAGM,QAAQ,4JAA4J,CAAC;AACjN"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/search-error.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { hasUnspacedRun } from '../text/segment.ts';\n\n// FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and\n// errors `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_PUNCTUATION = /[^\\p{L}\\p{M}\\p{N}_\\s]/u;\n\n// SQLite treats punctuation in an unquoted spaced-script token as MATCH syntax. Validate that\n// boundary before dispatch so DuckDB and Tantivy do not silently interpret the same input as text.\nexport function bareTermSyntaxError(terms: string): Error | null {\n  const outsideQuotes = terms.replace(/\"[^\"]*\"/g, ' ');\n  const suspects = (outsideQuotes.match(/\\S+/g) ?? []).filter((token) => {\n    if (!FTS5_PUNCTUATION.test(token) || hasUnspacedRun(token)) return false;\n    if (/^[\\p{L}_]\\w*\\s*:/u.test(token)) return false;\n    if (/^[\\p{L}\\p{N}_]+\\*(?:\\s|$)/u.test(token)) return false;\n    if (/^(?:AND|OR|NOT)$/u.test(token)) return false;\n    return true;\n  });\n  if (suspects.length === 0) return null;\n  return new SenseError('SEARCH_SYNTAX', `the punctuation in ${suspects.map((term) => `\\`${term}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n}\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_PUNCTUATION.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where can raise \"no such column\" through this same statement.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","hasUnspacedRun","FTS5_PUNCTUATION","bareTermSyntaxError","terms","outsideQuotes","replace","suspects","match","filter","token","test","length","map","term","join","searchError","err","scope","message","t","startsWith","col","exec","fromTerms","undefined","some","split","includes"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,qBAAqB;AAEpD,0FAA0F;AAC1F,mFAAmF;AACnF,MAAMC,mBAAmB;AAEzB,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,SAASC,oBAAoBC,KAAa;QAE7BC;IADlB,MAAMA,gBAAgBD,MAAME,OAAO,CAAC,YAAY;IAChD,MAAMC,WAAW,EAACF,uBAAAA,cAAcG,KAAK,CAAC,qBAApBH,kCAAAA,uBAA+B,EAAE,EAAEI,MAAM,CAAC,CAACC;QAC3D,IAAI,CAACR,iBAAiBS,IAAI,CAACD,UAAUT,eAAeS,QAAQ,OAAO;QACnE,IAAI,oBAAoBC,IAAI,CAACD,QAAQ,OAAO;QAC5C,IAAI,6BAA6BC,IAAI,CAACD,QAAQ,OAAO;QACrD,IAAI,oBAAoBC,IAAI,CAACD,QAAQ,OAAO;QAC5C,OAAO;IACT;IACA,IAAIH,SAASK,MAAM,KAAK,GAAG,OAAO;IAClC,OAAO,IAAIZ,WAAW,iBAAiB,CAAC,mBAAmB,EAAEO,SAASM,GAAG,CAAC,CAACC,OAAS,CAAC,EAAE,EAAEA,KAAK,EAAE,CAAC,EAAEC,IAAI,CAAC,MAAM,gFAAgF,EAAER,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;AAC/P;AAEA,OAAO,SAASS,YAAYC,GAAU,EAAEb,KAAa,EAAEc,KAAc;QAGjDd;QAGN;IALZ,MAAMe,UAAUF,IAAIE,OAAO;IAC3B,IAAI,CAAC,oDAAoDR,IAAI,CAACQ,UAAU,OAAOF;IAC/E,MAAMV,WAAW,EAACH,eAAAA,MAAMI,KAAK,CAAC,qBAAZJ,0BAAAA,eAAuB,EAAE,EAAEK,MAAM,CAAC,CAACW,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQnB,iBAAiBS,IAAI,CAACS;IACvG,wFAAwF;IACxF,qEAAqE;IACrE,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACJ,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMK,YAAYF,QAAQG,YAAYlB,SAASK,MAAM,GAAG,IAAIL,SAASmB,IAAI,CAAC,CAACN,IAAMA,EAAEO,KAAK,CAAC,mBAAmBC,QAAQ,CAACN;IACrH,IAAIE,aAAajB,SAASK,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIZ,WAAW,iBAAiB,GAAGmB,QAAQ,uBAAuB,EAAEZ,SAASM,GAAG,CAAC,CAACO,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEL,IAAI,CAAC,MAAM,gFAAgF,EAAER,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIe,QAAQG,aAAaP,UAAUO,WAAW;QAC5C,OAAO,IAAIzB,WAAW,iBAAiB,GAAGmB,QAAQ,yBAAyB,EAAED,MAAM,iHAAiH,CAAC;IACvM;IACA,OAAO,IAAIlB,WAAW,iBAAiB,GAAGmB,QAAQ,4JAA4J,CAAC;AACjN"}

package/dist/esm/scan/frontmatter.js

@@ -92,7 +92,7 @@ function warnStringifiedKeys(relPath, doc, warnings) {
     if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);
 }
 // Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.
-// yaml's message continues onto a source excerpt, so the first line is the sentence, minus the colon that introduced the part being dropped.
+// yaml's message continues onto a source snippet, so the first line is the sentence, minus the colon that introduced the part being dropped.
 function firstLine(message) {
     return message.split('\n')[0].replace(/:\s*$/, '');
 }

package/dist/esm/scan/frontmatter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source excerpt, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","countLines","s","n","i","charCodeAt","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,qGAAqG;AACrG,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gGAAgG;AAChG,qDAAqD;AACrD,OAAO,SAAS6B,WAAWC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEb,MAAM,EAAEe,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAEA,gEAAgE;AAChE,OAAO,SAASG,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIjB,KAAK,CAACkB,IAAI,CAAC,EAAE,CAACnB,MAAM;IACrC,MAAMwB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKzC,WAAW,OAAO;QAAEqC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKtB,KAAK,CAAC,GAAGuB,MAAMC,KAAK;QAAGH,MAAMC,KAAKtB,KAAK,CAACuB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACxB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAAS0B,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCpD,MAAMkD,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACzD,aAAayD,KAAKC,GAAG,GAAG,OAAOlD;YACpC8C,QAAQ;YACR,OAAOpD,MAAMyD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAACrD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASsD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMnD,cAAc4C,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAChE,oBAAoBiE,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQ1C,MAAM,GAAG,GAAG;QACtB,MAAMgD,SAASN,QAAQ1C,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE0C,QAAQ1C,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMiD,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASlE,WAAW,OAAO;QAAEkE,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source snippet, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","countLines","s","n","i","charCodeAt","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,qGAAqG;AACrG,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gGAAgG;AAChG,qDAAqD;AACrD,OAAO,SAAS6B,WAAWC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEb,MAAM,EAAEe,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAEA,gEAAgE;AAChE,OAAO,SAASG,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIjB,KAAK,CAACkB,IAAI,CAAC,EAAE,CAACnB,MAAM;IACrC,MAAMwB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKzC,WAAW,OAAO;QAAEqC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKtB,KAAK,CAAC,GAAGuB,MAAMC,KAAK;QAAGH,MAAMC,KAAKtB,KAAK,CAACuB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACxB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAAS0B,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCpD,MAAMkD,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACzD,aAAayD,KAAKC,GAAG,GAAG,OAAOlD;YACpC8C,QAAQ;YACR,OAAOpD,MAAMyD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAACrD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASsD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMnD,cAAc4C,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAChE,oBAAoBiE,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQ1C,MAAM,GAAG,GAAG;QACtB,MAAMgD,SAASN,QAAQ1C,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE0C,QAAQ1C,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMiD,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASlE,WAAW,OAAO;QAAEkE,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}

package/dist/esm/scan/index.js

@@ -6,6 +6,7 @@ import { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_
 export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
 export { listFiles, toPosixPath } from './list.js';
 export function parseFile(file, extractors = [], cfg) {
+    // Preserve the authored bytes: feature extractors and section offsets use the raw document.
     const raw = readFileSync(file.absPath, 'utf8');
     const warnings = [];
     const { fm, body: content } = splitFrontmatter(raw);

package/dist/esm/scan/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors and section offsets use the raw document.\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,4FAA4F;IAC5F,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}

package/dist/esm/store/duckdb/fieldStats.js

@@ -7,13 +7,10 @@ function classifyVariantType(name, field) {
     if (name === 'VARCHAR') return 'text';
     throw new Error(`duckdb: variant_typeof() returned unrecognized type "${name}" for column "${field}"`);
 }
-// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in
-// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same
-// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:
-// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking
-// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no
-// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300
-// columns runs in 7ms.
+// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates
+// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);
+// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over
+// 300 columns runs in 7ms.
 const AGGREGATES_PER_QUERY = 16;
 // COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).
 // '|' is the separator, not ',': DECIMAL's display name ("DECIMAL(4, 2)") contains a comma.

package/dist/esm/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in\n// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same\n// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:\n// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking\n// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no\n// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300\n// columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,qGAAqG;AACrG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,uBAAuB;AACvB,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates\n// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);\n// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over\n// 300 columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,kGAAkG;AAClG,iGAAiG;AACjG,kGAAkG;AAClG,2BAA2B;AAC3B,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}

package/dist/esm/store/duckdb/lexical.d.ts

@@ -2,10 +2,10 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
-export declare function markContentStale(conn: Connection): void;
+export declare function markContentStale(conn: Connection): Promise<void>;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;
-    markStale: () => void;
+    markStale: () => Promise<void>;
 };
 export {};