sensemaking 0.23.1 → 0.24.1

package/dist/esm/output/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output keeps them as-is.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every\n// character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// JSON.stringify throws on BigInt, and stores return int64 as BigInt (DuckDB does): a value\n// that fits a safe integer stays a number, a larger one becomes its decimal string.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nconst bigintSafe = (_key: string, value: unknown): unknown => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n\nexport function stringifyJson(value: unknown, indent?: number): string {\n  return JSON.stringify(value, bigintSafe, indent);\n}\n\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Names from a row wherever there is one: the statement's column list keeps duplicates the\n    // row collapsed. Nothing at all with no row and no columns -- a bare newline is an empty record.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: the result is never held as\n// rows and again as a rendered string. Table buffers (column width depends on the last row); json/csv write row by row.\nexport async function printRowStream(rows: AsyncIterable<Row>, format: RowFormat, columns: string[]): Promise<void> {\n  const iterator = rows[Symbol.asyncIterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = await iterator.next();\n  const rest: AsyncIterable<Row> = { [Symbol.asyncIterator]: () => iterator };\n\n  if (format === 'table') {\n    const collected: Row[] = first.done ? [] : [first.value];\n    for await (const row of rest) collected.push(row);\n    printRows(collected, format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    const indent = (row: Row) =>\n      stringifyJson(row, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for await (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for await (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return stringifyJson(rows, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\n// name, or name:weight when the weight is not the default 1.\nfunction signalLabels(signals: Record<string, number>): string {\n  return Object.entries(signals)\n    .map(([name, weight]) => (weight === 1 ? name : `${name}:${weight}`))\n    .join(', ');\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a preset without the vectors signal says what it uses instead.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)];\n}\n\nexport function renderMap(result: {\n  docs: { count: number; bytes: number };\n  fields: Row[];\n  fieldsTotal: number;\n  features: { on: string[]; off: string[] };\n  presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>;\n  hubs: Row[];\n  recent: Row[];\n  recentCaveat: string | null;\n}): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","test","csvLine","values","map","join","warnOversize","bytes","rowCount","console","warn","Math","round","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","stringifyJson","indent","JSON","stringify","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","asyncIterator","first","next","rest","collected","done","push","write","process","stdout","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","cells","col","natural","i","clip","w","padEnd","slice","formatRow","trimEnd","repeat","featureOffHint","featuresLine","features","off","on","signalLabels","signals","entries","weight","presetsLines","presets","p","files","embedded","vectors","renderMap","result","parts","docs","count","fields","fieldsTotal","hubs","includes","recentCaveat","recent","renderPeek","lines","path","tokens","parseError","key","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,sGAAsG;AACtG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKJ,OAAOD;IACjE,OAAO,WAAWM,IAAI,CAACF,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASG,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACN,UAAUO,IAAI,CAAC;AACnC;AAEA,SAASC,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAASd,gBAAgB;IAC7BgB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,4FAA4F;AAC5F,oFAAoF;AACpF,MAAMK,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,MAAMC,aAAa,CAACC,MAAcvB,QAA6B,OAAOA,UAAU,WAAYA,SAAS,CAACkB,YAAYlB,SAASkB,WAAWE,OAAOpB,SAASA,MAAMwB,QAAQ,KAAMxB;AAE1K,OAAO,SAASyB,cAAczB,KAAc,EAAE0B,MAAe;IAC3D,OAAOC,KAAKC,SAAS,CAAC5B,OAAOsB,YAAYI;AAC3C;AAEA,OAAO,SAASG,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,2FAA2F;QAC3F,iGAAiG;QACjG,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAAC/B,QAAQ0B;eAAWH,KAAKrB,GAAG,CAAC,CAAC8B,MAAQhC,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASD,GAAG,CAACC,KAAK;SAAI,CAAC9B,IAAI,CAAC;QACtGI,QAAQ2B,GAAG,CAACH;QACZ,yDAAyD;QACzD3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCjB,QAAQ2B,GAAG,CAACH;IACZ3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,+FAA+F;AAC/F,wHAAwH;AACxH,OAAO,eAAeW,eAAef,IAAwB,EAAEC,MAAiB,EAAEC,OAAiB;IACjG,MAAMc,WAAWhB,IAAI,CAACiB,OAAOC,aAAa,CAAC;IAC3C,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMC,QAAQ,MAAMH,SAASI,IAAI;IACjC,MAAMC,OAA2B;QAAE,CAACJ,OAAOC,aAAa,CAAC,EAAE,IAAMF;IAAS;IAE1E,IAAIf,WAAW,SAAS;QACtB,MAAMqB,YAAmBH,MAAMI,IAAI,GAAG,EAAE,GAAG;YAACJ,MAAMjD,KAAK;SAAC;QACxD,WAAW,MAAMuC,OAAOY,KAAMC,UAAUE,IAAI,CAACf;QAC7CV,UAAUuB,WAAWrB,QAAQC;QAC7B;IACF;IAEA,IAAIpB,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM0C,QAAQ,CAACnD;QACb,yDAAyD;QACzDQ,SAAS8B,OAAOC,UAAU,CAACvC;QAC3BoD,QAAQC,MAAM,CAACF,KAAK,CAACnD;IACvB;IAEA,IAAI2B,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,MAAML,SAAS,CAACa,MACdd,cAAcc,KAAK,GAChBmB,KAAK,CAAC,MACNjD,GAAG,CAAC,CAACkD,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzBjD,IAAI,CAAC;QACV,IAAIuC,MAAMI,IAAI,EAAE;YACdE,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAM7B,OAAOuB,MAAMjD,KAAK;YACxBa,WAAW;YACX,WAAW,MAAM0B,OAAOY,KAAM;gBAC5BI,MAAM;gBACNA,MAAM7B,OAAOa;gBACb1B;YACF;YACA0C,MAAM;QACR;IACF,OAAO,IAAIN,MAAMI,IAAI,EAAE;QACrB,MAAMpB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGqB,MAAM,GAAGhD,QAAQ0B,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACa,MAAMjD,KAAK;QACrCuD,MAAM,GAAGhD,QAAQ0B,OAAO,EAAE,CAAC;QAC3BsB,MAAM,GAAGhD,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASS,MAAMjD,KAAK,CAACwC,KAAK,GAAG,EAAE,CAAC;QAC5D3B,WAAW;QACX,WAAW,MAAM0B,OAAOY,KAAM;YAC5BI,MAAM,GAAGhD,QAAQ0B,MAAMxB,GAAG,CAAC,CAAC+B,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpD3B;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM+C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAO7B,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMgC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACxD,KAAKyD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASpB,WAAWd,IAAW,EAAEC,MAAc,EAAE2C,QAAQlB,QAAQC,MAAM,CAACzB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAON,cAAcK,MAAM;IAClD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAM6C,QAAQ7C,KAAKrB,GAAG,CAAC,CAAC8B,MAAQP,QAAQvB,GAAG,CAAC,CAACmE,MAAQ7E,KAAKwC,GAAG,CAACqC,IAAI;IAClE,MAAMC,UAAU7C,QAAQvB,GAAG,CAAC,CAACmE,KAAKE,IAAM9D,KAAKyD,GAAG,CAACG,IAAI1C,MAAM,KAAKyC,MAAMlE,GAAG,CAAC,CAAC8B,MAAQA,GAAG,CAACuC,EAAE,CAAC5C,MAAM;IAChG,MAAM6B,SAASW,SAASA,QAAQd,aAAaE,UAAUe,SAASH,SAASG;IAEzE,MAAME,OAAO,CAAC/E,OAAegF,IAAehF,MAAMkC,MAAM,IAAI8C,IAAIhF,MAAMiF,MAAM,CAACD,KAAK,GAAGhF,MAAMkF,KAAK,CAAC,GAAGF,IAAI,KAAKnB,WAAW;IACxH,MAAMsB,YAAY,CAAC3E,SACjBA,OACGC,GAAG,CAAC,CAACT,OAAO8E,IAAMC,KAAK/E,OAAO+D,MAAM,CAACe,EAAE,GACvCpE,IAAI,CAAC,MACL0E,OAAO;IAEZ,OAAO;QAACD,UAAUnD;QAAU+B,OAAOtD,GAAG,CAAC,CAACuE,IAAM,IAAIK,MAAM,CAACL,IAAItE,IAAI,CAAC;WAAUiE,MAAMlE,GAAG,CAAC0E;KAAW,CAACzE,IAAI,CAAC;AACzG;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAAS4E,eAAe9C,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAAS+C,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAACvD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEsD,SAASC,GAAG,CAAChF,GAAG,CAAC,CAAC+B,OAAS,GAAGA,KAAK,EAAE,EAAE8C,eAAe9C,MAAM,CAAC,CAAC,EAAE9B,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE8E,SAASE,EAAE,CAAChF,IAAI,CAAC,QAAQ+E,KAAK;AACpD;AAEA,6DAA6D;AAC7D,SAASE,aAAaC,OAA+B;IACnD,OAAOzD,OAAO0D,OAAO,CAACD,SACnBnF,GAAG,CAAC,CAAC,CAAC+B,MAAMsD,OAAO,GAAMA,WAAW,IAAItD,OAAO,GAAGA,KAAK,CAAC,EAAEsD,QAAQ,EAClEpF,IAAI,CAAC;AACV;AAEA,OAAO,SAASqF,aAAaC,OAAkG;IAC7H,6FAA6F;IAC7F,8EAA8E;IAC9E,OAAO;QAAC;WAAeA,QAAQvF,GAAG,CAAC,CAACwF,IAAM,CAAC,EAAE,EAAEA,EAAEzD,IAAI,CAAC,EAAE,EAAEyD,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEL,OAAO,CAACQ,OAAO,KAAK/F,YAAY,KAAK,CAAC,WAAW,EAAEsF,aAAaM,EAAEL,OAAO,EAAE,CAAC,CAAC,EAAE;KAAE;AACzL;AAEA,OAAO,SAASS,UAAUC,MASzB;IACC,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEzF,KAAKC,KAAK,CAACqF,OAAOE,IAAI,CAAC5F,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE2E,aAAae,OAAOd,QAAQ;QAAG;WAAOO,aAAaO,OAAON,OAAO;QAAG;QAAIpD,WAAW0D,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACxE,MAAM,EAAEqE,MAAMjD,IAAI,CAAC,CAAC,EAAE,EAAEgD,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACxE,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIoE,OAAOM,IAAI,CAAC1E,MAAM,GAAG,GAAGqE,MAAMjD,IAAI,CAAC,0BAA0BV,WAAW0D,OAAOM,IAAI,EAAE;SACpF,IAAIN,OAAOd,QAAQ,CAACC,GAAG,CAACoB,QAAQ,CAAC,SAASN,MAAMjD,IAAI,CAAC,CAAC,aAAa,EAAEgC,eAAe,QAAQ,CAAC,CAAC;IACnGiB,MAAMjD,IAAI,CAAC;IACX,IAAIgD,OAAOQ,YAAY,EAAEP,MAAMjD,IAAI,CAACgD,OAAOQ,YAAY;IACvDP,MAAMjD,IAAI,CAACV,WAAW0D,OAAOS,MAAM,EAAE;IACrC,OAAOR,MAAM7F,IAAI,CAAC;AACpB;AAEA,OAAO,SAASsG,WAAWV,MAAoR;IAC7S,MAAMW,QAAQ;QAAC,GAAGX,OAAOY,IAAI,CAAC,IAAI,EAAEZ,OAAOa,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAIb,OAAOc,UAAU,EAAEH,MAAM3D,IAAI,CAAC,CAAC,wDAAwD,EAAEgD,OAAOc,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAKrH,MAAM,IAAImC,OAAO0D,OAAO,CAACS,OAAOgB,WAAW,EAAGL,MAAM3D,IAAI,CAAC,CAAC,EAAE,EAAE+D,IAAI,EAAE,EAAErH,OAAO;IAC9F,IAAIsG,OAAOiB,QAAQ,CAACrF,MAAM,GAAG,GAAG;QAC9B+E,MAAM3D,IAAI,CAAC,IAAI;QACf,KAAK,MAAMkE,KAAKlB,OAAOiB,QAAQ,CAAEN,MAAM3D,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI+B,MAAM,CAACmC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAIb,OAAOuB,aAAa,GAAGvB,OAAOiB,QAAQ,CAACrF,MAAM,EAAE+E,MAAM3D,IAAI,CAAC,CAAC,IAAI,EAAEgD,OAAOuB,aAAa,GAAGvB,OAAOiB,QAAQ,CAACrF,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAIoE,OAAOb,GAAG,CAACoB,QAAQ,CAAC,aAAaI,MAAM3D,IAAI,CAAC,IAAI;IAC3D,IAAIgD,OAAOb,GAAG,CAACoB,QAAQ,CAAC,UAAU;QAChCI,MAAM3D,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMwE,WAAW,CAACC,OAAeC,OAAiB7D;YAChD,IAAIA,QAAQ,GAAG8C,MAAM3D,IAAI,CAAC,GAAGyE,MAAM,EAAE,EAAE5D,MAAM,GAAG,EAAE6D,MAAMtH,IAAI,CAAC,QAAQyD,QAAQ6D,MAAM9F,MAAM,GAAG,CAAC,GAAG,EAAEiC,QAAQ6D,MAAM9F,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAIoE,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGlB,MAAM3D,IAAI,CAAC;QAC1FwE,SAAS,aAAaxB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;QAC3DH,SAAS,cAAcxB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;QAChEJ,SAAS,aAAaxB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC/D;IACA,OAAOlB,MAAMvG,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/output.ts"],"sourcesContent":["// rows -> table (default) | json | csv\n\nexport type Row = Record<string, unknown>;\n\n// map/peek/status/path render structures rather than a row set, so csv has nothing to mean\n// there; the commands that emit rows take RowFormat.\nexport type Format = 'table' | 'json';\nexport type RowFormat = Format | 'csv';\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output keeps them as-is.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\n// RFC 4180, deliberately not cell(): csv is the redirect-to-file format, so it keeps every\n// character, newlines included; what it cannot carry (NULL vs empty, types) is what json is for.\n// An array (e.g. snippets) joins with the one delimiter our content provably cannot contain,\n// inside the quoted field newlines already trigger (DESIGN.md \"Rendering a list of snippets\").\nfunction csvField(value: unknown): string {\n  const text = value === null || value === undefined ? '' : Array.isArray(value) ? value.join('\\n') : String(value);\n  return /[\"\\r\\n,]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n}\n\nfunction csvLine(values: unknown[]): string {\n  return values.map(csvField).join(',');\n}\n\nfunction warnOversize(bytes: number, rowCount: number): void {\n  if (bytes <= OVERSIZE_BYTES) return;\n  console.warn(`warning: result is ${Math.round(bytes / 1000)} KB across ${rowCount} row(s); consider a LIMIT, fewer columns, snippet() instead of whole values, or --format csv redirected to a file`);\n}\n\n// JSON.stringify throws on BigInt, and stores return int64 as BigInt (DuckDB does): a value\n// that fits a safe integer stays a number, a larger one becomes its decimal string.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nconst bigintSafe = (_key: string, value: unknown): unknown => (typeof value === 'bigint' ? (value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value.toString()) : value);\n\nexport function stringifyJson(value: unknown, indent?: number): string {\n  return JSON.stringify(value, bigintSafe, indent);\n}\n\nexport function printRows(rows: Row[], format: RowFormat, columns?: string[]): void {\n  if (format === 'csv') {\n    // Names from a row wherever there is one: the statement's column list keeps duplicates the\n    // row collapsed. Nothing at all with no row and no columns -- a bare newline is an empty record.\n    const names = rows.length > 0 ? Object.keys(rows[0]) : [...new Set(columns ?? [])];\n    if (names.length === 0) return;\n    const rendered = [csvLine(names), ...rows.map((row) => csvLine(names.map((name) => row[name])))].join('\\n');\n    console.log(rendered);\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    warnOversize(Buffer.byteLength(rendered), rows.length);\n    return;\n  }\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  warnOversize(Buffer.byteLength(rendered), rows.length);\n}\n\n// Streaming counterpart for the one unbounded caller, `sense sql`: the result is never held as\n// rows and again as a rendered string. Table buffers (column width depends on the last row); json/csv write row by row.\nexport async function printRowStream(rows: AsyncIterable<Row>, format: RowFormat, columns: string[]): Promise<void> {\n  const iterator = rows[Symbol.asyncIterator]();\n  // One step before any write. FTS5 parses a MATCH expression at step time rather than at\n  // prepare time, so a bad search string throws here, with stdout still clean for the error.\n  const first = await iterator.next();\n  const rest: AsyncIterable<Row> = { [Symbol.asyncIterator]: () => iterator };\n\n  if (format === 'table') {\n    const collected: Row[] = first.done ? [] : [first.value];\n    for await (const row of rest) collected.push(row);\n    printRows(collected, format, columns);\n    return;\n  }\n\n  let bytes = 0;\n  let rowCount = 0;\n  const write = (text: string): void => {\n    // Bytes, not code units -- CJK is 3 bytes per character.\n    bytes += Buffer.byteLength(text);\n    process.stdout.write(text);\n  };\n\n  if (format === 'json') {\n    // Byte-identical to JSON.stringify(rows, null, 2) + newline: each row re-indented one\n    // level, comma-joined, inside the array brackets written here.\n    const indent = (row: Row) =>\n      stringifyJson(row, 2)\n        .split('\\n')\n        .map((line) => `  ${line}`)\n        .join('\\n');\n    if (first.done) {\n      write('[]\\n');\n    } else {\n      write('[\\n');\n      write(indent(first.value));\n      rowCount = 1;\n      for await (const row of rest) {\n        write(',\\n');\n        write(indent(row));\n        rowCount++;\n      }\n      write('\\n]\\n');\n    }\n  } else if (first.done) {\n    const names = [...new Set(columns)];\n    if (names.length > 0) write(`${csvLine(names)}\\n`);\n  } else {\n    const names = Object.keys(first.value);\n    write(`${csvLine(names)}\\n`);\n    write(`${csvLine(names.map((name) => first.value[name]))}\\n`);\n    rowCount = 1;\n    for await (const row of rest) {\n      write(`${csvLine(names.map((name) => row[name]))}\\n`);\n      rowCount++;\n    }\n  }\n  warnOversize(bytes, rowCount);\n}\n\n// Tables are for reading, and a row wider than the terminal wraps unreadably: widest columns\n// give up space first, down to a floor. --format json is never truncated.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\n// An array cell (snippets) becomes one physical line per element; every other value stays the\n// single line cell() already produced. No cell in any other command holds one today, so a row\n// with no array-valued cell yields exactly one physical line, unchanged from before this existed.\nfunction cellLines(value: unknown): string[] {\n  if (Array.isArray(value)) return value.length === 0 ? [''] : value.map((v) => cell(v));\n  return [cell(value)];\n}\n\nfunction renderRows(rows: Row[], format: Format, width = process.stdout.columns): string {\n  if (format === 'json') return stringifyJson(rows, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const linesByCell = rows.map((row) => columns.map((col) => cellLines(row[col])));\n  const heights = linesByCell.map((cellsInRow) => Math.max(1, ...cellsInRow.map((lines) => lines.length)));\n  const natural = columns.map((col, i) => Math.max(col.length, ...linesByCell.flatMap((cellsInRow) => cellsInRow[i]).map((line) => line.length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  // A row becomes as many physical lines as its tallest cell; other columns blank-pad on\n  // continuation lines (DESIGN.md \"Rendering a list of snippets\").\n  const body: string[] = [];\n  for (let r = 0; r < rows.length; r++) {\n    for (let li = 0; li < heights[r]; li++) body.push(formatRow(columns.map((_, c) => linesByCell[r][c][li] ?? '')));\n  }\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...body].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// Names the config key that turns a feature back on. Only the three features-block toggles\n// reach here; `embed` has its own status line.\nexport function featureOffHint(name: string): string {\n  return `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\n// name, or name:weight when the weight is not the default 1.\nfunction signalLabels(signals: Record<string, number>): string {\n  return Object.entries(signals)\n    .map(([name, weight]) => (weight === 1 ? name : `${name}:${weight}`))\n    .join(', ');\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>): string[] {\n  // \"0 embedded\" is ambiguous between a scope that declined vectors and one that has not built\n  // them yet, so a preset without the vectors signal says what it uses instead.\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded${p.signals.vectors !== undefined ? '' : ` (signals: ${signalLabels(p.signals)})`}`)];\n}\n\nexport function renderMap(result: {\n  docs: { count: number; bytes: number };\n  fields: Row[];\n  fieldsTotal: number;\n  features: { on: string[]; off: string[] };\n  presets: Array<{ name: string; files: number; embedded: number; signals: Record<string, number> }>;\n  hubs: Row[];\n  recent: Row[];\n  recentCaveat: string | null;\n}): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:');\n  if (result.recentCaveat) parts.push(result.recentCaveat);\n  parts.push(renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; parseError?: string | null; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  // Otherwise a refused parse is indistinguishable from a note that has no frontmatter, which\n  // is the confusion the whole quarantine design exists to remove.\n  if (result.parseError) lines.push(`  frontmatter: did not parse, so none of it is indexed (${result.parseError})`);\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n  } else {\n    const linkLine = (label: string, shown: string[], total: number) => {\n      if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n    };\n    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n    linkLine('links out', result.outbound, result.outboundTotal);\n    linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n    linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  }\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","csvField","text","undefined","Array","isArray","join","test","csvLine","values","map","warnOversize","bytes","rowCount","console","warn","Math","round","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","bigintSafe","_key","toString","stringifyJson","indent","JSON","stringify","printRows","rows","format","columns","names","length","Object","keys","Set","rendered","row","name","log","Buffer","byteLength","renderRows","printRowStream","iterator","Symbol","asyncIterator","first","next","rest","collected","done","push","write","process","stdout","split","line","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","cellLines","v","width","linesByCell","col","heights","cellsInRow","lines","natural","i","flatMap","clip","w","padEnd","slice","formatRow","trimEnd","body","r","li","_","c","repeat","featureOffHint","featuresLine","features","off","on","signalLabels","signals","entries","weight","presetsLines","presets","p","files","embedded","vectors","renderMap","result","parts","docs","count","fields","fieldsTotal","hubs","includes","recentCaveat","recent","renderPeek","path","tokens","parseError","key","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,uCAAuC;AASvC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,sGAAsG;AACtG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,6FAA6F;AAC7F,+FAA+F;AAC/F,SAASC,SAASH,KAAc;IAC9B,MAAMI,OAAOJ,UAAU,QAAQA,UAAUK,YAAY,KAAKC,MAAMC,OAAO,CAACP,SAASA,MAAMQ,IAAI,CAAC,QAAQP,OAAOD;IAC3G,OAAO,WAAWS,IAAI,CAACL,QAAQ,CAAC,CAAC,EAAEA,KAAKF,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC,GAAGE;AACnE;AAEA,SAASM,QAAQC,MAAiB;IAChC,OAAOA,OAAOC,GAAG,CAACT,UAAUK,IAAI,CAAC;AACnC;AAEA,SAASK,aAAaC,KAAa,EAAEC,QAAgB;IACnD,IAAID,SAAShB,gBAAgB;IAC7BkB,QAAQC,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACL,QAAQ,MAAM,WAAW,EAAEC,SAAS,iHAAiH,CAAC;AACtM;AAEA,4FAA4F;AAC5F,oFAAoF;AACpF,MAAMK,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,MAAMC,aAAa,CAACC,MAAczB,QAA6B,OAAOA,UAAU,WAAYA,SAAS,CAACoB,YAAYpB,SAASoB,WAAWE,OAAOtB,SAASA,MAAM0B,QAAQ,KAAM1B;AAE1K,OAAO,SAAS2B,cAAc3B,KAAc,EAAE4B,MAAe;IAC3D,OAAOC,KAAKC,SAAS,CAAC9B,OAAOwB,YAAYI;AAC3C;AAEA,OAAO,SAASG,UAAUC,IAAW,EAAEC,MAAiB,EAAEC,OAAkB;IAC1E,IAAID,WAAW,OAAO;QACpB,2FAA2F;QAC3F,iGAAiG;QACjG,MAAME,QAAQH,KAAKI,MAAM,GAAG,IAAIC,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE,IAAI;eAAI,IAAIO,IAAIL,oBAAAA,qBAAAA,UAAW,EAAE;SAAE;QAClF,IAAIC,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMI,WAAW;YAAC9B,QAAQyB;eAAWH,KAAKpB,GAAG,CAAC,CAAC6B,MAAQ/B,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASD,GAAG,CAACC,KAAK;SAAI,CAAClC,IAAI,CAAC;QACtGQ,QAAQ2B,GAAG,CAACH;QACZ,yDAAyD;QACzD3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;QACrD;IACF;IACA,MAAMI,WAAWM,WAAWd,MAAMC;IAClCjB,QAAQ2B,GAAG,CAACH;IACZ3B,aAAa+B,OAAOC,UAAU,CAACL,WAAWR,KAAKI,MAAM;AACvD;AAEA,+FAA+F;AAC/F,wHAAwH;AACxH,OAAO,eAAeW,eAAef,IAAwB,EAAEC,MAAiB,EAAEC,OAAiB;IACjG,MAAMc,WAAWhB,IAAI,CAACiB,OAAOC,aAAa,CAAC;IAC3C,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMC,QAAQ,MAAMH,SAASI,IAAI;IACjC,MAAMC,OAA2B;QAAE,CAACJ,OAAOC,aAAa,CAAC,EAAE,IAAMF;IAAS;IAE1E,IAAIf,WAAW,SAAS;QACtB,MAAMqB,YAAmBH,MAAMI,IAAI,GAAG,EAAE,GAAG;YAACJ,MAAMnD,KAAK;SAAC;QACxD,WAAW,MAAMyC,OAAOY,KAAMC,UAAUE,IAAI,CAACf;QAC7CV,UAAUuB,WAAWrB,QAAQC;QAC7B;IACF;IAEA,IAAIpB,QAAQ;IACZ,IAAIC,WAAW;IACf,MAAM0C,QAAQ,CAACrD;QACb,yDAAyD;QACzDU,SAAS8B,OAAOC,UAAU,CAACzC;QAC3BsD,QAAQC,MAAM,CAACF,KAAK,CAACrD;IACvB;IAEA,IAAI6B,WAAW,QAAQ;QACrB,sFAAsF;QACtF,+DAA+D;QAC/D,MAAML,SAAS,CAACa,MACdd,cAAcc,KAAK,GAChBmB,KAAK,CAAC,MACNhD,GAAG,CAAC,CAACiD,OAAS,CAAC,EAAE,EAAEA,MAAM,EACzBrD,IAAI,CAAC;QACV,IAAI2C,MAAMI,IAAI,EAAE;YACdE,MAAM;QACR,OAAO;YACLA,MAAM;YACNA,MAAM7B,OAAOuB,MAAMnD,KAAK;YACxBe,WAAW;YACX,WAAW,MAAM0B,OAAOY,KAAM;gBAC5BI,MAAM;gBACNA,MAAM7B,OAAOa;gBACb1B;YACF;YACA0C,MAAM;QACR;IACF,OAAO,IAAIN,MAAMI,IAAI,EAAE;QACrB,MAAMpB,QAAQ;eAAI,IAAII,IAAIL;SAAS;QACnC,IAAIC,MAAMC,MAAM,GAAG,GAAGqB,MAAM,GAAG/C,QAAQyB,OAAO,EAAE,CAAC;IACnD,OAAO;QACL,MAAMA,QAAQE,OAAOC,IAAI,CAACa,MAAMnD,KAAK;QACrCyD,MAAM,GAAG/C,QAAQyB,OAAO,EAAE,CAAC;QAC3BsB,MAAM,GAAG/C,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASS,MAAMnD,KAAK,CAAC0C,KAAK,GAAG,EAAE,CAAC;QAC5D3B,WAAW;QACX,WAAW,MAAM0B,OAAOY,KAAM;YAC5BI,MAAM,GAAG/C,QAAQyB,MAAMvB,GAAG,CAAC,CAAC8B,OAASD,GAAG,CAACC,KAAK,GAAG,EAAE,CAAC;YACpD3B;QACF;IACF;IACAF,aAAaC,OAAOC;AACtB;AAEA,6FAA6F;AAC7F,0EAA0E;AAC1E,MAAM+C,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAO7B,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMgC,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACxD,KAAKyD,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGvD,KAAKyD,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,8FAA8F;AAC9F,8FAA8F;AAC9F,kGAAkG;AAClG,SAASU,UAAU5E,KAAc;IAC/B,IAAIM,MAAMC,OAAO,CAACP,QAAQ,OAAOA,MAAMoC,MAAM,KAAK,IAAI;QAAC;KAAG,GAAGpC,MAAMY,GAAG,CAAC,CAACiE,IAAM9E,KAAK8E;IACnF,OAAO;QAAC9E,KAAKC;KAAO;AACtB;AAEA,SAAS8C,WAAWd,IAAW,EAAEC,MAAc,EAAE6C,QAAQpB,QAAQC,MAAM,CAACzB,OAAO;IAC7E,IAAID,WAAW,QAAQ,OAAON,cAAcK,MAAM;IAClD,IAAIA,KAAKI,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMF,UAAUG,OAAOC,IAAI,CAACN,IAAI,CAAC,EAAE;IACnC,MAAM+C,cAAc/C,KAAKpB,GAAG,CAAC,CAAC6B,MAAQP,QAAQtB,GAAG,CAAC,CAACoE,MAAQJ,UAAUnC,GAAG,CAACuC,IAAI;IAC7E,MAAMC,UAAUF,YAAYnE,GAAG,CAAC,CAACsE,aAAehE,KAAKyD,GAAG,CAAC,MAAMO,WAAWtE,GAAG,CAAC,CAACuE,QAAUA,MAAM/C,MAAM;IACrG,MAAMgD,UAAUlD,QAAQtB,GAAG,CAAC,CAACoE,KAAKK,IAAMnE,KAAKyD,GAAG,CAACK,IAAI5C,MAAM,KAAK2C,YAAYO,OAAO,CAAC,CAACJ,aAAeA,UAAU,CAACG,EAAE,EAAEzE,GAAG,CAAC,CAACiD,OAASA,KAAKzB,MAAM;IAC5I,MAAM6B,SAASa,SAASA,QAAQhB,aAAaE,UAAUoB,SAASN,SAASM;IAEzE,MAAMG,OAAO,CAACvF,OAAewF,IAAexF,MAAMoC,MAAM,IAAIoD,IAAIxF,MAAMyF,MAAM,CAACD,KAAK,GAAGxF,MAAM0F,KAAK,CAAC,GAAGF,IAAI,KAAKzB,WAAW;IACxH,MAAM4B,YAAY,CAAChF,SACjBA,OACGC,GAAG,CAAC,CAACZ,OAAOqF,IAAME,KAAKvF,OAAOiE,MAAM,CAACoB,EAAE,GACvC7E,IAAI,CAAC,MACLoF,OAAO;IAEZ,uFAAuF;IACvF,iEAAiE;IACjE,MAAMC,OAAiB,EAAE;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAI9D,KAAKI,MAAM,EAAE0D,IAAK;QACpC,IAAK,IAAIC,KAAK,GAAGA,KAAKd,OAAO,CAACa,EAAE,EAAEC,KAAMF,KAAKrC,IAAI,CAACmC,UAAUzD,QAAQtB,GAAG,CAAC,CAACoF,GAAGC;gBAAMlB;oBAAAA,sBAAAA,WAAW,CAACe,EAAE,CAACG,EAAE,CAACF,GAAG,cAArBhB,iCAAAA,sBAAyB;;IAC7G;IAEA,OAAO;QAACY,UAAUzD;QAAU+B,OAAOrD,GAAG,CAAC,CAAC4E,IAAM,IAAIU,MAAM,CAACV,IAAIhF,IAAI,CAAC;WAAUqF;KAAK,CAACrF,IAAI,CAAC;AACzF;AAEA,8EAA8E;AAE9E,2FAA2F;AAC3F,+CAA+C;AAC/C,OAAO,SAAS2F,eAAezD,IAAY;IACzC,OAAO,CAAC,SAAS,EAAEA,MAAM;AAC3B;AAEA,OAAO,SAAS0D,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAAClE,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEiE,SAASC,GAAG,CAAC1F,GAAG,CAAC,CAAC8B,OAAS,GAAGA,KAAK,EAAE,EAAEyD,eAAezD,MAAM,CAAC,CAAC,EAAElC,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAE6F,SAASE,EAAE,CAAC/F,IAAI,CAAC,QAAQ8F,KAAK;AACpD;AAEA,6DAA6D;AAC7D,SAASE,aAAaC,OAA+B;IACnD,OAAOpE,OAAOqE,OAAO,CAACD,SACnB7F,GAAG,CAAC,CAAC,CAAC8B,MAAMiE,OAAO,GAAMA,WAAW,IAAIjE,OAAO,GAAGA,KAAK,CAAC,EAAEiE,QAAQ,EAClEnG,IAAI,CAAC;AACV;AAEA,OAAO,SAASoG,aAAaC,OAAkG;IAC7H,6FAA6F;IAC7F,8EAA8E;IAC9E,OAAO;QAAC;WAAeA,QAAQjG,GAAG,CAAC,CAACkG,IAAM,CAAC,EAAE,EAAEA,EAAEpE,IAAI,CAAC,EAAE,EAAEoE,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,EAAEF,EAAEL,OAAO,CAACQ,OAAO,KAAK5G,YAAY,KAAK,CAAC,WAAW,EAAEmG,aAAaM,EAAEL,OAAO,EAAE,CAAC,CAAC,EAAE;KAAE;AACzL;AAEA,OAAO,SAASS,UAAUC,MASzB;IACC,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEpG,KAAKC,KAAK,CAACgG,OAAOE,IAAI,CAACvG,KAAK,GAAG,MAAM,IAAI,CAAC;QAAEsF,aAAae,OAAOd,QAAQ;QAAG;WAAOO,aAAaO,OAAON,OAAO;QAAG;QAAI/D,WAAWqE,OAAOI,MAAM,EAAE;KAAS;IACrM,IAAIJ,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACnF,MAAM,EAAEgF,MAAM5D,IAAI,CAAC,CAAC,EAAE,EAAE2D,OAAOK,WAAW,GAAGL,OAAOI,MAAM,CAACnF,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI+E,OAAOM,IAAI,CAACrF,MAAM,GAAG,GAAGgF,MAAM5D,IAAI,CAAC,0BAA0BV,WAAWqE,OAAOM,IAAI,EAAE;SACpF,IAAIN,OAAOd,QAAQ,CAACC,GAAG,CAACoB,QAAQ,CAAC,SAASN,MAAM5D,IAAI,CAAC,CAAC,aAAa,EAAE2C,eAAe,QAAQ,CAAC,CAAC;IACnGiB,MAAM5D,IAAI,CAAC;IACX,IAAI2D,OAAOQ,YAAY,EAAEP,MAAM5D,IAAI,CAAC2D,OAAOQ,YAAY;IACvDP,MAAM5D,IAAI,CAACV,WAAWqE,OAAOS,MAAM,EAAE;IACrC,OAAOR,MAAM5G,IAAI,CAAC;AACpB;AAEA,OAAO,SAASqH,WAAWV,MAAoR;IAC7S,MAAMhC,QAAQ;QAAC,GAAGgC,OAAOW,IAAI,CAAC,IAAI,EAAEX,OAAOY,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,4FAA4F;IAC5F,iEAAiE;IACjE,IAAIZ,OAAOa,UAAU,EAAE7C,MAAM3B,IAAI,CAAC,CAAC,wDAAwD,EAAE2D,OAAOa,UAAU,CAAC,CAAC,CAAC;IACjH,KAAK,MAAM,CAACC,KAAKjI,MAAM,IAAIqC,OAAOqE,OAAO,CAACS,OAAOe,WAAW,EAAG/C,MAAM3B,IAAI,CAAC,CAAC,EAAE,EAAEyE,IAAI,EAAE,EAAEjI,OAAO;IAC9F,IAAImH,OAAOgB,QAAQ,CAAC/F,MAAM,GAAG,GAAG;QAC9B+C,MAAM3B,IAAI,CAAC,IAAI;QACf,KAAK,MAAM4E,KAAKjB,OAAOgB,QAAQ,CAAEhD,MAAM3B,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI0C,MAAM,CAACkC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAIZ,OAAOsB,aAAa,GAAGtB,OAAOgB,QAAQ,CAAC/F,MAAM,EAAE+C,MAAM3B,IAAI,CAAC,CAAC,IAAI,EAAE2D,OAAOsB,aAAa,GAAGtB,OAAOgB,QAAQ,CAAC/F,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAI+E,OAAOb,GAAG,CAACoB,QAAQ,CAAC,aAAavC,MAAM3B,IAAI,CAAC,IAAI;IAC3D,IAAI2D,OAAOb,GAAG,CAACoB,QAAQ,CAAC,UAAU;QAChCvC,MAAM3B,IAAI,CAAC,IAAI;IACjB,OAAO;QACL,MAAMkF,WAAW,CAACC,OAAeC,OAAiBvE;YAChD,IAAIA,QAAQ,GAAGc,MAAM3B,IAAI,CAAC,GAAGmF,MAAM,EAAE,EAAEtE,MAAM,GAAG,EAAEuE,MAAMpI,IAAI,CAAC,QAAQ6D,QAAQuE,MAAMxG,MAAM,GAAG,CAAC,GAAG,EAAEiC,QAAQuE,MAAMxG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;QACtI;QACA,IAAI+E,OAAO0B,aAAa,GAAG1B,OAAO2B,eAAe,GAAG3B,OAAO4B,cAAc,GAAG,GAAG5D,MAAM3B,IAAI,CAAC;QAC1FkF,SAAS,aAAavB,OAAO6B,QAAQ,EAAE7B,OAAO0B,aAAa;QAC3DH,SAAS,cAAcvB,OAAO8B,UAAU,EAAE9B,OAAO2B,eAAe;QAChEJ,SAAS,aAAavB,OAAO+B,SAAS,EAAE/B,OAAO4B,cAAc;IAC/D;IACA,OAAO5D,MAAM3E,IAAI,CAAC;AACpB"}

package/dist/esm/output/search-error.d.ts

@@ -1 +1,2 @@
+export declare function bareTermSyntaxError(terms: string): Error | null;
 export declare function searchError(err: Error, terms: string, scope?: string): Error;

package/dist/esm/output/search-error.js

@@ -1,7 +1,23 @@
 import { SenseError } from '../errors.js';
+import { hasUnspacedRun } from '../text/segment.js';
 // FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and
 // errors `no such column: to` -- true about the parse, misleading about the input.
-const FTS5_PUNCTUATION = /[^\p{L}\p{N}_\s]/u;
+const FTS5_PUNCTUATION = /[^\p{L}\p{M}\p{N}_\s]/u;
+// SQLite treats punctuation in an unquoted spaced-script token as MATCH syntax. Validate that
+// boundary before dispatch so DuckDB and Tantivy do not silently interpret the same input as text.
+export function bareTermSyntaxError(terms) {
+    var _outsideQuotes_match;
+    const outsideQuotes = terms.replace(/"[^"]*"/g, ' ');
+    const suspects = ((_outsideQuotes_match = outsideQuotes.match(/\S+/g)) !== null && _outsideQuotes_match !== void 0 ? _outsideQuotes_match : []).filter((token)=>{
+        if (!FTS5_PUNCTUATION.test(token) || hasUnspacedRun(token)) return false;
+        if (/^[\p{L}_]\w*\s*:/u.test(token)) return false;
+        if (/^[\p{L}\p{N}_]+\*(?:\s|$)/u.test(token)) return false;
+        if (/^(?:AND|OR|NOT)$/u.test(token)) return false;
+        return true;
+    });
+    if (suspects.length === 0) return null;
+    return new SenseError('SEARCH_SYNTAX', `the punctuation in ${suspects.map((term)=>`\`${term}\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '"${suspects[0]}"'. Searchable columns are title, summary, text.`);
+}
 export function searchError(err, terms, scope) {
     var _terms_match;
     var _exec;

package/dist/esm/output/search-error.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/search-error.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\n\n// FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and\n// errors `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_PUNCTUATION = /[^\\p{L}\\p{N}_\\s]/u;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_PUNCTUATION.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where can raise \"no such column\" through this same statement.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","FTS5_PUNCTUATION","searchError","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","map","join"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAE1C,0FAA0F;AAC1F,mFAAmF;AACnF,MAAMC,mBAAmB;AAEzB,OAAO,SAASC,YAAYC,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAGN;IALZ,MAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,MAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQX,iBAAiBM,IAAI,CAACI;IACvG,wFAAwF;IACxF,qEAAqE;IACrE,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,CAACP,IAAMA,EAAEQ,KAAK,CAAC,mBAAmBC,QAAQ,CAACP;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIjB,WAAW,iBAAiB,GAAGM,QAAQ,uBAAuB,EAAEE,SAASa,GAAG,CAAC,CAACV,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEW,IAAI,CAAC,MAAM,gFAAgF,EAAEd,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIhB,WAAW,iBAAiB,GAAGM,QAAQ,yBAAyB,EAAED,MAAM,iHAAiH,CAAC;IACvM;IACA,OAAO,IAAIL,WAAW,iBAAiB,GAAGM,QAAQ,4JAA4J,CAAC;AACjN"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/search-error.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { hasUnspacedRun } from '../text/segment.ts';\n\n// FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and\n// errors `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_PUNCTUATION = /[^\\p{L}\\p{M}\\p{N}_\\s]/u;\n\n// SQLite treats punctuation in an unquoted spaced-script token as MATCH syntax. Validate that\n// boundary before dispatch so DuckDB and Tantivy do not silently interpret the same input as text.\nexport function bareTermSyntaxError(terms: string): Error | null {\n  const outsideQuotes = terms.replace(/\"[^\"]*\"/g, ' ');\n  const suspects = (outsideQuotes.match(/\\S+/g) ?? []).filter((token) => {\n    if (!FTS5_PUNCTUATION.test(token) || hasUnspacedRun(token)) return false;\n    if (/^[\\p{L}_]\\w*\\s*:/u.test(token)) return false;\n    if (/^[\\p{L}\\p{N}_]+\\*(?:\\s|$)/u.test(token)) return false;\n    if (/^(?:AND|OR|NOT)$/u.test(token)) return false;\n    return true;\n  });\n  if (suspects.length === 0) return null;\n  return new SenseError('SEARCH_SYNTAX', `the punctuation in ${suspects.map((term) => `\\`${term}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n}\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_PUNCTUATION.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where can raise \"no such column\" through this same statement.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","hasUnspacedRun","FTS5_PUNCTUATION","bareTermSyntaxError","terms","outsideQuotes","replace","suspects","match","filter","token","test","length","map","term","join","searchError","err","scope","message","t","startsWith","col","exec","fromTerms","undefined","some","split","includes"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,qBAAqB;AAEpD,0FAA0F;AAC1F,mFAAmF;AACnF,MAAMC,mBAAmB;AAEzB,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,SAASC,oBAAoBC,KAAa;QAE7BC;IADlB,MAAMA,gBAAgBD,MAAME,OAAO,CAAC,YAAY;IAChD,MAAMC,WAAW,EAACF,uBAAAA,cAAcG,KAAK,CAAC,qBAApBH,kCAAAA,uBAA+B,EAAE,EAAEI,MAAM,CAAC,CAACC;QAC3D,IAAI,CAACR,iBAAiBS,IAAI,CAACD,UAAUT,eAAeS,QAAQ,OAAO;QACnE,IAAI,oBAAoBC,IAAI,CAACD,QAAQ,OAAO;QAC5C,IAAI,6BAA6BC,IAAI,CAACD,QAAQ,OAAO;QACrD,IAAI,oBAAoBC,IAAI,CAACD,QAAQ,OAAO;QAC5C,OAAO;IACT;IACA,IAAIH,SAASK,MAAM,KAAK,GAAG,OAAO;IAClC,OAAO,IAAIZ,WAAW,iBAAiB,CAAC,mBAAmB,EAAEO,SAASM,GAAG,CAAC,CAACC,OAAS,CAAC,EAAE,EAAEA,KAAK,EAAE,CAAC,EAAEC,IAAI,CAAC,MAAM,gFAAgF,EAAER,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;AAC/P;AAEA,OAAO,SAASS,YAAYC,GAAU,EAAEb,KAAa,EAAEc,KAAc;QAGjDd;QAGN;IALZ,MAAMe,UAAUF,IAAIE,OAAO;IAC3B,IAAI,CAAC,oDAAoDR,IAAI,CAACQ,UAAU,OAAOF;IAC/E,MAAMV,WAAW,EAACH,eAAAA,MAAMI,KAAK,CAAC,qBAAZJ,0BAAAA,eAAuB,EAAE,EAAEK,MAAM,CAAC,CAACW,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQnB,iBAAiBS,IAAI,CAACS;IACvG,wFAAwF;IACxF,qEAAqE;IACrE,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACJ,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMK,YAAYF,QAAQG,YAAYlB,SAASK,MAAM,GAAG,IAAIL,SAASmB,IAAI,CAAC,CAACN,IAAMA,EAAEO,KAAK,CAAC,mBAAmBC,QAAQ,CAACN;IACrH,IAAIE,aAAajB,SAASK,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIZ,WAAW,iBAAiB,GAAGmB,QAAQ,uBAAuB,EAAEZ,SAASM,GAAG,CAAC,CAACO,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEL,IAAI,CAAC,MAAM,gFAAgF,EAAER,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIe,QAAQG,aAAaP,UAAUO,WAAW;QAC5C,OAAO,IAAIzB,WAAW,iBAAiB,GAAGmB,QAAQ,yBAAyB,EAAED,MAAM,iHAAiH,CAAC;IACvM;IACA,OAAO,IAAIlB,WAAW,iBAAiB,GAAGmB,QAAQ,4JAA4J,CAAC;AACjN"}

package/dist/esm/scan/frontmatter.js

@@ -92,7 +92,7 @@ function warnStringifiedKeys(relPath, doc, warnings) {
     if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);
 }
 // Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.
-// yaml's message continues onto a source excerpt, so the first line is the sentence, minus the colon that introduced the part being dropped.
+// yaml's message continues onto a source snippet, so the first line is the sentence, minus the colon that introduced the part being dropped.
 function firstLine(message) {
     return message.split('\n')[0].replace(/:\s*$/, '');
 }

package/dist/esm/scan/frontmatter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source excerpt, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","countLines","s","n","i","charCodeAt","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,qGAAqG;AACrG,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gGAAgG;AAChG,qDAAqD;AACrD,OAAO,SAAS6B,WAAWC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEb,MAAM,EAAEe,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAEA,gEAAgE;AAChE,OAAO,SAASG,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIjB,KAAK,CAACkB,IAAI,CAAC,EAAE,CAACnB,MAAM;IACrC,MAAMwB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKzC,WAAW,OAAO;QAAEqC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKtB,KAAK,CAAC,GAAGuB,MAAMC,KAAK;QAAGH,MAAMC,KAAKtB,KAAK,CAACuB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACxB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAAS0B,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCpD,MAAMkD,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACzD,aAAayD,KAAKC,GAAG,GAAG,OAAOlD;YACpC8C,QAAQ;YACR,OAAOpD,MAAMyD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAACrD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASsD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMnD,cAAc4C,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAChE,oBAAoBiE,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQ1C,MAAM,GAAG,GAAG;QACtB,MAAMgD,SAASN,QAAQ1C,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE0C,QAAQ1C,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMiD,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASlE,WAAW,OAAO;QAAEkE,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source snippet, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","countLines","s","n","i","charCodeAt","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,qGAAqG;AACrG,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gGAAgG;AAChG,qDAAqD;AACrD,OAAO,SAAS6B,WAAWC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEb,MAAM,EAAEe,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAEA,gEAAgE;AAChE,OAAO,SAASG,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIjB,KAAK,CAACkB,IAAI,CAAC,EAAE,CAACnB,MAAM;IACrC,MAAMwB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKzC,WAAW,OAAO;QAAEqC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKtB,KAAK,CAAC,GAAGuB,MAAMC,KAAK;QAAGH,MAAMC,KAAKtB,KAAK,CAACuB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACxB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAAS0B,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCpD,MAAMkD,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACzD,aAAayD,KAAKC,GAAG,GAAG,OAAOlD;YACpC8C,QAAQ;YACR,OAAOpD,MAAMyD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAACrD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASsD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMnD,cAAc4C,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAChE,oBAAoBiE,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQ1C,MAAM,GAAG,GAAG;QACtB,MAAMgD,SAASN,QAAQ1C,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE0C,QAAQ1C,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMiD,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASlE,WAAW,OAAO;QAAEkE,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}

package/dist/esm/scan/index.js

@@ -6,6 +6,7 @@ import { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_
 export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
 export { listFiles, toPosixPath } from './list.js';
 export function parseFile(file, extractors = [], cfg) {
+    // Preserve the authored bytes: feature extractors and section offsets use the raw document.
     const raw = readFileSync(file.absPath, 'utf8');
     const warnings = [];
     const { fm, body: content } = splitFrontmatter(raw);

package/dist/esm/scan/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors and section offsets use the raw document.\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,iGAAiG;AAEjG,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,4FAA4F;IAC5F,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}

package/dist/esm/store/cache.js

@@ -4,7 +4,8 @@ import { STATE_DIR } from '../config/index.js';
 // Deletes the cache directory and nothing else: the next open() reconciles anyway. Engine-neutral,
 // since every store keeps its files under the same STATE_DIR.
 export function clearCache(cfg) {
-    rmSync(join(cfg.baseDir, STATE_DIR), {
+    var _cfg_configDir;
+    rmSync(join((_cfg_configDir = cfg.configDir) !== null && _cfg_configDir !== void 0 ? _cfg_configDir : cfg.baseDir, STATE_DIR), {
         recursive: true,
         force: true
     });

package/dist/esm/store/cache.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/cache.ts"],"sourcesContent":["import { rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { STATE_DIR } from '../config/index.ts';\n\n// Deletes the cache directory and nothing else: the next open() reconciles anyway. Engine-neutral,\n// since every store keeps its files under the same STATE_DIR.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["rmSync","join","STATE_DIR","clearCache","cfg","baseDir","recursive","force"],"mappings":"AAAA,SAASA,MAAM,QAAQ,UAAU;AACjC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,mGAAmG;AACnG,8DAA8D;AAC9D,OAAO,SAASC,WAAWC,GAAmB;IAC5CJ,OAAOC,KAAKG,IAAIC,OAAO,EAAEH,YAAY;QAAEI,WAAW;QAAMC,OAAO;IAAK;AACtE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/cache.ts"],"sourcesContent":["import { rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { STATE_DIR } from '../config/index.ts';\n\n// Deletes the cache directory and nothing else: the next open() reconciles anyway. Engine-neutral,\n// since every store keeps its files under the same STATE_DIR.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.configDir ?? cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["rmSync","join","STATE_DIR","clearCache","cfg","configDir","baseDir","recursive","force"],"mappings":"AAAA,SAASA,MAAM,QAAQ,UAAU;AACjC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,mGAAmG;AACnG,8DAA8D;AAC9D,OAAO,SAASC,WAAWC,GAAmB;QAChCA;IAAZJ,OAAOC,MAAKG,iBAAAA,IAAIC,SAAS,cAAbD,4BAAAA,iBAAiBA,IAAIE,OAAO,EAAEJ,YAAY;QAAEK,WAAW;QAAMC,OAAO;IAAK;AACvF"}

package/dist/esm/store/duckdb/fieldStats.js

@@ -7,13 +7,10 @@ function classifyVariantType(name, field) {
     if (name === 'VARCHAR') return 'text';
     throw new Error(`duckdb: variant_typeof() returned unrecognized type "${name}" for column "${field}"`);
 }
-// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in
-// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same
-// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:
-// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking
-// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no
-// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300
-// columns runs in 7ms.
+// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates
+// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);
+// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over
+// 300 columns runs in 7ms.
 const AGGREGATES_PER_QUERY = 16;
 // COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).
 // '|' is the separator, not ',': DECIMAL's display name ("DECIMAL(4, 2)") contains a comma.

package/dist/esm/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in\n// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same\n// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:\n// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking\n// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no\n// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300\n// columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,qGAAqG;AACrG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,uBAAuB;AACvB,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates\n// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);\n// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over\n// 300 columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,kGAAkG;AAClG,iGAAiG;AACjG,kGAAkG;AAClG,2BAA2B;AAC3B,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}

package/dist/esm/store/duckdb/lexical.d.ts

@@ -2,10 +2,10 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
-export declare function markContentStale(conn: Connection): void;
+export declare function markContentStale(conn: Connection): Promise<void>;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;
-    markStale: () => void;
+    markStale: () => Promise<void>;
 };
 export {};

package/dist/esm/store/duckdb/lexical.js

@@ -1,7 +1,8 @@
 // D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /
-// unspaced scripts, JS excerpts for every doc (no snippet()).
+// unspaced scripts, JS snippets for every doc (no snippet()).
 import { SenseError } from '../../errors.js';
-import { hasUnspacedRun } from '../../text/segment.js';
+import { foldForSearch, matchesSearchPhrase, searchTokens } from '../../text/segment.js';
+import { getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 const FIELDS = [
     'title',
@@ -50,50 +51,79 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
+// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of
+// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.
+const FTS_STALE_META_KEY = 'fts_stale';
 // One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
 // stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
 const ftsState = new WeakMap();
-function stateFor(conn) {
+async function stateFor(conn) {
     let state = ftsState.get(conn);
     if (!state) {
+        // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that
+        // never built an index, or one a prior process marked stale and never got to clear.
         state = {
-            stale: true
+            stale: await getMeta(conn, FTS_STALE_META_KEY) !== '0'
         };
         ftsState.set(conn, state);
     }
     return state;
 }
-// Called whenever this store's reconcileContent changes `content`, so the next lexical query
-// rebuilds instead of searching stale content.
-export function markContentStale(conn) {
-    stateFor(conn).stale = true;
+// Called whenever this store's reconcileContent changes `content`; must run inside that same
+// transaction (reconcile.ts) so a crash never lands the content write without the stale mark.
+export async function markContentStale(conn) {
+    ftsState.set(conn, {
+        stale: true
+    });
+    await setMeta(conn, FTS_STALE_META_KEY, '1');
 }
-// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
-// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
+// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,
+// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.
 function splitTerms(terms) {
     const words = [];
+    const bareWords = [];
+    const phraseWords = [];
     const substrings = [];
+    const phrases = [];
+    let emptyPhrase = false;
     const withoutPhrases = terms.replace(/"([^"]*)"/g, (_m, inner)=>{
         const phrase = inner.trim();
-        if (phrase.length === 0) return ' ';
-        substrings.push(phrase);
-        if (!hasUnspacedRun(phrase)) {
-            for (const w of phrase.split(/\s+/))if (w.length > 0) words.push(w);
+        const phraseTokens = searchTokens(phrase);
+        if (phraseTokens.length === 0) {
+            emptyPhrase = true;
+            return ' ';
+        }
+        phrases.push(phraseTokens);
+        for (const token of phraseTokens){
+            if (token.unspaced) substrings.push(token.text);
+            else {
+                words.push(token.text);
+                phraseWords.push(token.text);
+            }
         }
         return ' ';
     });
     for (const tok of withoutPhrases.split(/\s+/)){
         if (tok.length === 0) continue;
-        if (hasUnspacedRun(tok)) substrings.push(tok);
-        else words.push(tok);
+        for (const token of searchTokens(tok)){
+            if (token.unspaced) substrings.push(token.text);
+            else {
+                words.push(token.text);
+                bareWords.push(token.text);
+            }
+        }
     }
     return {
         words,
-        substrings
+        bareWords,
+        phraseWords,
+        substrings,
+        phrases,
+        emptyPhrase
     };
 }
-// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only
-// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).
+// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays
+// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.
 async function ensureFtsFresh(conn, state) {
     if (!state.stale) return;
     await conn.exec('INSTALL fts; LOAD fts;');
@@ -101,70 +131,94 @@ async function ensureFtsFresh(conn, state) {
         // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's
         // default 571-word list would otherwise drop common words (e.g. "and") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.
         await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);
+        // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant
+        // rebuild next time), never a cleared mark serving a half-built index as fresh.
+        await setMeta(conn, FTS_STALE_META_KEY, '0');
     });
     state.stale = false;
 }
 // Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
 // score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
-function buildScoreAndGate(words, substrings) {
+function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
     const scoreParts = [];
     const gateParts = [];
     const scoreParams = [];
     const gateParams = [];
     if (words.length > 0) {
-        const wordQuery = words.join(' ');
+        const wordQuery = words.map(foldForSearch).join(' ');
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
             scoreParams.push(wordQuery);
         }
-        gateParts.push(`fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`);
-        gateParams.push(wordQuery);
+        if (bareWords.length > 0) {
+            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            gateParts.push(nativeGate);
+            gateParams.push(bareWords.map(foldForSearch).join(' '));
+        }
+        if (phraseWords.length > 0) {
+            const nativeGate = `fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`;
+            const fallback = phraseWords.map((_word)=>`(${FIELDS.map((field)=>`contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`).join(' AND ');
+            gateParts.push(`(${nativeGate} OR (${fallback}))`);
+            gateParams.push(phraseWords.map(foldForSearch).join(' '));
+            gateParams.push(...phraseWords.flatMap((word)=>FIELDS.map(()=>foldForSearch(word))));
+        }
     }
     for (const raw of substrings){
         const needle = raw.toLowerCase();
         for (const field of FIELDS){
-            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);
-            scoreParams.push(needle);
+            scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(strip_accents(lower(content.${field})), ?) AS INTEGER)`);
+            scoreParams.push(foldForSearch(needle));
         }
-        gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);
-        gateParams.push(needle);
+        gateParts.push(`(${FIELDS.map((field)=>`contains(strip_accents(lower(content.${field})), ?)`).join(' OR ')})`);
+        gateParams.push(...FIELDS.map(()=>foldForSearch(needle)));
     }
     return {
         scoreSql: scoreParts.join(' + '),
-        gateSql: gateParts.join(' AND '),
+        gateSql: gateParts.join(' AND ') || 'TRUE',
         params: [
             ...scoreParams,
             ...gateParams
         ]
     };
 }
-// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).
-// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).
+// Ranked query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).
+// Phrase rows carry authored fields through the post-filter; the public result still contains paths only.
 export async function queryLexical(conn, terms, opts, state) {
     const unsupported = unsupportedOperator(terms);
     if (unsupported !== null) {
         throw new SenseError('STORE_CAPABILITY_MISSING', `store "duckdb" does not implement FTS5's ${unsupported.label} ("${unsupported.token}") in this build; rephrase "${terms.trim()}" without it, or set "store" to "sqlite" in this tree's config to search it as written`);
     }
     const { whereJoin, whereCond, scopeCond, limit } = opts;
-    const { words, substrings } = splitTerms(terms);
-    if (words.length === 0 && substrings.length === 0) return [];
+    const { words, bareWords, phraseWords, substrings, phrases, emptyPhrase } = splitTerms(terms);
+    if (emptyPhrase || words.length === 0 && substrings.length === 0) return [];
     if (words.length > 0) await ensureFtsFresh(conn, state);
-    const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);
-    const sql = `SELECT path, NULL AS hit FROM (
-    SELECT content."path" AS path, (${scoreSql}) AS score
+    const { scoreSql, gateSql, params } = buildScoreAndGate(words, bareWords, phraseWords, substrings);
+    const selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
+    const sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';
+    const sql = `SELECT path, score${phrases.length > 0 ? ', title, summary, text' : ''} FROM (
+    SELECT content."path" AS path, (${scoreSql}) AS score${selectFields}
     FROM content
     ${whereJoin}
     WHERE ${gateSql}
     ${whereCond} ${scopeCond}
-  ) sq ORDER BY score DESC, path LIMIT ?`;
+    ) sq ORDER BY score DESC, path${sqlLimit}`;
     const stmt = await conn.prepare(sql);
-    return await stmt.all(...params, limit);
+    const rows = await stmt.all(...params, ...phrases.length > 0 ? [] : [
+        limit
+    ]);
+    const matches = phrases.length === 0 ? rows : rows.filter((row)=>phrases.every((phrase)=>FIELDS.some((field)=>{
+                var _row_field;
+                return matchesSearchPhrase((_row_field = row[field]) !== null && _row_field !== void 0 ? _row_field : '', phrase);
+            })));
+    return matches.slice(0, limit).map(({ path })=>({
+            path
+        }));
 }
-// `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches
-// `content`), and reconcileContent (reconcile.ts) marks it stale again on every content change.
+// `stale`'s first read for a connection comes from meta.fts_stale (a fresh connection can't know
+// otherwise), and reconcileContent (reconcile.ts) marks it stale again on every content change.
 export function createLexicalIndex(conn) {
     return {
-        query: (terms, opts)=>queryLexical(conn, terms, opts, stateFor(conn)),
+        query: async (terms, opts)=>queryLexical(conn, terms, opts, await stateFor(conn)),
         markStale: ()=>markContentStale(conn)
     };
 }