sensemaking 0.23.1 → 0.24.0

package/dist/cjs/scan/frontmatter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source excerpt, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["RESERVED_COLUMNS","countLines","looksLikeDatetime","mapValue","normalizeDate","normalizeText","parseFrontmatter","splitFrontmatter","Set","ACCEPTED_YAML_CODES","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","test","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","BigInt","isSafeInteger","JSON","stringify","s","n","i","charCodeAt","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","visit","Pair","_key","pair","isCollection","key","BREAK","push","firstLine","message","split","parseDocument","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QA+CGC;eAAAA;;QA5BAC;eAAAA;;QAkBAC;eAAAA;;QAZAC;eAAAA;;QAnBAC;eAAAA;;QAgFAC;eAAAA;;QAhCAC;eAAAA;;;oBA1DmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAI5C,IAAMP,mBAAmB,IAAIQ,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW;AAEtI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,IAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAEjD,SAASH,cAAcK,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,IAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,oBAAoB;AAEnB,SAASd,kBAAkBQ,KAAa;IAC7C,OAAOM,kBAAkBC,IAAI,CAACP;AAChC;AAIO,SAASN,cAAcM,KAAa;IACzC,IAAMQ,IAAIH,aAAaI,IAAI,CAACT;IAC5B,IAAIQ,MAAM,MAAM,OAAOR;IACvB,IAA6BQ,sBAAAA,OAApBE,OAAoBF,OAAdG,OAAcH,OAARI,OAAQJ;IAC7B,IAAMK,SAASD,SAASX,aAAaW,SAAS,MAAM,KAAKA,KAAKT,OAAO,CAAC,KAAK;IAC3E,IAAMW,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,AAAC,GAAS,OAAPF,QAAO,SAAO,AAAC,GAAwBA,OAAtBA,OAAOG,KAAK,CAAC,GAAG,IAAG,KAAmB,OAAhBH,OAAOG,KAAK,CAAC;IAC3H,IAAMC,aAAa,AAAC,GAAUN,OAARD,MAAK,KAAUI,OAAPH,MAAc,OAAPG;IACrC,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAejB,QAAQiB;AACxD;AAIO,SAASxB,SAASO,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOsB,OAAOtB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOkB,OAAOK,aAAa,CAACvB,SAASsB,OAAOtB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAON,cAAcM;IACpD,OAAOwB,KAAKC,SAAS,CAACzB;AACxB;AAIO,SAAST,WAAWmC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEX,MAAM,EAAEa,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAGO,SAAS9B,iBAAiBiC,GAAW;IAC1C,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAId,KAAK,CAACe,IAAI,CAAC,EAAE,CAAChB,MAAM;IACrC,IAAMqB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKpC,WAAW,OAAO;QAAEgC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKnB,KAAK,CAAC,GAAGoB,MAAMC,KAAK;QAAGH,MAAMC,KAAKnB,KAAK,CAACoB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACrB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAASuB,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCC,IAAAA,WAAK,EAACH,KAAK;QACTI,MAAAA,SAAAA,KAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACC,IAAAA,kBAAY,EAACD,KAAKE,GAAG,GAAG,OAAO/C;YACpCyC,QAAQ;YACR,OAAOC,WAAK,CAACM,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIP,OAAOD,SAASS,IAAI,CAAC,AAAC,YAAmB,OAARX,SAAQ;AAC/C;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASY,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAClD,OAAO,CAAC,SAAS;AACjD;AAEO,SAASP,iBAAiB2C,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,IAAMD,MAAMc,IAAAA,mBAAa,EAACrB,IAAI;QAAEsB,UAAU;IAAS;IACnD,IAAMC,UAAUhB,IAAIiB,MAAM,CAACC,MAAM,CAAC,SAACC;eAAQ,CAAC5D,oBAAoB6D,GAAG,CAACD,IAAIE,IAAI;;IAC5E,IAAIL,QAAQzC,MAAM,GAAG,GAAG;QACtB,IAAM+C,SAASN,QAAQzC,MAAM,GAAG,IAAI,AAAC,SAA2B,OAAnByC,QAAQzC,MAAM,GAAG,GAAE,YAAU;QAC1E,IAAMgD,aAAa,AAAC,GAAkCD,OAAhCX,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,GAAW,OAAPU;QACtDrB,SAASS,IAAI,CAAC,AAAC,YAA2Ea,OAAhExB,SAAQ,0DAAmE,OAAXwB;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOxB,IAAIyB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,IAAMI,cAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDX,SAASS,IAAI,CAAC,AAAC,YAA2Ea,OAAhExB,SAAQ,0DAAmE,OAAXwB;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS/D,WAAW,OAAO;QAAE+D,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,CAAA,OAAOC,qCAAP,SAAOA,KAAG,MAAM,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,IAAMD,cAAa;QACnBtB,SAASS,IAAI,CAAC,AAAC,YAAsBa,OAAXxB,SAAQ,KAAc,OAAXwB,aAAW;QAChD,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IACAzB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEuB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so reconcile.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// Only BAD_SCALAR_START qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain scalar, so `aliases: [@handle]` can only mean [\"@handle\"].\n// Every other code has a second reading (a swallowed key, a dropped URL, a silently overwritten duplicate) that would write values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// Exactly `s.split('\\n').length` without allocating the array: callers that need only the count\n// run it per file per feature on the parse hot path.\nexport function countLines(s: string): number {\n  let n = 1;\n  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) === 10) n++;\n  return n;\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can hold a value nobody meant: `created: {{date}}` is valid YAML for a flow map used as a mapping key, so it raises no error and stores {\"{ date }\": null}.\n// No error code catches that; yaml's own warning lacks a path and fires once per document, so this reports it here instead, with the path.\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Anything past ACCEPTED_YAML_CODES is quarantined: no frontmatter columns, `_parse_error` carries the reason. Recovering it would write values nobody wrote, worse than absence since no query can see it; content, links and sections never touch frontmatter, so a broken note stays searchable.\n// yaml's message continues onto a source snippet, so the first line is the sentence, minus the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["RESERVED_COLUMNS","countLines","looksLikeDatetime","mapValue","normalizeDate","normalizeText","parseFrontmatter","splitFrontmatter","Set","ACCEPTED_YAML_CODES","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","test","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","BigInt","isSafeInteger","JSON","stringify","s","n","i","charCodeAt","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","visit","Pair","_key","pair","isCollection","key","BREAK","push","firstLine","message","split","parseDocument","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QA+CGC;eAAAA;;QA5BAC;eAAAA;;QAkBAC;eAAAA;;QAZAC;eAAAA;;QAnBAC;eAAAA;;QAgFAC;eAAAA;;QAhCAC;eAAAA;;;oBA1DmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAI5C,IAAMP,mBAAmB,IAAIQ,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW;AAEtI,wJAAwJ;AACxJ,iJAAiJ;AACjJ,IAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAEjD,SAASH,cAAcK,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,IAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,oBAAoB;AAEnB,SAASd,kBAAkBQ,KAAa;IAC7C,OAAOM,kBAAkBC,IAAI,CAACP;AAChC;AAIO,SAASN,cAAcM,KAAa;IACzC,IAAMQ,IAAIH,aAAaI,IAAI,CAACT;IAC5B,IAAIQ,MAAM,MAAM,OAAOR;IACvB,IAA6BQ,sBAAAA,OAApBE,OAAoBF,OAAdG,OAAcH,OAARI,OAAQJ;IAC7B,IAAMK,SAASD,SAASX,aAAaW,SAAS,MAAM,KAAKA,KAAKT,OAAO,CAAC,KAAK;IAC3E,IAAMW,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,AAAC,GAAS,OAAPF,QAAO,SAAO,AAAC,GAAwBA,OAAtBA,OAAOG,KAAK,CAAC,GAAG,IAAG,KAAmB,OAAhBH,OAAOG,KAAK,CAAC;IAC3H,IAAMC,aAAa,AAAC,GAAUN,OAARD,MAAK,KAAUI,OAAPH,MAAc,OAAPG;IACrC,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAejB,QAAQiB;AACxD;AAIO,SAASxB,SAASO,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOsB,OAAOtB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOkB,OAAOK,aAAa,CAACvB,SAASsB,OAAOtB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAON,cAAcM;IACpD,OAAOwB,KAAKC,SAAS,CAACzB;AACxB;AAIO,SAAST,WAAWmC,CAAS;IAClC,IAAIC,IAAI;IACR,IAAK,IAAIC,IAAI,GAAGA,IAAIF,EAAEX,MAAM,EAAEa,IAAK,IAAIF,EAAEG,UAAU,CAACD,OAAO,IAAID;IAC/D,OAAOA;AACT;AAGO,SAAS9B,iBAAiBiC,GAAW;IAC1C,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAId,KAAK,CAACe,IAAI,CAAC,EAAE,CAAChB,MAAM;IACrC,IAAMqB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKpC,WAAW,OAAO;QAAEgC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKnB,KAAK,CAAC,GAAGoB,MAAMC,KAAK;QAAGH,MAAMC,KAAKnB,KAAK,CAACoB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACrB,MAAM;IAAE;AAC3F;AAEA,qLAAqL;AACrL,2IAA2I;AAC3I,SAASuB,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCC,IAAAA,WAAK,EAACH,KAAK;QACTI,MAAAA,SAAAA,KAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACC,IAAAA,kBAAY,EAACD,KAAKE,GAAG,GAAG,OAAO/C;YACpCyC,QAAQ;YACR,OAAOC,WAAK,CAACM,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIP,OAAOD,SAASS,IAAI,CAAC,AAAC,YAAmB,OAARX,SAAQ;AAC/C;AAEA,oSAAoS;AACpS,6IAA6I;AAC7I,SAASY,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAClD,OAAO,CAAC,SAAS;AACjD;AAEO,SAASP,iBAAiB2C,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,IAAMD,MAAMc,IAAAA,mBAAa,EAACrB,IAAI;QAAEsB,UAAU;IAAS;IACnD,IAAMC,UAAUhB,IAAIiB,MAAM,CAACC,MAAM,CAAC,SAACC;eAAQ,CAAC5D,oBAAoB6D,GAAG,CAACD,IAAIE,IAAI;;IAC5E,IAAIL,QAAQzC,MAAM,GAAG,GAAG;QACtB,IAAM+C,SAASN,QAAQzC,MAAM,GAAG,IAAI,AAAC,SAA2B,OAAnByC,QAAQzC,MAAM,GAAG,GAAE,YAAU;QAC1E,IAAMgD,aAAa,AAAC,GAAkCD,OAAhCX,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,GAAW,OAAPU;QACtDrB,SAASS,IAAI,CAAC,AAAC,YAA2Ea,OAAhExB,SAAQ,0DAAmE,OAAXwB;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOxB,IAAIyB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,IAAMI,cAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDX,SAASS,IAAI,CAAC,AAAC,YAA2Ea,OAAhExB,SAAQ,0DAAmE,OAAXwB;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS/D,WAAW,OAAO;QAAE+D,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,CAAA,OAAOC,qCAAP,SAAOA,KAAG,MAAM,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,IAAMD,cAAa;QACnBtB,SAASS,IAAI,CAAC,AAAC,YAAsBa,OAAXxB,SAAQ,KAAc,OAAXwB,aAAW;QAChD,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IACAzB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEuB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}

package/dist/cjs/scan/index.js

@@ -35,6 +35,7 @@ var _frontmatterts = require("./frontmatter.js");
 var _listts = require("./list.js");
 function parseFile(file) {
     var extractors = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : [], cfg = arguments.length > 2 ? arguments[2] : void 0;
+    // Preserve the authored bytes: feature extractors and section offsets use the raw document.
     var raw = (0, _nodefs.readFileSync)(file.absPath, 'utf8');
     var warnings = [];
     var _splitFrontmatter = (0, _frontmatterts.splitFrontmatter)(raw), fm = _splitFrontmatter.fm, content = _splitFrontmatter.body;

package/dist/cjs/scan/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","looksLikeDatetime","normalizeDate","parseFile","toPosixPath","file","extractors","cfg","raw","readFileSync","absPath","warnings","splitFrontmatter","fm","body","content","data","parseError","parseFrontmatter","relPath","mapped","Object","keys","key","has","push","value","mapValue","Number","isNaN","Date","parse","blocks","search","title","normalizeText","summary","text","textFromBlocks","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":";;;;;;;;;;;QAU2CA;eAAAA,+BAAgB;;QAElDC;eAAAA,iBAAS;;QAFTC;eAAAA,gCAAiB;;QAAEC;eAAAA,4BAAa;;QAoBzBC;eAAAA;;QAlBIC;eAAAA,mBAAW;;;sBAZF;uBACP;uBAGS;6BACkF;sBAO1E;AAkBhC,SAASD,UAAUE,IAAc;QAAEC,aAAAA,iEAAwB,EAAE,EAAEC;IACpE,IAAMC,MAAMC,IAAAA,oBAAY,EAACJ,KAAKK,OAAO,EAAE;IACvC,IAAMC,WAAqB,EAAE;IAE7B,IAA8BC,oBAAAA,IAAAA,+BAAgB,EAACJ,MAAvCK,KAAsBD,kBAAtBC,IAAIC,AAAMC,UAAYH,kBAAlBE;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIC,IAAAA,+BAAgB,EAACb,KAAKc,OAAO,EAAEN,IAAIF,WAAlIK,OAAqBH,KAArBG,MAAMC,aAAeJ,KAAfI;IACd,IAAMG,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAOC,IAAI,CAACN,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMO,MAAN;YACH,IAAIxB,+BAAgB,CAACyB,GAAG,CAACD,MAAM;gBAC7BZ,SAASc,IAAI,CAAC,AAAC,YAAwDF,OAA7ClB,KAAKc,OAAO,EAAC,kCAAoC,OAAJI,KAAI;gBAC3E;YACF;YACA,IAAMG,QAAQC,IAAAA,uBAAQ,EAACX,IAAI,CAACO,IAAI;YAChC,IAAI,OAAOG,UAAU,YAAYzB,IAAAA,gCAAiB,EAACyB,UAAUE,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACL,SAAS;gBAC5Ff,SAASc,IAAI,CAAC,AAAC,YAA4BF,OAAjBlB,KAAKc,OAAO,EAAC,MAAgCO,OAA5BH,KAAI,0BAA8B,OAANG,OAAM;YAC/E;YACAN,MAAM,CAACG,IAAI,GAAGG;QAChB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,yFAAyF;IACzF,4EAA4E;IAC5E,IAAMM,SAASD,IAAAA,cAAK,EAAChB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMkB,SAAS;QAAEC,OAAOC,IAAAA,4BAAa,EAACnB,KAAKkB,KAAK;QAAGE,SAASD,IAAAA,4BAAa,EAACnB,KAAKoB,OAAO;QAAGC,MAAMC,IAAAA,uBAAc,EAACN;IAAQ;IAEtH,OAAO;QACLO,KAAK;YACHpB,SAASd,KAAKc,OAAO;YACrBqB,SAASnC,KAAKmC,OAAO;YACrBC,SAASpC,KAAKoC,OAAO;YACrBC,MAAMrC,KAAKqC,IAAI;YACfC,SAAStC,KAAKsC,OAAO;YACrB3B,MAAMI;YACNH,YAAAA;YACAgB,QAAAA;YACAW,WAAWvB,OAAOwB,WAAW,CAACvC,WAAWwC,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYvC,KAAKO,SAASkB,QAAQjB,MAAMT,KAAKyB;iBAAQ;;QAC7I;QACArB,UAAAA;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors and section offsets use the raw document.\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","looksLikeDatetime","normalizeDate","parseFile","toPosixPath","file","extractors","cfg","raw","readFileSync","absPath","warnings","splitFrontmatter","fm","body","content","data","parseError","parseFrontmatter","relPath","mapped","Object","keys","key","has","push","value","mapValue","Number","isNaN","Date","parse","blocks","search","title","normalizeText","summary","text","textFromBlocks","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":";;;;;;;;;;;QAU2CA;eAAAA,+BAAgB;;QAElDC;eAAAA,iBAAS;;QAFTC;eAAAA,gCAAiB;;QAAEC;eAAAA,4BAAa;;QAoBzBC;eAAAA;;QAlBIC;eAAAA,mBAAW;;;sBAZF;uBACP;uBAGS;6BACkF;sBAO1E;AAkBhC,SAASD,UAAUE,IAAc;QAAEC,aAAAA,iEAAwB,EAAE,EAAEC;IACpE,4FAA4F;IAC5F,IAAMC,MAAMC,IAAAA,oBAAY,EAACJ,KAAKK,OAAO,EAAE;IACvC,IAAMC,WAAqB,EAAE;IAE7B,IAA8BC,oBAAAA,IAAAA,+BAAgB,EAACJ,MAAvCK,KAAsBD,kBAAtBC,IAAIC,AAAMC,UAAYH,kBAAlBE;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIC,IAAAA,+BAAgB,EAACb,KAAKc,OAAO,EAAEN,IAAIF,WAAlIK,OAAqBH,KAArBG,MAAMC,aAAeJ,KAAfI;IACd,IAAMG,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAOC,IAAI,CAACN,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMO,MAAN;YACH,IAAIxB,+BAAgB,CAACyB,GAAG,CAACD,MAAM;gBAC7BZ,SAASc,IAAI,CAAC,AAAC,YAAwDF,OAA7ClB,KAAKc,OAAO,EAAC,kCAAoC,OAAJI,KAAI;gBAC3E;YACF;YACA,IAAMG,QAAQC,IAAAA,uBAAQ,EAACX,IAAI,CAACO,IAAI;YAChC,IAAI,OAAOG,UAAU,YAAYzB,IAAAA,gCAAiB,EAACyB,UAAUE,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACL,SAAS;gBAC5Ff,SAASc,IAAI,CAAC,AAAC,YAA4BF,OAAjBlB,KAAKc,OAAO,EAAC,MAAgCO,OAA5BH,KAAI,0BAA8B,OAANG,OAAM;YAC/E;YACAN,MAAM,CAACG,IAAI,GAAGG;QAChB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,yFAAyF;IACzF,4EAA4E;IAC5E,IAAMM,SAASD,IAAAA,cAAK,EAAChB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMkB,SAAS;QAAEC,OAAOC,IAAAA,4BAAa,EAACnB,KAAKkB,KAAK;QAAGE,SAASD,IAAAA,4BAAa,EAACnB,KAAKoB,OAAO;QAAGC,MAAMC,IAAAA,uBAAc,EAACN;IAAQ;IAEtH,OAAO;QACLO,KAAK;YACHpB,SAASd,KAAKc,OAAO;YACrBqB,SAASnC,KAAKmC,OAAO;YACrBC,SAASpC,KAAKoC,OAAO;YACrBC,MAAMrC,KAAKqC,IAAI;YACfC,SAAStC,KAAKsC,OAAO;YACrB3B,MAAMI;YACNH,YAAAA;YACAgB,QAAAA;YACAW,WAAWvB,OAAOwB,WAAW,CAACvC,WAAWwC,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYvC,KAAKO,SAASkB,QAAQjB,MAAMT,KAAKyB;iBAAQ;;QAC7I;QACArB,UAAAA;IACF;AACF"}

package/dist/cjs/store/duckdb/fieldStats.js

@@ -202,13 +202,10 @@ function classifyVariantType(name, field) {
     if (name === 'VARCHAR') return 'text';
     throw new Error('duckdb: variant_typeof() returned unrecognized type "'.concat(name, '" for column "').concat(field, '"'));
 }
-// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in
-// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same
-// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:
-// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking
-// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no
-// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300
-// columns runs in 7ms.
+// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates
+// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);
+// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over
+// 300 columns runs in 7ms.
 var AGGREGATES_PER_QUERY = 16;
 function fieldStats(conn, columns, scopeWhere) {
     return _async_to_generator(function() {

package/dist/cjs/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in\n// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same\n// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:\n// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking\n// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no\n// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300\n// columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["fieldStats","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","conn","columns","scopeWhere","stats","start","chunk","exprs","stmt","row","length","slice","map","i","quoted","quoteIdent","prepare","join","get","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type","entries"],"mappings":";;;;+BAuBsBA;;;eAAAA;;;wBAvBK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG3B,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,AAAC,wDAA4EH,OAArBD,MAAK,kBAAsB,OAANC,OAAM;AACrG;AAEA,qGAAqG;AACrG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,uBAAuB;AACvB,IAAMI,uBAAuB;AAItB,SAAeP,WAAWQ,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;;YAChFC,OACGC,OACDC,OACAC,OAIAC,MACAC,KACD,2BAAA,mBAAA,uBAAA,WAAA;;;;oBATDL;oBACGC,QAAQ;;;yBAAGA,CAAAA,QAAQH,QAAQQ,MAAM,AAAD;;;;oBACjCJ,QAAQJ,QAAQS,KAAK,CAACN,OAAOA,QAAQL;oBACrCO,QAAQD,MAAMM,GAAG,CAAC,SAACjB,MAAMkB;wBAC7B,IAAMC,SAASC,IAAAA,oBAAU,EAACpB;wBAC1B,OAAO,AAAC,SAAuBkB,OAAfC,QAAO,UAAiDA,OAAzCD,GAAE,yCAAsEC,OAA/BA,QAAO,0BAAmDD,OAA3BC,QAAO,sBAAsB,OAAFD;oBACpI;oBACa;;wBAAMZ,KAAKe,OAAO,CAAC,AAAC,UAA8Cb,OAArCI,MAAMU,IAAI,CAAC,OAAM,sBAA+B,OAAXd;;;oBAAzEK,OAAO;oBACA;;wBAAMA,KAAKU,GAAG;;;oBAArBT,MAAO;oBACR,kCAAA,2BAAA;;;4BAAA,mCAAA,iBAAOI,oBAAGlB;gCACCc,OAE6BA;4BAF3C,IAAMU,MAAM,EAAEV,QAAAA,GAAG,CAAC,AAAC,IAAK,OAAFI,GAAI,cAAZJ,mBAAAA,QAA2B,IAAIW,KAAK,CAAC,KAAKC,MAAM,CAACC;4BAC/D,IAAMC,QAAQ,AAAC,qBAAG,IAAIC,IAAIL,IAAIP,GAAG,CAAC,SAACa;uCAAM/B,oBAAoB+B,GAAG9B;iCAAS+B,IAAI;4BAC7EtB,MAAMuB,IAAI,CAAC;gCAAE/B,OAAOD;gCAAMiC,UAAUC,QAAOpB,SAAAA,GAAG,CAAC,AAAC,IAAK,OAAFI,GAAI,cAAZJ,oBAAAA,SAAgB;gCAAIqB,MAAMP,MAAMN,IAAI,CAAC;4BAAK;wBACvF;wBAJA,IAAK,YAAmBX,MAAMyB,OAAO,yBAAhC,6BAAA,QAAA,yBAAA;;wBAAA;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBARqC1B,SAASL;;;;;;oBAcrD;;wBAAOI;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates\n// per projection, not the column width (301 columns: 1148ms unchunked vs 88ms at chunk size 25);\n// 16 sits inside the measured flat 10-25 range. sqlite/turso need none of this: GROUP_CONCAT over\n// 300 columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["fieldStats","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","conn","columns","scopeWhere","stats","start","chunk","exprs","stmt","row","length","slice","map","i","quoted","quoteIdent","prepare","join","get","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type","entries"],"mappings":";;;;+BAoBsBA;;;eAAAA;;;wBApBK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG3B,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,AAAC,wDAA4EH,OAArBD,MAAK,kBAAsB,OAANC,OAAM;AACrG;AAEA,kGAAkG;AAClG,iGAAiG;AACjG,kGAAkG;AAClG,2BAA2B;AAC3B,IAAMI,uBAAuB;AAItB,SAAeP,WAAWQ,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;;YAChFC,OACGC,OACDC,OACAC,OAIAC,MACAC,KACD,2BAAA,mBAAA,uBAAA,WAAA;;;;oBATDL;oBACGC,QAAQ;;;yBAAGA,CAAAA,QAAQH,QAAQQ,MAAM,AAAD;;;;oBACjCJ,QAAQJ,QAAQS,KAAK,CAACN,OAAOA,QAAQL;oBACrCO,QAAQD,MAAMM,GAAG,CAAC,SAACjB,MAAMkB;wBAC7B,IAAMC,SAASC,IAAAA,oBAAU,EAACpB;wBAC1B,OAAO,AAAC,SAAuBkB,OAAfC,QAAO,UAAiDA,OAAzCD,GAAE,yCAAsEC,OAA/BA,QAAO,0BAAmDD,OAA3BC,QAAO,sBAAsB,OAAFD;oBACpI;oBACa;;wBAAMZ,KAAKe,OAAO,CAAC,AAAC,UAA8Cb,OAArCI,MAAMU,IAAI,CAAC,OAAM,sBAA+B,OAAXd;;;oBAAzEK,OAAO;oBACA;;wBAAMA,KAAKU,GAAG;;;oBAArBT,MAAO;oBACR,kCAAA,2BAAA;;;4BAAA,mCAAA,iBAAOI,oBAAGlB;gCACCc,OAE6BA;4BAF3C,IAAMU,MAAM,EAAEV,QAAAA,GAAG,CAAC,AAAC,IAAK,OAAFI,GAAI,cAAZJ,mBAAAA,QAA2B,IAAIW,KAAK,CAAC,KAAKC,MAAM,CAACC;4BAC/D,IAAMC,QAAQ,AAAC,qBAAG,IAAIC,IAAIL,IAAIP,GAAG,CAAC,SAACa;uCAAM/B,oBAAoB+B,GAAG9B;iCAAS+B,IAAI;4BAC7EtB,MAAMuB,IAAI,CAAC;gCAAE/B,OAAOD;gCAAMiC,UAAUC,QAAOpB,SAAAA,GAAG,CAAC,AAAC,IAAK,OAAFI,GAAI,cAAZJ,oBAAAA,SAAgB;gCAAIqB,MAAMP,MAAMN,IAAI,CAAC;4BAAK;wBACvF;wBAJA,IAAK,YAAmBX,MAAMyB,OAAO,yBAAhC,6BAAA,QAAA,yBAAA;;wBAAA;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBARqC1B,SAASL;;;;;;oBAcrD;;wBAAOI;;;;IACT"}

package/dist/cjs/store/duckdb/lexical.d.cts

@@ -2,10 +2,10 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
-export declare function markContentStale(conn: Connection): void;
+export declare function markContentStale(conn: Connection): Promise<void>;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;
-    markStale: () => void;
+    markStale: () => Promise<void>;
 };
 export {};

package/dist/cjs/store/duckdb/lexical.d.ts

@@ -2,10 +2,10 @@ import type { Connection, LexicalHit, LexicalQueryOptions } from '../types.js';
 interface FtsIndexState {
     stale: boolean;
 }
-export declare function markContentStale(conn: Connection): void;
+export declare function markContentStale(conn: Connection): Promise<void>;
 export declare function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]>;
 export declare function createLexicalIndex(conn: Connection): {
     query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>;
-    markStale: () => void;
+    markStale: () => Promise<void>;
 };
 export {};

package/dist/cjs/store/duckdb/lexical.js

@@ -1,5 +1,5 @@
 // D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /
-// unspaced scripts, JS excerpts for every doc (no snippet()).
+// unspaced scripts, JS snippets for every doc (no snippet()).
 "use strict";
 Object.defineProperty(exports, "__esModule", {
     value: true
@@ -23,6 +23,7 @@ _export(exports, {
 });
 var _errorsts = require("../../errors.js");
 var _segmentts = require("../../text/segment.js");
+var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -240,50 +241,102 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
+// Durable across connections/processes (meta table, shared.ts); `stale` is the in-memory cache of
+// it for this connection's lifetime, so a watcher's repeat queries don't re-read meta every time.
+var FTS_STALE_META_KEY = 'fts_stale';
 // One state per connection, not per store instance: reconcileContent (reconcile.ts) marks it
 // stale from outside createLexicalIndex's closure, whether a one-shot open or a watcher's builder ran it.
 var ftsState = new WeakMap();
 function stateFor(conn) {
-    var state = ftsState.get(conn);
-    if (!state) {
-        state = {
-            stale: true
-        };
-        ftsState.set(conn, state);
-    }
-    return state;
+    return _async_to_generator(function() {
+        var state, _tmp;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    state = ftsState.get(conn);
+                    if (!!state) return [
+                        3,
+                        2
+                    ];
+                    _tmp = {};
+                    return [
+                        4,
+                        (0, _sharedts.getMeta)(conn, FTS_STALE_META_KEY)
+                    ];
+                case 1:
+                    // Anything but a persisted '0' (missing key, or '1') means a rebuild is owed: a cache that
+                    // never built an index, or one a prior process marked stale and never got to clear.
+                    state = (_tmp.stale = _state.sent() !== '0', _tmp);
+                    ftsState.set(conn, state);
+                    _state.label = 2;
+                case 2:
+                    return [
+                        2,
+                        state
+                    ];
+            }
+        });
+    })();
 }
 function markContentStale(conn) {
-    stateFor(conn).stale = true;
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    ftsState.set(conn, {
+                        stale: true
+                    });
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(conn, FTS_STALE_META_KEY, '1')
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
 }
-// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
-// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
+// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token,
+// so unspaced terms use contains() and quoted phrases use the shared authored-field verifier.
 function splitTerms(terms) {
     var words = [];
+    var bareWords = [];
+    var phraseWords = [];
     var substrings = [];
+    var phrases = [];
+    var emptyPhrase = false;
     var withoutPhrases = terms.replace(/"([^"]*)"/g, function(_m, inner) {
         var phrase = inner.trim();
-        if (phrase.length === 0) return ' ';
-        substrings.push(phrase);
-        if (!(0, _segmentts.hasUnspacedRun)(phrase)) {
-            var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+        var phraseTokens = (0, _segmentts.searchTokens)(phrase);
+        if (phraseTokens.length === 0) {
+            emptyPhrase = true;
+            return ' ';
+        }
+        phrases.push(phraseTokens);
+        var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+        try {
+            for(var _iterator = phraseTokens[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                var token = _step.value;
+                if (token.unspaced) substrings.push(token.text);
+                else {
+                    words.push(token.text);
+                    phraseWords.push(token.text);
+                }
+            }
+        } catch (err) {
+            _didIteratorError = true;
+            _iteratorError = err;
+        } finally{
             try {
-                for(var _iterator = phrase.split(/\s+/)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                    var w = _step.value;
-                    if (w.length > 0) words.push(w);
+                if (!_iteratorNormalCompletion && _iterator.return != null) {
+                    _iterator.return();
                 }
-            } catch (err) {
-                _didIteratorError = true;
-                _iteratorError = err;
             } finally{
-                try {
-                    if (!_iteratorNormalCompletion && _iterator.return != null) {
-                        _iterator.return();
-                    }
-                } finally{
-                    if (_didIteratorError) {
-                        throw _iteratorError;
-                    }
+                if (_didIteratorError) {
+                    throw _iteratorError;
                 }
             }
         }
@@ -294,8 +347,30 @@ function splitTerms(terms) {
         for(var _iterator = withoutPhrases.split(/\s+/)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var tok = _step.value;
             if (tok.length === 0) continue;
-            if ((0, _segmentts.hasUnspacedRun)(tok)) substrings.push(tok);
-            else words.push(tok);
+            var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+            try {
+                for(var _iterator1 = (0, _segmentts.searchTokens)(tok)[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                    var token = _step1.value;
+                    if (token.unspaced) substrings.push(token.text);
+                    else {
+                        words.push(token.text);
+                        bareWords.push(token.text);
+                    }
+                }
+            } catch (err) {
+                _didIteratorError1 = true;
+                _iteratorError1 = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                        _iterator1.return();
+                    }
+                } finally{
+                    if (_didIteratorError1) {
+                        throw _iteratorError1;
+                    }
+                }
+            }
         }
     } catch (err) {
         _didIteratorError = true;
@@ -313,11 +388,15 @@ function splitTerms(terms) {
     }
     return {
         words: words,
-        substrings: substrings
+        bareWords: bareWords,
+        phraseWords: phraseWords,
+        substrings: substrings,
+        phrases: phrases,
+        emptyPhrase: emptyPhrase
     };
 }
-// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only
-// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).
+// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays
+// the full rebuild whenever `state.stale`, which meta.fts_stale keeps true across processes until cleared below.
 function ensureFtsFresh(conn, state) {
     return _async_to_generator(function() {
         return _ts_generator(this, function(_state) {
@@ -346,6 +425,14 @@ function ensureFtsFresh(conn, state) {
                                                 conn.exec("PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)")
                                             ];
                                         case 1:
+                                            _state.sent();
+                                            // Same transaction as the rebuild: a crash before commit leaves the mark set (a redundant
+                                            // rebuild next time), never a cleared mark serving a half-built index as fresh.
+                                            return [
+                                                4,
+                                                (0, _sharedts.setMeta)(conn, FTS_STALE_META_KEY, '0')
+                                            ];
+                                        case 2:
                                             _state.sent();
                                             return [
                                                 2
@@ -367,13 +454,13 @@ function ensureFtsFresh(conn, state) {
 }
 // Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
 // score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
-function buildScoreAndGate(words, substrings) {
+function buildScoreAndGate(words, bareWords, phraseWords, substrings) {
     var scoreParts = [];
     var gateParts = [];
     var scoreParams = [];
     var gateParams = [];
     if (words.length > 0) {
-        var wordQuery = words.join(' ');
+        var wordQuery = words.map(_segmentts.foldForSearch).join(' ');
         var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
         try {
             for(var _iterator = FIELDS[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
@@ -395,38 +482,63 @@ function buildScoreAndGate(words, substrings) {
                 }
             }
         }
-        gateParts.push('fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL');
-        gateParams.push(wordQuery);
+        if (bareWords.length > 0) {
+            var nativeGate = 'fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL';
+            gateParts.push(nativeGate);
+            gateParams.push(bareWords.map(_segmentts.foldForSearch).join(' '));
+        }
+        if (phraseWords.length > 0) {
+            var _gateParams;
+            var nativeGate1 = 'fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL';
+            var fallback = phraseWords.map(function(_word) {
+                return "(".concat(FIELDS.map(function(field) {
+                    return "contains(strip_accents(lower(content.".concat(field, ")), ?)");
+                }).join(' OR '), ")");
+            }).join(' AND ');
+            gateParts.push("(".concat(nativeGate1, " OR (").concat(fallback, "))"));
+            gateParams.push(phraseWords.map(_segmentts.foldForSearch).join(' '));
+            (_gateParams = gateParams).push.apply(_gateParams, _to_consumable_array(phraseWords.flatMap(function(word) {
+                return FIELDS.map(function() {
+                    return (0, _segmentts.foldForSearch)(word);
+                });
+            })));
+        }
     }
     var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
     try {
-        for(var _iterator1 = substrings[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+        var _loop = function() {
             var raw = _step1.value;
+            var _gateParams;
             var needle = raw.toLowerCase();
-            var _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+            var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
             try {
-                for(var _iterator2 = FIELDS[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true){
-                    var field1 = _step2.value;
-                    scoreParts.push("".concat(FIELD_WEIGHT[field1], ".0 * CAST(contains(lower(content.").concat(field1, "), ?) AS INTEGER)"));
-                    scoreParams.push(needle);
+                for(var _iterator = FIELDS[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                    var field = _step.value;
+                    scoreParts.push("".concat(FIELD_WEIGHT[field], ".0 * CAST(contains(strip_accents(lower(content.").concat(field, ")), ?) AS INTEGER)"));
+                    scoreParams.push((0, _segmentts.foldForSearch)(needle));
                 }
             } catch (err) {
-                _didIteratorError2 = true;
-                _iteratorError2 = err;
+                _didIteratorError = true;
+                _iteratorError = err;
             } finally{
                 try {
-                    if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
-                        _iterator2.return();
+                    if (!_iteratorNormalCompletion && _iterator.return != null) {
+                        _iterator.return();
                     }
                 } finally{
-                    if (_didIteratorError2) {
-                        throw _iteratorError2;
+                    if (_didIteratorError) {
+                        throw _iteratorError;
                     }
                 }
             }
-            gateParts.push("contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)");
-            gateParams.push(needle);
-        }
+            gateParts.push("(".concat(FIELDS.map(function(field) {
+                return "contains(strip_accents(lower(content.".concat(field, ")), ?)");
+            }).join(' OR '), ")"));
+            (_gateParams = gateParams).push.apply(_gateParams, _to_consumable_array(FIELDS.map(function() {
+                return (0, _segmentts.foldForSearch)(needle);
+            })));
+        };
+        for(var _iterator1 = substrings[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true)_loop();
     } catch (err) {
         _didIteratorError1 = true;
         _iteratorError1 = err;
@@ -443,13 +555,13 @@ function buildScoreAndGate(words, substrings) {
     }
     return {
         scoreSql: scoreParts.join(' + '),
-        gateSql: gateParts.join(' AND '),
+        gateSql: gateParts.join(' AND ') || 'TRUE',
         params: _to_consumable_array(scoreParams).concat(_to_consumable_array(gateParams))
     };
 }
 function queryLexical(conn, terms, opts, state) {
     return _async_to_generator(function() {
-        var _stmt, unsupported, whereJoin, whereCond, scopeCond, limit, _splitTerms, words, substrings, _buildScoreAndGate, scoreSql, gateSql, params, sql, stmt;
+        var _stmt, unsupported, whereJoin, whereCond, scopeCond, limit, _splitTerms, words, bareWords, phraseWords, substrings, phrases, emptyPhrase, _buildScoreAndGate, scoreSql, gateSql, params, selectFields, sqlLimit, sql, stmt, rows, matches;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -458,8 +570,8 @@ function queryLexical(conn, terms, opts, state) {
                         throw new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "duckdb" does not implement FTS5\'s '.concat(unsupported.label, ' ("').concat(unsupported.token, '") in this build; rephrase "').concat(terms.trim(), '" without it, or set "store" to "sqlite" in this tree\'s config to search it as written'));
                     }
                     whereJoin = opts.whereJoin, whereCond = opts.whereCond, scopeCond = opts.scopeCond, limit = opts.limit;
-                    _splitTerms = splitTerms(terms), words = _splitTerms.words, substrings = _splitTerms.substrings;
-                    if (words.length === 0 && substrings.length === 0) return [
+                    _splitTerms = splitTerms(terms), words = _splitTerms.words, bareWords = _splitTerms.bareWords, phraseWords = _splitTerms.phraseWords, substrings = _splitTerms.substrings, phrases = _splitTerms.phrases, emptyPhrase = _splitTerms.emptyPhrase;
+                    if (emptyPhrase || words.length === 0 && substrings.length === 0) return [
                         2,
                         []
                     ];
@@ -475,8 +587,10 @@ function queryLexical(conn, terms, opts, state) {
                     _state.sent();
                     _state.label = 2;
                 case 2:
-                    _buildScoreAndGate = buildScoreAndGate(words, substrings), scoreSql = _buildScoreAndGate.scoreSql, gateSql = _buildScoreAndGate.gateSql, params = _buildScoreAndGate.params;
-                    sql = 'SELECT path, NULL AS hit FROM (\n    SELECT content."path" AS path, ('.concat(scoreSql, ") AS score\n    FROM content\n    ").concat(whereJoin, "\n    WHERE ").concat(gateSql, "\n    ").concat(whereCond, " ").concat(scopeCond, "\n  ) sq ORDER BY score DESC, path LIMIT ?");
+                    _buildScoreAndGate = buildScoreAndGate(words, bareWords, phraseWords, substrings), scoreSql = _buildScoreAndGate.scoreSql, gateSql = _buildScoreAndGate.gateSql, params = _buildScoreAndGate.params;
+                    selectFields = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';
+                    sqlLimit = phrases.length > 0 ? '' : ' LIMIT ?';
+                    sql = "SELECT path, score".concat(phrases.length > 0 ? ', title, summary, text' : '', ' FROM (\n    SELECT content."path" AS path, (').concat(scoreSql, ") AS score").concat(selectFields, "\n    FROM content\n    ").concat(whereJoin, "\n    WHERE ").concat(gateSql, "\n    ").concat(whereCond, " ").concat(scopeCond, "\n    ) sq ORDER BY score DESC, path").concat(sqlLimit);
                     return [
                         4,
                         conn.prepare(sql)
@@ -485,14 +599,28 @@ function queryLexical(conn, terms, opts, state) {
                     stmt = _state.sent();
                     return [
                         4,
-                        (_stmt = stmt).all.apply(_stmt, _to_consumable_array(params).concat([
+                        (_stmt = stmt).all.apply(_stmt, _to_consumable_array(params).concat(_to_consumable_array(phrases.length > 0 ? [] : [
                             limit
-                        ]))
+                        ])))
                     ];
                 case 4:
+                    rows = _state.sent();
+                    matches = phrases.length === 0 ? rows : rows.filter(function(row) {
+                        return phrases.every(function(phrase) {
+                            return FIELDS.some(function(field) {
+                                var _row_field;
+                                return (0, _segmentts.matchesSearchPhrase)((_row_field = row[field]) !== null && _row_field !== void 0 ? _row_field : '', phrase);
+                            });
+                        });
+                    });
                     return [
                         2,
-                        _state.sent()
+                        matches.slice(0, limit).map(function(param) {
+                            var path = param.path;
+                            return {
+                                path: path
+                            };
+                        })
                     ];
             }
         });
@@ -501,7 +629,30 @@ function queryLexical(conn, terms, opts, state) {
 function createLexicalIndex(conn) {
     return {
         query: function query(terms, opts) {
-            return queryLexical(conn, terms, opts, stateFor(conn));
+            return _async_to_generator(function() {
+                var _tmp;
+                return _ts_generator(this, function(_state) {
+                    switch(_state.label){
+                        case 0:
+                            _tmp = [
+                                conn,
+                                terms,
+                                opts
+                            ];
+                            return [
+                                4,
+                                stateFor(conn)
+                            ];
+                        case 1:
+                            return [
+                                2,
+                                queryLexical.apply(void 0, _tmp.concat([
+                                    _state.sent()
+                                ]))
+                            ];
+                    }
+                });
+            })();
         },
         markStale: function markStale() {
             return markContentStale(conn);