sensemaking 0.7.2 → 0.9.0

package/dist/esm/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\n// Table output is for reading; a row wider than the terminal wraps into an unreadable\n// block (a `find` header measured 330 chars once `summary` is present, which drove one\n// user to pipe --format json through a script for every interactive query). Widest\n// columns give up space first, down to a floor, and only as far as the terminal needs.\n// --format json is never truncated: it is the machine-readable surface.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: 'table' | 'json', width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nfunction featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, `${featuresLine(result.features)}\\n`, renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push('\\nhubs: off (features.rank)');\n  parts.push('\\nrecent:', renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","renderRows","console","log","length","warn","Math","round","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","process","stdout","columns","JSON","stringify","Object","keys","cells","map","row","col","natural","i","clip","w","padEnd","slice","formatRow","values","join","trimEnd","repeat","featuresLine","features","off","name","on","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,WAAWH,MAAMC;IAClCG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,sFAAsF;AACtF,uFAAuF;AACvF,mFAAmF;AACnF,uFAAuF;AACvF,wEAAwE;AACxE,MAAMI,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAOP,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMU,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACd,KAAKe,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASX,WAAWH,IAAW,EAAEC,MAAwB,EAAEuB,QAAQC,QAAQC,MAAM,CAACC,OAAO;IACvF,IAAI1B,WAAW,QAAQ,OAAO2B,KAAKC,SAAS,CAAC7B,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMqB,UAAUG,OAAOC,IAAI,CAAC/B,IAAI,CAAC,EAAE;IACnC,MAAMgC,QAAQhC,KAAKiC,GAAG,CAAC,CAACC,MAAQP,QAAQM,GAAG,CAAC,CAACE,MAAQxC,KAAKuC,GAAG,CAACC,IAAI;IAClE,MAAMC,UAAUT,QAAQM,GAAG,CAAC,CAACE,KAAKE,IAAM7B,KAAKe,GAAG,CAACY,IAAI7B,MAAM,KAAK0B,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAAC/B,MAAM;IAChG,MAAMO,SAASW,SAASA,QAAQd,aAAaE,UAAUwB,SAASZ,SAASY;IAEzE,MAAME,OAAO,CAAC1C,OAAe2C,IAAe3C,MAAMU,MAAM,IAAIiC,IAAI3C,MAAM4C,MAAM,CAACD,KAAK,GAAG3C,MAAM6C,KAAK,CAAC,GAAGF,IAAI,KAAK5B,WAAW;IACxH,MAAM+B,YAAY,CAACC,SACjBA,OACGV,GAAG,CAAC,CAACrC,OAAOyC,IAAMC,KAAK1C,OAAOiB,MAAM,CAACwB,EAAE,GACvCO,IAAI,CAAC,MACLC,OAAO;IAEZ,OAAO;QAACH,UAAUf;QAAUd,OAAOoB,GAAG,CAAC,CAACM,IAAM,IAAIO,MAAM,CAACP,IAAIK,IAAI,CAAC;WAAUZ,MAAMC,GAAG,CAACS;KAAW,CAACE,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,SAASG,aAAaC,QAAyC;IAC7D,MAAMC,MAAMD,SAASC,GAAG,CAAC3C,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE0C,SAASC,GAAG,CAAChB,GAAG,CAAC,CAACiB,OAAS,GAAGA,KAAK,WAAW,EAAEA,KAAK,CAAC,CAAC,EAAEN,IAAI,CAAC,OAAO,GAAG;IACzH,OAAO,CAAC,UAAU,EAAEI,SAASG,EAAE,CAACP,IAAI,CAAC,QAAQK,KAAK;AACpD;AAEA,OAAO,SAASG,UAAUC,MAA6J;IACrL,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEhD,KAAKC,KAAK,CAAC4C,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE,GAAGV,aAAaM,OAAOL,QAAQ,EAAE,EAAE,CAAC;QAAE7C,WAAWkD,OAAOK,MAAM,EAAE;KAAS;IACnK,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACpD,MAAM,EAAEgD,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACpD,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI+C,OAAOQ,IAAI,CAACvD,MAAM,GAAG,GAAGgD,MAAMM,IAAI,CAAC,0BAA0BzD,WAAWkD,OAAOQ,IAAI,EAAE;SACpF,IAAIR,OAAOL,QAAQ,CAACC,GAAG,CAACa,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC;IAC1DN,MAAMM,IAAI,CAAC,aAAazD,WAAWkD,OAAOU,MAAM,EAAE;IAClD,OAAOT,MAAMV,IAAI,CAAC;AACpB;AAEA,OAAO,SAASoB,WAAWX,MAAiO;IAC1P,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAKxE,MAAM,IAAIkC,OAAOuC,OAAO,CAAChB,OAAOiB,WAAW,EAAGL,MAAML,IAAI,CAAC,CAAC,EAAE,EAAEQ,IAAI,EAAE,EAAExE,OAAO;IAC9F,IAAIyD,OAAOkB,QAAQ,CAACjE,MAAM,GAAG,GAAG;QAC9B2D,MAAML,IAAI,CAAC,IAAI;QACf,KAAK,MAAMY,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAML,IAAI,CAAC,CAAC,EAAE,EAAE,IAAId,MAAM,CAAC0B,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;IAChJ,OAAO,IAAId,OAAOJ,GAAG,CAACa,QAAQ,CAAC,aAAaG,MAAML,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOJ,GAAG,CAACa,QAAQ,CAAC,UAAU;QAChCG,MAAML,IAAI,CAAC,IAAI;QACf,OAAOK,MAAMrB,IAAI,CAAC;IACpB;IACA,MAAMiC,WAAW,CAACC,OAAeC,OAAiB9D;QAChD,IAAIA,QAAQ,GAAGgD,MAAML,IAAI,CAAC,GAAGkB,MAAM,EAAE,EAAE7D,MAAM,GAAG,EAAE8D,MAAMnC,IAAI,CAAC,QAAQ3B,QAAQ8D,MAAMzE,MAAM,GAAG,CAAC,GAAG,EAAEW,QAAQ8D,MAAMzE,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAI+C,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGjB,MAAML,IAAI,CAAC;IAC1FiB,SAAS,aAAaxB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;IAC3DH,SAAS,cAAcxB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;IAChEJ,SAAS,aAAaxB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC7D,OAAOjB,MAAMrB,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\n// Table output is for reading; a row wider than the terminal wraps into an unreadable\n// block (a `find` header measured 330 chars once `summary` is present, which drove one\n// user to pipe --format json through a script for every interactive query). Widest\n// columns give up space first, down to a floor, and only as far as the terminal needs.\n// --format json is never truncated: it is the machine-readable surface.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: 'table' | 'json', width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// `embed` is not a features-block key (it's derived from presets), so its off hint reads\n// differently from links/sections/rank's plain `features.<name>`.\nexport function featureOffHint(name: string): string {\n  return name === 'embed' ? 'no preset has semantic on' : `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number }>): string[] {\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded`)];\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; presets: Array<{ name: string; files: number; embedded: number }>; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:', renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","renderRows","console","log","length","warn","Math","round","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","process","stdout","columns","JSON","stringify","Object","keys","cells","map","row","col","natural","i","clip","w","padEnd","slice","formatRow","values","join","trimEnd","repeat","featureOffHint","name","featuresLine","features","off","on","presetsLines","presets","p","files","embedded","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,WAAWH,MAAMC;IAClCG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,sFAAsF;AACtF,uFAAuF;AACvF,mFAAmF;AACnF,uFAAuF;AACvF,wEAAwE;AACxE,MAAMI,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAOP,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMU,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACd,KAAKe,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASX,WAAWH,IAAW,EAAEC,MAAwB,EAAEuB,QAAQC,QAAQC,MAAM,CAACC,OAAO;IACvF,IAAI1B,WAAW,QAAQ,OAAO2B,KAAKC,SAAS,CAAC7B,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMqB,UAAUG,OAAOC,IAAI,CAAC/B,IAAI,CAAC,EAAE;IACnC,MAAMgC,QAAQhC,KAAKiC,GAAG,CAAC,CAACC,MAAQP,QAAQM,GAAG,CAAC,CAACE,MAAQxC,KAAKuC,GAAG,CAACC,IAAI;IAClE,MAAMC,UAAUT,QAAQM,GAAG,CAAC,CAACE,KAAKE,IAAM7B,KAAKe,GAAG,CAACY,IAAI7B,MAAM,KAAK0B,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAAC/B,MAAM;IAChG,MAAMO,SAASW,SAASA,QAAQd,aAAaE,UAAUwB,SAASZ,SAASY;IAEzE,MAAME,OAAO,CAAC1C,OAAe2C,IAAe3C,MAAMU,MAAM,IAAIiC,IAAI3C,MAAM4C,MAAM,CAACD,KAAK,GAAG3C,MAAM6C,KAAK,CAAC,GAAGF,IAAI,KAAK5B,WAAW;IACxH,MAAM+B,YAAY,CAACC,SACjBA,OACGV,GAAG,CAAC,CAACrC,OAAOyC,IAAMC,KAAK1C,OAAOiB,MAAM,CAACwB,EAAE,GACvCO,IAAI,CAAC,MACLC,OAAO;IAEZ,OAAO;QAACH,UAAUf;QAAUd,OAAOoB,GAAG,CAAC,CAACM,IAAM,IAAIO,MAAM,CAACP,IAAIK,IAAI,CAAC;WAAUZ,MAAMC,GAAG,CAACS;KAAW,CAACE,IAAI,CAAC;AACzG;AAEA,8EAA8E;AAE9E,yFAAyF;AACzF,kEAAkE;AAClE,OAAO,SAASG,eAAeC,IAAY;IACzC,OAAOA,SAAS,UAAU,8BAA8B,CAAC,SAAS,EAAEA,MAAM;AAC5E;AAEA,OAAO,SAASC,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAAC7C,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE4C,SAASC,GAAG,CAAClB,GAAG,CAAC,CAACe,OAAS,GAAGA,KAAK,EAAE,EAAED,eAAeC,MAAM,CAAC,CAAC,EAAEJ,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAEM,SAASE,EAAE,CAACR,IAAI,CAAC,QAAQO,KAAK;AACpD;AAEA,OAAO,SAASE,aAAaC,OAAiE;IAC5F,OAAO;QAAC;WAAeA,QAAQrB,GAAG,CAAC,CAACsB,IAAM,CAAC,EAAE,EAAEA,EAAEP,IAAI,CAAC,EAAE,EAAEO,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,CAAC;KAAE;AACvG;AAEA,OAAO,SAASC,UAAUC,MAAgO;IACxP,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEtD,KAAKC,KAAK,CAACkD,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,IAAI,CAAC;QAAEd,aAAaU,OAAOT,QAAQ;QAAG;WAAOG,aAAaM,OAAOL,OAAO;QAAG;QAAInD,WAAWwD,OAAOK,MAAM,EAAE;KAAS;IACrM,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC1D,MAAM,EAAEsD,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC1D,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIqD,OAAOQ,IAAI,CAAC7D,MAAM,GAAG,GAAGsD,MAAMM,IAAI,CAAC,0BAA0B/D,WAAWwD,OAAOQ,IAAI,EAAE;SACpF,IAAIR,OAAOT,QAAQ,CAACC,GAAG,CAACiB,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC,CAAC,aAAa,EAAEnB,eAAe,QAAQ,CAAC,CAAC;IACnGa,MAAMM,IAAI,CAAC,aAAa/D,WAAWwD,OAAOU,MAAM,EAAE;IAClD,OAAOT,MAAMhB,IAAI,CAAC;AACpB;AAEA,OAAO,SAAS0B,WAAWX,MAAwP;IACjR,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAK9E,MAAM,IAAIkC,OAAO6C,OAAO,CAAChB,OAAOiB,WAAW,EAAGL,MAAML,IAAI,CAAC,CAAC,EAAE,EAAEQ,IAAI,EAAE,EAAE9E,OAAO;IAC9F,IAAI+D,OAAOkB,QAAQ,CAACvE,MAAM,GAAG,GAAG;QAC9BiE,MAAML,IAAI,CAAC,IAAI;QACf,KAAK,MAAMY,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAML,IAAI,CAAC,CAAC,EAAE,EAAE,IAAIpB,MAAM,CAACgC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAId,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAACvE,MAAM,EAAEiE,MAAML,IAAI,CAAC,CAAC,IAAI,EAAEP,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAACvE,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAIqD,OAAOR,GAAG,CAACiB,QAAQ,CAAC,aAAaG,MAAML,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOR,GAAG,CAACiB,QAAQ,CAAC,UAAU;QAChCG,MAAML,IAAI,CAAC,IAAI;QACf,OAAOK,MAAM3B,IAAI,CAAC;IACpB;IACA,MAAMwC,WAAW,CAACC,OAAeC,OAAiBrE;QAChD,IAAIA,QAAQ,GAAGsD,MAAML,IAAI,CAAC,GAAGmB,MAAM,EAAE,EAAEpE,MAAM,GAAG,EAAEqE,MAAM1C,IAAI,CAAC,QAAQ3B,QAAQqE,MAAMhF,MAAM,GAAG,CAAC,GAAG,EAAEW,QAAQqE,MAAMhF,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAIqD,OAAO4B,aAAa,GAAG5B,OAAO6B,eAAe,GAAG7B,OAAO8B,cAAc,GAAG,GAAGlB,MAAML,IAAI,CAAC;IAC1FkB,SAAS,aAAazB,OAAO+B,QAAQ,EAAE/B,OAAO4B,aAAa;IAC3DH,SAAS,cAAczB,OAAOgC,UAAU,EAAEhC,OAAO6B,eAAe;IAChEJ,SAAS,aAAazB,OAAOiC,SAAS,EAAEjC,OAAO8B,cAAc;IAC7D,OAAOlB,MAAM3B,IAAI,CAAC;AACpB"}

package/dist/esm/progress.d.ts

@@ -0,0 +1,5 @@
+export interface Progress {
+    tick(done: number): void;
+    finish(): void;
+}
+export declare function progress(label: string, total: number, threshold?: number): Progress;

package/dist/esm/progress.js

@@ -0,0 +1,44 @@
+// Long-work progress on stderr only: stdout stays the parseable answer, so progress can
+// never corrupt --format json (the same channel warnings already use). TTY-aware: a human
+// terminal gets one self-overwriting line; a non-TTY consumer (agent, pipe, CI) gets at
+// most a few append-only lines -- an agent transcript gains lines, not redraw frames.
+const NON_TTY_MILESTONES = 4;
+// `total` known upfront; `label` names the work ("embedding notes", "reparsing files").
+// Silent entirely below `threshold` -- short work needs no narration.
+export function progress(label, total, threshold = 200) {
+    if (total < threshold) return {
+        tick () {},
+        finish () {}
+    };
+    const started = Date.now();
+    const isTTY = process.stderr.isTTY === true;
+    let lastShown = 0;
+    let wrote = false;
+    const line = (done)=>{
+        const elapsed = (Date.now() - started) / 1000;
+        // Estimate from observed rate, only once there is a rate to observe.
+        const eta = done > 0 && done < total ? `, ~${Math.max(1, Math.round(elapsed / done * (total - done)))}s left` : '';
+        return `${label}: ${done}/${total}${eta}`;
+    };
+    return {
+        tick (done) {
+            if (isTTY) {
+                process.stderr.write(`\r${line(done)}`);
+                wrote = true;
+                return;
+            }
+            // Sparse milestones: first tick, then every 1/NON_TTY_MILESTONES of the total.
+            if (done === lastShown) return;
+            if (lastShown === 0 || done - lastShown >= total / NON_TTY_MILESTONES) {
+                process.stderr.write(`${line(done)}\n`);
+                lastShown = done;
+                wrote = true;
+            }
+        },
+        finish () {
+            if (!wrote) return;
+            if (isTTY) process.stderr.write(`\r${line(total)}\n`);
+            else if (lastShown < total) process.stderr.write(`${line(total)}\n`);
+        }
+    };
+}

package/dist/esm/progress.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/progress.ts"],"sourcesContent":["// Long-work progress on stderr only: stdout stays the parseable answer, so progress can\n// never corrupt --format json (the same channel warnings already use). TTY-aware: a human\n// terminal gets one self-overwriting line; a non-TTY consumer (agent, pipe, CI) gets at\n// most a few append-only lines -- an agent transcript gains lines, not redraw frames.\n\nconst NON_TTY_MILESTONES = 4;\n\nexport interface Progress {\n  tick(done: number): void;\n  finish(): void;\n}\n\n// `total` known upfront; `label` names the work (\"embedding notes\", \"reparsing files\").\n// Silent entirely below `threshold` -- short work needs no narration.\nexport function progress(label: string, total: number, threshold = 200): Progress {\n  if (total < threshold) return { tick() {}, finish() {} };\n\n  const started = Date.now();\n  const isTTY = process.stderr.isTTY === true;\n  let lastShown = 0;\n  let wrote = false;\n\n  const line = (done: number) => {\n    const elapsed = (Date.now() - started) / 1000;\n    // Estimate from observed rate, only once there is a rate to observe.\n    const eta = done > 0 && done < total ? `, ~${Math.max(1, Math.round((elapsed / done) * (total - done)))}s left` : '';\n    return `${label}: ${done}/${total}${eta}`;\n  };\n\n  return {\n    tick(done: number) {\n      if (isTTY) {\n        process.stderr.write(`\\r${line(done)}`);\n        wrote = true;\n        return;\n      }\n      // Sparse milestones: first tick, then every 1/NON_TTY_MILESTONES of the total.\n      if (done === lastShown) return;\n      if (lastShown === 0 || done - lastShown >= total / NON_TTY_MILESTONES) {\n        process.stderr.write(`${line(done)}\\n`);\n        lastShown = done;\n        wrote = true;\n      }\n    },\n    finish() {\n      if (!wrote) return;\n      if (isTTY) process.stderr.write(`\\r${line(total)}\\n`);\n      else if (lastShown < total) process.stderr.write(`${line(total)}\\n`);\n    },\n  };\n}\n"],"names":["NON_TTY_MILESTONES","progress","label","total","threshold","tick","finish","started","Date","now","isTTY","process","stderr","lastShown","wrote","line","done","elapsed","eta","Math","max","round","write"],"mappings":"AAAA,wFAAwF;AACxF,0FAA0F;AAC1F,wFAAwF;AACxF,sFAAsF;AAEtF,MAAMA,qBAAqB;AAO3B,wFAAwF;AACxF,sEAAsE;AACtE,OAAO,SAASC,SAASC,KAAa,EAAEC,KAAa,EAAEC,YAAY,GAAG;IACpE,IAAID,QAAQC,WAAW,OAAO;QAAEC,SAAQ;QAAGC,WAAU;IAAE;IAEvD,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMC,QAAQC,QAAQC,MAAM,CAACF,KAAK,KAAK;IACvC,IAAIG,YAAY;IAChB,IAAIC,QAAQ;IAEZ,MAAMC,OAAO,CAACC;QACZ,MAAMC,UAAU,AAACT,CAAAA,KAAKC,GAAG,KAAKF,OAAM,IAAK;QACzC,qEAAqE;QACrE,MAAMW,MAAMF,OAAO,KAAKA,OAAOb,QAAQ,CAAC,GAAG,EAAEgB,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAAC,AAACJ,UAAUD,OAASb,CAAAA,QAAQa,IAAG,IAAK,MAAM,CAAC,GAAG;QAClH,OAAO,GAAGd,MAAM,EAAE,EAAEc,KAAK,CAAC,EAAEb,QAAQe,KAAK;IAC3C;IAEA,OAAO;QACLb,MAAKW,IAAY;YACf,IAAIN,OAAO;gBACTC,QAAQC,MAAM,CAACU,KAAK,CAAC,CAAC,EAAE,EAAEP,KAAKC,OAAO;gBACtCF,QAAQ;gBACR;YACF;YACA,+EAA+E;YAC/E,IAAIE,SAASH,WAAW;YACxB,IAAIA,cAAc,KAAKG,OAAOH,aAAaV,QAAQH,oBAAoB;gBACrEW,QAAQC,MAAM,CAACU,KAAK,CAAC,GAAGP,KAAKC,MAAM,EAAE,CAAC;gBACtCH,YAAYG;gBACZF,QAAQ;YACV;QACF;QACAR;YACE,IAAI,CAACQ,OAAO;YACZ,IAAIJ,OAAOC,QAAQC,MAAM,CAACU,KAAK,CAAC,CAAC,EAAE,EAAEP,KAAKZ,OAAO,EAAE,CAAC;iBAC/C,IAAIU,YAAYV,OAAOQ,QAAQC,MAAM,CAACU,KAAK,CAAC,GAAGP,KAAKZ,OAAO,EAAE,CAAC;QACrE;IACF;AACF"}

package/dist/esm/scan.d.ts

@@ -1,16 +1,20 @@
 import type { Config } from './config.js';
 import type { Feature } from './features/types.js';
+export declare const RESERVED_COLUMNS: Set<string>;
 export interface FileStat {
     relPath: string;
     absPath: string;
     mtimeMs: number;
     size: number;
+    presets: string[];
+    embed: boolean;
 }
 export declare function listFiles(cfg: Config, baseDir: string): FileStat[];
 export interface ParsedDoc {
     relPath: string;
     mtimeMs: number;
     size: number;
+    presets: string[];
     data: Record<string, string | number | bigint | null>;
     search: {
         title: string;

package/dist/esm/scan.js

@@ -3,9 +3,13 @@ import { join } from 'node:path';
 import fastGlob from 'fast-glob';
 import removeMarkdown from 'remove-markdown';
 import { parseDocument } from 'yaml';
+import { presetNames, presetSemanticEnabled } from './config.js';
 // Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
-// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).
-const RESERVED_COLUMNS = new Set([
+// Reserved: colliding with these would clash with the `frontmatter` table's own columns or
+// the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert
+// can tell a feature-owned column (e.g. rank's `_rank`) apart from a parsed frontmatter
+// column and leave the former alone on reparse.
+export const RESERVED_COLUMNS = new Set([
     'path',
     '_mtime',
     '_size',
@@ -26,18 +30,44 @@ function stripText(value) {
     const withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
     return normalizeText(withoutTables);
 }
+// Presets are the file selection: each preset's include/exclude picks its files (fast-glob,
+// tsconfig-standard semantics, resolved relative to baseDir). Unlike the old layer model,
+// presets are views, not partitions -- they may overlap freely, and a file's set of covering
+// presets (not a single owner) is what indexing and embedding derive from. A preset whose
+// globs match nothing is valid. Files matched by no preset at all are simply not indexed.
 export function listFiles(cfg, baseDir) {
-    const relPaths = fastGlob.sync(cfg.scan.include, {
-        cwd: baseDir
-    }).sort();
+    const coverage = new Map();
+    for (const name of presetNames(cfg)){
+        var _preset_exclude;
+        const preset = cfg.presets[name];
+        const matched = fastGlob.sync(preset.include, {
+            cwd: baseDir,
+            ignore: (_preset_exclude = preset.exclude) !== null && _preset_exclude !== void 0 ? _preset_exclude : []
+        });
+        for (const relPath of matched){
+            var _coverage_get;
+            const set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
+            set.add(name);
+            coverage.set(relPath, set);
+        }
+    }
+    const relPaths = [
+        ...coverage.keys()
+    ].sort();
     return relPaths.map((relPath)=>{
         const absPath = join(baseDir, relPath);
         const st = statSync(absPath);
+        const presets = [
+            ...coverage.get(relPath)
+        ].sort();
+        const embed = presets.some((name)=>presetSemanticEnabled(cfg, name));
         return {
             relPath,
             absPath,
             mtimeMs: st.mtimeMs,
-            size: st.size
+            size: st.size,
+            presets,
+            embed
         };
     });
 }
@@ -117,6 +147,7 @@ export function parseFile(file, extractors = []) {
             relPath: file.relPath,
             mtimeMs: file.mtimeMs,
             size: file.size,
+            presets: file.presets,
             data: mapped,
             search,
             extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","keys","has","search","title","summary","text","extracted","fromEntries","filter","f","extract","name"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAIrC,6EAA6E;AAE7E,8IAA8I;AAC9I,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAErG,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBZ,eAAeW;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AASA,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWlB,SAASmB,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAU3B,KAAKkB,SAASQ;QAC9B,MAAME,KAAK7B,SAAS4B;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAaA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASC,SAASxB,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO0B,OAAOC,aAAa,CAAC3B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAO4B,KAAKC,SAAS,CAAC7B;AACxB;AAEA,gEAAgE;AAChE,SAAS8B,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKvC,WAAW,OAAO;QAAEiC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBtB,OAAe,EAAEe,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM/C,cAAcsC;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,iBAAiB,EAAEwB,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,8BAA8B,EAAE,AAAC+B,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS/C,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO+C,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAO6B;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMxC,aAAa+D,KAAKlC,OAAO,EAAE;IACvC,MAAMsB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAKnC,OAAO,EAAEe,IAAIQ;IACnE,MAAMe,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACZ,MAAO;QACnC,IAAInD,iBAAiBgE,GAAG,CAACH,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAKnC,OAAO,CAAC,8BAA8B,EAAEuC,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGlC,SAASwB,IAAI,CAACU,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMI,SAAS;QAAEC,OAAOhE,cAAciD,KAAKe,KAAK;QAAGC,SAASjE,cAAciD,KAAKgB,OAAO;QAAGC,MAAM5D,UAAUmD;IAAS;IAElH,OAAO;QACLb,KAAK;YACHxB,SAASmC,KAAKnC,OAAO;YACrBG,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACfyB,MAAMS;YACNK;YACAI,WAAWP,OAAOQ,WAAW,CAACZ,WAAWa,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEpD,GAAG,CAAC,CAACmD;oBAAeA;uBAAT;oBAACA,EAAEE,IAAI;qBAAEF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYtC,KAAKyB,SAASM;iBAAQ;;QAC1H;QACApB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport { presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or\n// the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert\n// can tell a feature-owned column (e.g. rank's `_rank`) apart from a parsed frontmatter\n// column and leave the former alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff any covering preset has semantic !== false\n}\n\n// Presets are the file selection: each preset's include/exclude picks its files (fast-glob,\n// tsconfig-standard semantics, resolved relative to baseDir). Unlike the old layer model,\n// presets are views, not partitions -- they may overlap freely, and a file's set of covering\n// presets (not a single owner) is what indexing and embedding derive from. A preset whose\n// globs match nothing is valid. Files matched by no preset at all are simply not indexed.\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    const matched = fastGlob.sync(preset.include, { cwd: baseDir, ignore: preset.exclude ?? [] });\n    for (const relPath of matched) {\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  const relPaths = [...coverage.keys()].sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = presets.some((name) => presetSemanticEnabled(cfg, name));\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","coverage","Map","name","preset","presets","matched","sync","include","cwd","ignore","exclude","relPath","set","get","add","relPaths","keys","sort","map","absPath","st","embed","some","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","has","search","title","summary","text","extracted","fromEntries","filter","f","extract"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAErC,SAASC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAGjE,6EAA6E;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,wFAAwF;AACxF,gDAAgD;AAChD,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW,EAAE;AAE9G,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBd,eAAea;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAWA,4FAA4F;AAC5F,0FAA0F;AAC1F,6FAA6F;AAC7F,0FAA0F;AAC1F,0FAA0F;AAC1F,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,KAAK,MAAMC,QAAQnB,YAAYe,KAAM;YAEmCK;QADtE,MAAMA,SAASL,IAAIM,OAAO,CAACF,KAAK;QAChC,MAAMG,UAAUzB,SAAS0B,IAAI,CAACH,OAAOI,OAAO,EAAE;YAAEC,KAAKT;YAASU,MAAM,GAAEN,kBAAAA,OAAOO,OAAO,cAAdP,6BAAAA,kBAAkB,EAAE;QAAC;QAC3F,KAAK,MAAMQ,WAAWN,QAAS;gBACjBL;YAAZ,MAAMY,OAAMZ,gBAAAA,SAASa,GAAG,CAACF,sBAAbX,2BAAAA,gBAAyB,IAAId;YACzC0B,IAAIE,GAAG,CAACZ;YACRF,SAASY,GAAG,CAACD,SAASC;QACxB;IACF;IAEA,MAAMG,WAAW;WAAIf,SAASgB,IAAI;KAAG,CAACC,IAAI;IAC1C,OAAOF,SAASG,GAAG,CAAC,CAACP;QACnB,MAAMQ,UAAUxC,KAAKoB,SAASY;QAC9B,MAAMS,KAAK1C,SAASyC;QACpB,MAAMf,UAAU;eAAKJ,SAASa,GAAG,CAACF;SAAyB,CAACM,IAAI;QAChE,MAAMI,QAAQjB,QAAQkB,IAAI,CAAC,CAACpB,OAASlB,sBAAsBc,KAAKI;QAChE,OAAO;YAAES;YAASQ;YAASI,SAASH,GAAGG,OAAO;YAAEC,MAAMJ,GAAGI,IAAI;YAAEpB;YAASiB;QAAM;IAChF;AACF;AAcA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASI,SAASrC,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOsC,OAAOtC,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOuC,OAAOC,aAAa,CAACxC,SAASsC,OAAOtC,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOyC,KAAKC,SAAS,CAAC1C;AACxB;AAEA,gEAAgE;AAChE,SAAS2C,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKpD,WAAW,OAAO;QAAE8C,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiB/B,OAAe,EAAEwB,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM9D,cAAcqD;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,iBAAiB,EAAEiC,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,8BAA8B,EAAE,AAACwC,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS5D,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO4D,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAOsC;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMvD,aAAa8E,KAAKpC,OAAO,EAAE;IACvC,MAAMwB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAK5C,OAAO,EAAEwB,IAAIQ;IACnE,MAAMe,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAO5C,IAAI,CAACiC,MAAO;QACnC,IAAIhE,iBAAiB4E,GAAG,CAACF,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAK5C,OAAO,CAAC,8BAA8B,EAAEgD,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGlC,SAASwB,IAAI,CAACU,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMG,SAAS;QAAEC,OAAO5E,cAAc8D,KAAKc,KAAK;QAAGC,SAAS7E,cAAc8D,KAAKe,OAAO;QAAGC,MAAMxE,UAAUgE;IAAS;IAElH,OAAO;QACLb,KAAK;YACHjC,SAAS4C,KAAK5C,OAAO;YACrBY,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACfpB,SAASmD,KAAKnD,OAAO;YACrB6C,MAAMS;YACNI;YACAI,WAAWN,OAAOO,WAAW,CAACX,WAAWY,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEpD,GAAG,CAAC,CAACmD;oBAAeA;uBAAT;oBAACA,EAAEnE,IAAI;qBAAEmE,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYrC,KAAKyB,SAASK;iBAAQ;;QAC1H;QACAnB;IACF;AACF"}

package/package.json

@@ -1,15 +1,25 @@
 {
   "name": "sensemaking",
-  "version": "0.7.2",
-  "description": "Query a knowledge base you build with an agent: filter notes by frontmatter, then search inside them",
+  "version": "0.9.0",
+  "description": "Query and search a tree of markdown notes: SQL over frontmatter and links, ranked search over the prose — words, links, and meaning fused",
   "keywords": [
     "markdown",
     "frontmatter",
     "sql",
     "sqlite",
+    "fts5",
     "query",
+    "search",
+    "semantic-search",
+    "knowledge-base",
+    "rag",
     "obsidian",
     "notes",
+    "agent",
+    "agent-memory",
+    "memory-consolidation",
+    "llm-wiki",
+    "dreaming",
     "cli"
   ],
   "homepage": "https://github.com/kmalakoff/sensemaking",

package/schema.json

@@ -4,7 +4,7 @@
   "description": "Config for sense -- SQL over the frontmatter, content, links, and structure of a markdown tree.",
   "type": "object",
   "additionalProperties": false,
-  "required": ["scan", "queries"],
+  "required": ["presets", "queries"],
   "properties": {
     "$schema": {
       "type": "string",
@@ -12,31 +12,54 @@
     },
     "version": {
       "type": "integer",
-      "enum": [1, 2],
+      "enum": [1, 2, 3],
       "description": "Config format version. Older versions are auto-migrated (and the file rewritten) on load; a version newer than this sense build supports makes it exit with an error rather than misinterpret the file. Omit to default to 1."
     },
-    "scan": {
+    "presets": {
       "type": "object",
-      "additionalProperties": false,
-      "required": ["include"],
-      "description": "Which files become rows in the `frontmatter` table.",
-      "properties": {
-        "include": {
-          "type": "array",
-          "items": { "type": "string" },
-          "minItems": 1,
-          "description": "Glob patterns (globby syntax), resolved relative to this config file's own directory -- never the invocation cwd."
+      "description": "File selection, index-time, and named search defaults. A preset is self-contained: no inheritance between presets, and no ordering -- presets may overlap freely (they are views, not partitions), and a preset whose globs match nothing is valid. A file is indexed iff any preset's include/exclude covers it (union across presets); it is embedded iff any covering preset has semantic on. `default` is used when a command names no preset, and must be declared. `sense init` generates `default` plus one `large` example.",
+      "minProperties": 1,
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["include"],
+        "properties": {
+          "include": {
+            "type": "array",
+            "items": { "type": "string" },
+            "minItems": 1,
+            "description": "Glob patterns (globby syntax), resolved relative to this config file's own directory -- never the invocation cwd. Required for every preset, including `default`."
+          },
+          "exclude": {
+            "type": "array",
+            "items": { "type": "string" },
+            "minItems": 1,
+            "description": "Glob patterns excluded from this preset's `include`, tsconfig-standard semantics."
+          },
+          "k": {
+            "type": "integer",
+            "minimum": 1,
+            "description": "Default result count for `search` when this preset is in effect (named, or `default` when none is). A --k on the invocation, or a saved search's own `k`, overrides it. Defaults to 10 when the preset omits it too."
+          },
+          "semantic": {
+            "type": "boolean",
+            "description": "Vector participation for files this preset covers. Absent or true means on (the default); only ever meaningfully written as false. A file covered only by semantic-false presets is indexed but never embedded, so it searches lexically automatically -- no error, just fewer signals."
+          },
+          "where": {
+            "type": "string",
+            "description": "Standing SQL condition against frontmatter alias `f`, applied when `search` runs under this preset without an explicit --where. An explicit --where (or a saved search's own `where`) replaces it rather than ANDing with it, so a caller can always widen back to the preset's whole scope."
+          }
         }
       }
     },
     "features": {
       "type": "object",
       "additionalProperties": false,
-      "description": "Independently toggleable features. Absent block or key means enabled; set false to disable. Toggling a feature rebuilds the cache. Verbs degrade rather than fail: `find` without links is BM25-only, `map` without rank omits hubs, `peek` without sections omits the outline.",
+      "description": "Global feature defaults. Absent block or key means enabled. `embed` is not a member of this block -- vector participation is controlled per preset by `semantic` (see \"presets\" and the top-level \"embed\" block for provider settings). Toggling a feature rebuilds the cache. Commands degrade rather than fail: `search` without links is BM25-only, `map` without rank omits hubs, `peek` without sections omits the outline.",
       "properties": {
         "links": {
           "type": "boolean",
-          "description": "Extract wikilinks and relative markdown links into the `links` table (src, target as written, dst resolved path or NULL for dead links; an ambiguous basename resolves to the lexicographically first match). Powers backlinks, graph expansion in `find`, and `peek`'s link lists."
+          "description": "Extract wikilinks and relative markdown links into the `links` table (src, target as written, dst resolved path or NULL for dead links; an ambiguous basename resolves to the lexicographically first match). Powers backlinks, graph expansion in `search`, and `peek`'s link lists."
         },
         "sections": {
           "type": "boolean",
@@ -45,52 +68,64 @@
         "rank": {
           "type": "boolean",
           "description": "PageRank over resolved links into `frontmatter._rank` at reconcile -- a static importance prior. Powers `map`'s hub list; usable in any ORDER BY. Requires `links`."
-        },
-        "embed": {
-          "description": "Semantic vectors, opt-in (absent = off; most trees don't need them). When on, vectors are computed and kept fresh at reconcile; expansion runs only on `find --semantic` invocations -- default `find` results are unchanged. `true` = defaults (static type, minishlab/potion-retrieval-32M, downloaded to ~/.cache/sensemaking on first use). Object form: `model` (Hugging Face id or local path), `type` (`static` = built-in pure-JS Model2Vec loader; `api` = OpenAI-compatible POST /embeddings), `url` (api base, e.g. http://localhost:11434/v1), `key` (name of the env var holding the bearer token). Changing model or type rebuilds the cache like a feature toggle.",
-          "oneOf": [
-            { "type": "boolean" },
-            {
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "model": { "type": "string" },
-                "type": { "type": "string", "enum": ["static", "api"] },
-                "url": { "type": "string" },
-                "key": { "type": "string" }
-              }
-            }
-          ]
         }
       }
     },
-    "defaults": {
+    "embed": {
       "type": "object",
       "additionalProperties": false,
-      "description": "Tree-declared defaults for the verbs.",
+      "description": "Embed provider settings only -- presence alone does not enable embedding. Vector participation is decided per preset by that preset's `semantic` field; this block only says which model/provider to use when at least one preset wants vectors. `model` (Hugging Face id or local path, default minishlab/potion-retrieval-32M), `type` (`static` = built-in pure-JS Model2Vec loader, default; `api` = OpenAI-compatible POST /embeddings), `url` (api base, e.g. http://localhost:11434/v1), `key` (name of the env var holding the bearer token). Changing model or type rebuilds the cache like a feature toggle.",
       "properties": {
-        "find": {
-          "type": "object",
-          "additionalProperties": false,
-          "description": "Default scope for `find`. Use when a tree holds a layer that should stay queryable but out of ordinary search -- generated output, or unverified source extractions that must not be cited as conclusions.",
-          "properties": {
-            "where": {
-              "type": "string",
-              "description": "SQL condition against frontmatter alias `f`, e.g. \"f.type != 'raw'\". Applied when `find` runs without --where; an explicit --where replaces it (so `--where \"1=1\"` searches the whole tree). Reported by `sense status`."
-            }
-          }
-        }
+        "model": { "type": "string" },
+        "type": { "type": "string", "enum": ["static", "api"] },
+        "url": { "type": "string" },
+        "key": { "type": "string" }
       }
     },
-    "checks": {
-      "type": "object",
-      "description": "Assertions over saved queries, for trees whose queries encode invariants: `\"checks\": { \"dead-links\": \"empty\" }` makes `sense check` fail (exit 1) when that query returns rows. Without an entry, `check` reports zero-row queries as `ok, but returns 0 rows` -- informational, since an ordinary query returning nothing is sometimes a bug and sometimes just an empty result. Keys must name saved queries.",
-      "additionalProperties": { "type": "string", "enum": ["empty"] }
-    },
     "queries": {
       "type": "object",
-      "description": "Named SQL queries runnable as `sense <name> [params...]`. Tables: `frontmatter` (one row per file, one column per discovered frontmatter key, plus `path`/`_mtime`/`_size`/`_rank`), `content` (FTS5: `title`, `summary`, `text`, `path`), `links` (`src`, `target`, `dst`), and `sections` (`path`, `idx`, `level`, `heading`, `start_line`, `end_line`, `tokens`). `?` placeholders bind to CLI positional args in order. `has(field, value)`: array membership on a JSON-array field, substring match on a string (so has(f.status, 'active') also matches 'inactive'), false on NULL. Exact matches: `=` for scalars, `EXISTS (SELECT 1 FROM json_each(f.tags) WHERE value = ?)` for array members. Canonical query: `SELECT f.path, content.title, content.summary, snippet(content, -1, '«', '»', '…', 10) AS hit FROM frontmatter f JOIN content ON content.path = f.path WHERE content MATCH ? ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10`. Reserved frontmatter keys: `path`, `_mtime`, `_size`, `_rank`, `content`, `links`, `sections`. Reserved query names (unreachable as subcommands): `init`, `query`, `find`, `map`, `peek`, `watch`, `status`, `rebuild`, `check`.",
-      "additionalProperties": { "type": "string" }
+      "description": "Named queries runnable as `sense <name> [params...]`, one of three shapes: a raw SQL string, `{ sql }`, or a saved search. SQL (string or `{ sql }`): `?` placeholders bind to CLI positional args in order; deterministic and enumerating, including raw FTS5 `MATCH` for word search under your own SQL -- \"0 rows = not in the tree\" lives here. Tables: `frontmatter` (one row per file, one column per discovered frontmatter key, plus `path`/`_mtime`/`_size`/`_rank`), `content` (FTS5: `title`, `summary`, `text`, `path`), `links` (`src`, `target`, `dst`), `sections` (`path`, `idx`, `level`, `heading`, `start_line`, `end_line`, `tokens`), and `preset_files` (`preset`, `path`) -- which presets cover which files. `has(field, value)`: array membership on a JSON-array field, substring match on a string (so has(f.status, 'active') also matches 'inactive'), false on NULL. Exact matches: `=` for scalars, `EXISTS (SELECT 1 FROM json_each(f.tags) WHERE value = ?)` for array members. Canonical query: `SELECT f.path, content.title, content.summary, CASE WHEN length(content.text) <= 16384 THEN snippet(content, -1, '«', '»', '…', 10) END AS hit FROM frontmatter f JOIN content ON content.path = f.path WHERE content MATCH ? ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10`. The CASE bounds snippet(), which re-tokenizes each matched document and costs seconds per query once a tree holds a megabyte-scale note; `search` applies the same bound internally. Saved search object: one text driving every engine the scoped preset has -- word match, links, vectors -- fused into one ranked list, `via` labeling which engine produced each row; `search` text must be non-empty (a saved query saves a question -- a scope without one is just flags). `preset` names one declared preset (defaults to `default`); `include` is an ad hoc glob scope that replaces the preset's include/exclude entirely, same as `search --include`; `where` and `k` behave like the `search` command's flags. `semantic: false` opts a query down to lexical only -- the rare precision opt-out, not a gate (semantic participation otherwise follows the scoped preset's `semantic` automatically). `sense <name>` behaves like `sense search <search> [--preset] [--include] [--where] [--k] [--lexical]` with zero flags; it takes no positional parameters. `sense check` probes every saved query/search (lexically, k=1 for searches) so a broken one fails loudly at check time -- it makes no assertion about the result itself; a returned row set is the reader's judgment. Reserved frontmatter keys: `path`, `_mtime`, `_size`, `_rank`, `content`, `links`, `sections`. Reserved query names (unreachable as subcommands, any shape): `init`, `query`, `search`, `find`, `map`, `peek`, `watch`, `status`, `rebuild`, `check`.",
+      "additionalProperties": {
+        "oneOf": [
+          { "type": "string" },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["sql"],
+            "properties": {
+              "sql": { "type": "string", "description": "Raw SQL, same as the string shape -- `?` placeholders bind to CLI positional args in order." }
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["search"],
+            "properties": {
+              "search": {
+                "type": "string",
+                "minLength": 1,
+                "description": "One text driving every engine the scoped preset has -- word match, links, vectors -- fused into one ranked list. Same syntax as `sense search \"<text>\"`. Must be non-empty."
+              },
+              "preset": {
+                "type": "string",
+                "description": "Preset name to search under. Defaults to `default`. An unknown name is an error listing declared presets."
+              },
+              "include": {
+                "type": "array",
+                "items": { "type": "string" },
+                "minItems": 1,
+                "description": "Ad hoc glob scope, same as `search --include`. Replaces the preset's include/exclude entirely rather than layering on top of it."
+              },
+              "k": { "type": "integer", "minimum": 1, "description": "Result count, same as `search --k`. Defaults to the preset's own `k`, then 10. A --k on the invocation overrides this." },
+              "where": { "type": "string", "description": "SQL condition against frontmatter alias `f`, same as `search --where`. A --where on the invocation replaces this rather than ANDing with it." },
+              "semantic": {
+                "type": "boolean",
+                "description": "false opts this query down to lexical-only, same as `search --lexical` -- the rare precision opt-out. Semantic participation otherwise follows the scoped preset's `semantic` automatically, so true is redundant with the default."
+              }
+            }
+          }
+        ]
+      }
     }
   }
 }

package/skills/sense/EXAMPLES.md

@@ -6,20 +6,21 @@ the filesystem, on the paths that earned it.
 ## A. "Do the notes say anything about X?"
 
 ```
-sense find "pricing OR billing OR invoicing" --k 10 --format json
+sense search "pricing OR billing OR invoicing" --k 10 --format json
 ```
 
 ```json
 [
   { "path": "notes/pricing-model.md", "title": "Pricing model",
-    "summary": "Tiered per-seat pricing; floor and discount rules", "hit": "«Pricing» floor for annual…", "via": "match+link", "score": 0.0333 },
+    "summary": "Tiered per-seat pricing; floor and discount rules", "hit": "«Pricing» floor for annual…", "via": "match+link", "score": 0.0333, "lines": null },
   { "path": "notes/renewal-playbook.md", "title": "Renewal playbook",
-    "summary": "Renewal sequence and owners", "hit": "…«billing» contact confirms the PO…", "via": "match", "score": 0.0313 }
+    "summary": "Renewal sequence and owners", "hit": "…«billing» contact confirms the PO…", "via": "match", "score": 0.0313, "lines": "L81-140" }
 ]
 ```
 
-~30 tokens per row; often the `summary` answers the question with no read at all. A row with
-`via: "link"` never contained the terms — it is linked from notes that did.
+Tens of tokens per row; often the `summary` answers the question with no read at all. A row with
+`via: "link"` never contained the terms — it is linked from notes that did. `lines`, when
+set, is the section that earned the row, a direct `Read` range.
 
 ## B. Known-field filtering (no search)
 
@@ -78,27 +79,30 @@ sense query "SELECT DISTINCT type FROM frontmatter"    # what a field's values a
 ## F. "The notes say it in different words"
 
 ```
-sense find "children dying from poor nutrition" --semantic --k 3 --format json
+sense search "children dying from poor nutrition" --k 3 --format json
 ```
 
 ```json
 [
   { "path": "notes/malnutrition-outcomes.md", "title": "Malnutrition outcomes",
-    "summary": "Stunting and mortality by region", "hit": null, "via": "vector", "score": 0.0167, "lines": "L14-52" }
+    "summary": "Stunting and mortality by region", "hit": null, "via": "vector", "score": 0.0167, "similarity": 0.61, "lines": "L14-52" }
 ]
 ```
 
-A `via: "vector"` row never contained the terms — it is semantically near them; `lines` is the
-best-matching section, a direct `Read` range. Only on trees whose config enables `features.embed`.
+A `via: "vector"` row never contained the terms — it is semantically near them; `similarity` is
+the cosine against the chunk `lines` names, a direct `Read` range. Vector rows appear whenever
+the scope's preset has semantic on (the default); a result of only vector rows means the words
+themselves are nowhere in the scope.
 
 ## Consequences
 
 | query | what happens | bounded alternative |
 |---|---|---|
 | `SELECT text FROM content` | returns the tree's entire prose | `snippet(content, -1, '«', '»', '…', 10)` excerpts the match |
-| `sense find "pricing"` | matches only that word's stem | OR-in synonyms and instances: `"pricing OR billing OR invoicing"` |
-| `sense find "pricing model details"` | bare words AND-join; one absent word = zero rows | OR the words, or quote an exact phrase |
-| `sense find "customer-facing OR on-site"` | bare punctuation is FTS5 syntax (`-` reads as a column filter) | double-quote: `"customer-facing" OR "on-site"` |
+| `snippet()` on a tree holding a megabyte-scale note | re-tokenizes the whole document per matched row: seconds per query | bound it: `CASE WHEN length(content.text) <= 16384 THEN snippet(...) END`, or select `summary` instead |
+| `sense search "pricing"` | lexically matches only that word's stem (vector rows still widen by meaning) | OR-in synonyms and instances: `"pricing OR billing OR invoicing"` |
+| `sense search "pricing model details"` | bare words AND-join; one absent word = zero lexical rows | OR the words, or quote an exact phrase |
+| `sense search "customer-facing OR on-site"` | bare punctuation is FTS5 syntax (`-` reads as a column filter) | double-quote: `"customer-facing" OR "on-site"` |
 | `Read` of a large file for one section | costs the whole file | `peek`, then `Read` the line range |
 | row queries without `LIMIT` | unbounded output (aggregates are already bounded) | `LIMIT n` |
 | saving one-off queries to config | config churn | ad-hoc `sense query`; save reusable views |