sensemaking 0.6.0 → 0.7.1

package/dist/esm/output.js

@@ -6,19 +6,41 @@ function cell(value) {
     return String(value !== null && value !== void 0 ? value : '').replace(/\s*\n\s*/g, ' ');
 }
 export function printRows(rows, format) {
-    const rendered = render(rows, format);
+    const rendered = renderRows(rows, format);
     console.log(rendered);
     if (rendered.length > OVERSIZE_BYTES) {
         console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);
     }
 }
-function render(rows, format) {
+// Table output is for reading; a row wider than the terminal wraps into an unreadable
+// block (a `find` header measured 330 chars once `summary` is present, which drove one
+// user to pipe --format json through a script for every interactive query). Widest
+// columns give up space first, down to a floor, and only as far as the terminal needs.
+// --format json is never truncated: it is the machine-readable surface.
+const MIN_COLUMN = 12;
+const TRUNCATED = '…';
+function fitWidths(widths, limit) {
+    const gaps = (widths.length - 1) * 2;
+    const fitted = [
+        ...widths
+    ];
+    for(;;){
+        const total = fitted.reduce((a, b)=>a + b, 0) + gaps;
+        if (total <= limit) return fitted;
+        const widest = fitted.indexOf(Math.max(...fitted));
+        if (fitted[widest] <= MIN_COLUMN) return fitted;
+        fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));
+    }
+}
+function renderRows(rows, format, width = process.stdout.columns) {
     if (format === 'json') return JSON.stringify(rows, null, 2);
     if (rows.length === 0) return '(0 rows)';
     const columns = Object.keys(rows[0]);
     const cells = rows.map((row)=>columns.map((col)=>cell(row[col])));
-    const widths = columns.map((col, i)=>Math.max(col.length, ...cells.map((row)=>row[i].length)));
-    const formatRow = (values)=>values.map((value, i)=>value.padEnd(widths[i])).join('  ');
+    const natural = columns.map((col, i)=>Math.max(col.length, ...cells.map((row)=>row[i].length)));
+    const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;
+    const clip = (value, w)=>value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`;
+    const formatRow = (values)=>values.map((value, i)=>clip(value, widths[i])).join('  ').trimEnd();
     return [
         formatRow(columns),
         widths.map((w)=>'-'.repeat(w)).join('  '),
@@ -34,12 +56,12 @@ export function renderMap(result) {
     const parts = [
         `docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`,
         `${featuresLine(result.features)}\n`,
-        render(result.fields, 'table')
+        renderRows(result.fields, 'table')
     ];
     if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);
-    if (result.hubs.length > 0) parts.push('\nhubs (by link rank):', render(result.hubs, 'table'));
+    if (result.hubs.length > 0) parts.push('\nhubs (by link rank):', renderRows(result.hubs, 'table'));
     else if (result.features.off.includes('rank')) parts.push('\nhubs: off (features.rank)');
-    parts.push('\nrecent:', render(result.recent, 'table'));
+    parts.push('\nrecent:', renderRows(result.recent, 'table'));
     return parts.join('\n');
 }
 export function renderPeek(result) {

package/dist/esm/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = render(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\nfunction render(rows: Row[], format: 'table' | 'json'): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const widths = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nfunction featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, `${featuresLine(result.features)}\\n`, render(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', render(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push('\\nhubs: off (features.rank)');\n  parts.push('\\nrecent:', render(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","render","console","log","length","warn","Math","round","JSON","stringify","columns","Object","keys","cells","map","row","col","widths","i","max","formatRow","values","padEnd","join","w","repeat","featuresLine","features","off","name","on","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","linkLine","label","shown","total","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,OAAOH,MAAMC;IAC9BG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,SAASH,OAAOH,IAAW,EAAEC,MAAwB;IACnD,IAAIA,WAAW,QAAQ,OAAOS,KAAKC,SAAS,CAACX,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMM,UAAUC,OAAOC,IAAI,CAACd,IAAI,CAAC,EAAE;IACnC,MAAMe,QAAQf,KAAKgB,GAAG,CAAC,CAACC,MAAQL,QAAQI,GAAG,CAAC,CAACE,MAAQvB,KAAKsB,GAAG,CAACC,IAAI;IAClE,MAAMC,SAASP,QAAQI,GAAG,CAAC,CAACE,KAAKE,IAAMZ,KAAKa,GAAG,CAACH,IAAIZ,MAAM,KAAKS,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAACd,MAAM;IAE/F,MAAMgB,YAAY,CAACC,SAAqBA,OAAOP,GAAG,CAAC,CAACpB,OAAOwB,IAAMxB,MAAM4B,MAAM,CAACL,MAAM,CAACC,EAAE,GAAGK,IAAI,CAAC;IAE/F,OAAO;QAACH,UAAUV;QAAUO,OAAOH,GAAG,CAAC,CAACU,IAAM,IAAIC,MAAM,CAACD,IAAID,IAAI,CAAC;WAAUV,MAAMC,GAAG,CAACM;KAAW,CAACG,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,SAASG,aAAaC,QAAyC;IAC7D,MAAMC,MAAMD,SAASC,GAAG,CAACxB,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEuB,SAASC,GAAG,CAACd,GAAG,CAAC,CAACe,OAAS,GAAGA,KAAK,WAAW,EAAEA,KAAK,CAAC,CAAC,EAAEN,IAAI,CAAC,OAAO,GAAG;IACzH,OAAO,CAAC,UAAU,EAAEI,SAASG,EAAE,CAACP,IAAI,CAAC,QAAQK,KAAK;AACpD;AAEA,OAAO,SAASG,UAAUC,MAA6J;IACrL,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAE7B,KAAKC,KAAK,CAACyB,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE,GAAGV,aAAaM,OAAOL,QAAQ,EAAE,EAAE,CAAC;QAAE1B,OAAO+B,OAAOK,MAAM,EAAE;KAAS;IAC/J,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACjC,MAAM,EAAE6B,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACjC,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI4B,OAAOQ,IAAI,CAACpC,MAAM,GAAG,GAAG6B,MAAMM,IAAI,CAAC,0BAA0BtC,OAAO+B,OAAOQ,IAAI,EAAE;SAChF,IAAIR,OAAOL,QAAQ,CAACC,GAAG,CAACa,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC;IAC1DN,MAAMM,IAAI,CAAC,aAAatC,OAAO+B,OAAOU,MAAM,EAAE;IAC9C,OAAOT,MAAMV,IAAI,CAAC;AACpB;AAEA,OAAO,SAASoB,WAAWX,MAAiO;IAC1P,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAKrD,MAAM,IAAIiB,OAAOqC,OAAO,CAAChB,OAAOiB,WAAW,EAAGL,MAAML,IAAI,CAAC,CAAC,EAAE,EAAEQ,IAAI,EAAE,EAAErD,OAAO;IAC9F,IAAIsC,OAAOkB,QAAQ,CAAC9C,MAAM,GAAG,GAAG;QAC9BwC,MAAML,IAAI,CAAC,IAAI;QACf,KAAK,MAAMY,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAML,IAAI,CAAC,CAAC,EAAE,EAAE,IAAId,MAAM,CAAC0B,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;IAChJ,OAAO,IAAId,OAAOJ,GAAG,CAACa,QAAQ,CAAC,aAAaG,MAAML,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOJ,GAAG,CAACa,QAAQ,CAAC,UAAU;QAChCG,MAAML,IAAI,CAAC,IAAI;QACf,OAAOK,MAAMrB,IAAI,CAAC;IACpB;IACA,MAAMiC,WAAW,CAACC,OAAeC,OAAiBC;QAChD,IAAIA,QAAQ,GAAGf,MAAML,IAAI,CAAC,GAAGkB,MAAM,EAAE,EAAEE,MAAM,GAAG,EAAED,MAAMnC,IAAI,CAAC,QAAQoC,QAAQD,MAAMtD,MAAM,GAAG,CAAC,GAAG,EAAEuD,QAAQD,MAAMtD,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAI4B,OAAO4B,aAAa,GAAG5B,OAAO6B,eAAe,GAAG7B,OAAO8B,cAAc,GAAG,GAAGlB,MAAML,IAAI,CAAC;IAC1FiB,SAAS,aAAaxB,OAAO+B,QAAQ,EAAE/B,OAAO4B,aAAa;IAC3DJ,SAAS,cAAcxB,OAAOgC,UAAU,EAAEhC,OAAO6B,eAAe;IAChEL,SAAS,aAAaxB,OAAOiC,SAAS,EAAEjC,OAAO8B,cAAc;IAC7D,OAAOlB,MAAMrB,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\n// Table output is for reading; a row wider than the terminal wraps into an unreadable\n// block (a `find` header measured 330 chars once `summary` is present, which drove one\n// user to pipe --format json through a script for every interactive query). Widest\n// columns give up space first, down to a floor, and only as far as the terminal needs.\n// --format json is never truncated: it is the machine-readable surface.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: 'table' | 'json', width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nfunction featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, `${featuresLine(result.features)}\\n`, renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push('\\nhubs: off (features.rank)');\n  parts.push('\\nrecent:', renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","renderRows","console","log","length","warn","Math","round","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","process","stdout","columns","JSON","stringify","Object","keys","cells","map","row","col","natural","i","clip","w","padEnd","slice","formatRow","values","join","trimEnd","repeat","featuresLine","features","off","name","on","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,WAAWH,MAAMC;IAClCG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,sFAAsF;AACtF,uFAAuF;AACvF,mFAAmF;AACnF,uFAAuF;AACvF,wEAAwE;AACxE,MAAMI,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAOP,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMU,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACd,KAAKe,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASX,WAAWH,IAAW,EAAEC,MAAwB,EAAEuB,QAAQC,QAAQC,MAAM,CAACC,OAAO;IACvF,IAAI1B,WAAW,QAAQ,OAAO2B,KAAKC,SAAS,CAAC7B,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMqB,UAAUG,OAAOC,IAAI,CAAC/B,IAAI,CAAC,EAAE;IACnC,MAAMgC,QAAQhC,KAAKiC,GAAG,CAAC,CAACC,MAAQP,QAAQM,GAAG,CAAC,CAACE,MAAQxC,KAAKuC,GAAG,CAACC,IAAI;IAClE,MAAMC,UAAUT,QAAQM,GAAG,CAAC,CAACE,KAAKE,IAAM7B,KAAKe,GAAG,CAACY,IAAI7B,MAAM,KAAK0B,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAAC/B,MAAM;IAChG,MAAMO,SAASW,SAASA,QAAQd,aAAaE,UAAUwB,SAASZ,SAASY;IAEzE,MAAME,OAAO,CAAC1C,OAAe2C,IAAe3C,MAAMU,MAAM,IAAIiC,IAAI3C,MAAM4C,MAAM,CAACD,KAAK,GAAG3C,MAAM6C,KAAK,CAAC,GAAGF,IAAI,KAAK5B,WAAW;IACxH,MAAM+B,YAAY,CAACC,SACjBA,OACGV,GAAG,CAAC,CAACrC,OAAOyC,IAAMC,KAAK1C,OAAOiB,MAAM,CAACwB,EAAE,GACvCO,IAAI,CAAC,MACLC,OAAO;IAEZ,OAAO;QAACH,UAAUf;QAAUd,OAAOoB,GAAG,CAAC,CAACM,IAAM,IAAIO,MAAM,CAACP,IAAIK,IAAI,CAAC;WAAUZ,MAAMC,GAAG,CAACS;KAAW,CAACE,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,SAASG,aAAaC,QAAyC;IAC7D,MAAMC,MAAMD,SAASC,GAAG,CAAC3C,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE0C,SAASC,GAAG,CAAChB,GAAG,CAAC,CAACiB,OAAS,GAAGA,KAAK,WAAW,EAAEA,KAAK,CAAC,CAAC,EAAEN,IAAI,CAAC,OAAO,GAAG;IACzH,OAAO,CAAC,UAAU,EAAEI,SAASG,EAAE,CAACP,IAAI,CAAC,QAAQK,KAAK;AACpD;AAEA,OAAO,SAASG,UAAUC,MAA6J;IACrL,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEhD,KAAKC,KAAK,CAAC4C,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE,GAAGV,aAAaM,OAAOL,QAAQ,EAAE,EAAE,CAAC;QAAE7C,WAAWkD,OAAOK,MAAM,EAAE;KAAS;IACnK,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACpD,MAAM,EAAEgD,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACpD,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI+C,OAAOQ,IAAI,CAACvD,MAAM,GAAG,GAAGgD,MAAMM,IAAI,CAAC,0BAA0BzD,WAAWkD,OAAOQ,IAAI,EAAE;SACpF,IAAIR,OAAOL,QAAQ,CAACC,GAAG,CAACa,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC;IAC1DN,MAAMM,IAAI,CAAC,aAAazD,WAAWkD,OAAOU,MAAM,EAAE;IAClD,OAAOT,MAAMV,IAAI,CAAC;AACpB;AAEA,OAAO,SAASoB,WAAWX,MAAiO;IAC1P,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAKxE,MAAM,IAAIkC,OAAOuC,OAAO,CAAChB,OAAOiB,WAAW,EAAGL,MAAML,IAAI,CAAC,CAAC,EAAE,EAAEQ,IAAI,EAAE,EAAExE,OAAO;IAC9F,IAAIyD,OAAOkB,QAAQ,CAACjE,MAAM,GAAG,GAAG;QAC9B2D,MAAML,IAAI,CAAC,IAAI;QACf,KAAK,MAAMY,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAML,IAAI,CAAC,CAAC,EAAE,EAAE,IAAId,MAAM,CAAC0B,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;IAChJ,OAAO,IAAId,OAAOJ,GAAG,CAACa,QAAQ,CAAC,aAAaG,MAAML,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOJ,GAAG,CAACa,QAAQ,CAAC,UAAU;QAChCG,MAAML,IAAI,CAAC,IAAI;QACf,OAAOK,MAAMrB,IAAI,CAAC;IACpB;IACA,MAAMiC,WAAW,CAACC,OAAeC,OAAiB9D;QAChD,IAAIA,QAAQ,GAAGgD,MAAML,IAAI,CAAC,GAAGkB,MAAM,EAAE,EAAE7D,MAAM,GAAG,EAAE8D,MAAMnC,IAAI,CAAC,QAAQ3B,QAAQ8D,MAAMzE,MAAM,GAAG,CAAC,GAAG,EAAEW,QAAQ8D,MAAMzE,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAI+C,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGjB,MAAML,IAAI,CAAC;IAC1FiB,SAAS,aAAaxB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;IAC3DH,SAAS,cAAcxB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;IAChEJ,SAAS,aAAaxB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC7D,OAAOjB,MAAMrB,IAAI,CAAC;AACpB"}

package/dist/esm/scan.d.ts

@@ -11,7 +11,7 @@ export interface ParsedDoc {
     relPath: string;
     mtimeMs: number;
     size: number;
-    data: Record<string, string | number | null>;
+    data: Record<string, string | number | bigint | null>;
     search: {
         title: string;
         summary: string;

package/dist/esm/scan.js

@@ -41,10 +41,16 @@ export function listFiles(cfg, baseDir) {
         };
     });
 }
+// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole
+// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.
+// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`
+// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so
+// that mismatch is visible rather than a silent zero-row answer.
 function mapValue(value) {
     if (value === null || value === undefined) return null;
-    if (typeof value === 'boolean') return value ? 1 : 0;
-    if (typeof value === 'string' || typeof value === 'number') return value;
+    if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
+    if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;
+    if (typeof value === 'string') return value;
     return JSON.stringify(value);
 }
 // The delimiter split is all this package used gray-matter for.

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nfunction mapValue(value: unknown): string | number | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return value ? 1 : 0;\n  if (typeof value === 'string' || typeof value === 'number') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","keys","has","search","title","summary","text","extracted","fromEntries","filter","f","extract","name"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAIrC,6EAA6E;AAE7E,8IAA8I;AAC9I,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAErG,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBZ,eAAeW;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AASA,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWlB,SAASmB,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAU3B,KAAKkB,SAASQ;QAC9B,MAAME,KAAK7B,SAAS4B;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAaA,SAASC,SAASxB,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOA,QAAQ,IAAI;IACnD,IAAI,OAAOA,UAAU,YAAY,OAAOA,UAAU,UAAU,OAAOA;IACnE,OAAOyB,KAAKC,SAAS,CAAC1B;AACxB;AAEA,gEAAgE;AAChE,SAAS2B,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKpC,WAAW,OAAO;QAAE8B,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBnB,OAAe,EAAEY,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM5C,cAAcmC;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEvB,QAAQ,iBAAiB,EAAEqB,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEvB,QAAQ,8BAA8B,EAAE,AAAC4B,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS5C,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO4C,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEvB,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAO0B;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMrC,aAAa4D,KAAK/B,OAAO,EAAE;IACvC,MAAMmB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAKhC,OAAO,EAAEY,IAAIQ;IACnE,MAAMe,SAAiD,CAAC;IAExD,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACZ,MAAO;QACnC,IAAIhD,iBAAiB6D,GAAG,CAACH,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAKhC,OAAO,CAAC,8BAA8B,EAAEoC,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAG/B,SAASqB,IAAI,CAACU,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMI,SAAS;QAAEC,OAAO7D,cAAc8C,KAAKe,KAAK;QAAGC,SAAS9D,cAAc8C,KAAKgB,OAAO;QAAGC,MAAMzD,UAAUgD;IAAS;IAElH,OAAO;QACLb,KAAK;YACHrB,SAASgC,KAAKhC,OAAO;YACrBG,SAAS6B,KAAK7B,OAAO;YACrBC,MAAM4B,KAAK5B,IAAI;YACfsB,MAAMS;YACNK;YACAI,WAAWP,OAAOQ,WAAW,CAACZ,WAAWa,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEjD,GAAG,CAAC,CAACgD;oBAAeA;uBAAT;oBAACA,EAAEE,IAAI;qBAAEF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYtC,KAAKyB,SAASM;iBAAQ;;QAC1H;QACApB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","keys","has","search","title","summary","text","extracted","fromEntries","filter","f","extract","name"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAIrC,6EAA6E;AAE7E,8IAA8I;AAC9I,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAErG,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBZ,eAAeW;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AASA,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWlB,SAASmB,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAU3B,KAAKkB,SAASQ;QAC9B,MAAME,KAAK7B,SAAS4B;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAaA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASC,SAASxB,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO0B,OAAOC,aAAa,CAAC3B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAO4B,KAAKC,SAAS,CAAC7B;AACxB;AAEA,gEAAgE;AAChE,SAAS8B,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKvC,WAAW,OAAO;QAAEiC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBtB,OAAe,EAAEe,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM/C,cAAcsC;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,iBAAiB,EAAEwB,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,8BAA8B,EAAE,AAAC+B,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS/C,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO+C,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAO6B;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMxC,aAAa+D,KAAKlC,OAAO,EAAE;IACvC,MAAMsB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAKnC,OAAO,EAAEe,IAAIQ;IACnE,MAAMe,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACZ,MAAO;QACnC,IAAInD,iBAAiBgE,GAAG,CAACH,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAKnC,OAAO,CAAC,8BAA8B,EAAEuC,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGlC,SAASwB,IAAI,CAACU,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMI,SAAS;QAAEC,OAAOhE,cAAciD,KAAKe,KAAK;QAAGC,SAASjE,cAAciD,KAAKgB,OAAO;QAAGC,MAAM5D,UAAUmD;IAAS;IAElH,OAAO;QACLb,KAAK;YACHxB,SAASmC,KAAKnC,OAAO;YACrBG,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACfyB,MAAMS;YACNK;YACAI,WAAWP,OAAOQ,WAAW,CAACZ,WAAWa,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEpD,GAAG,CAAC,CAACmD;oBAAeA;uBAAT;oBAACA,EAAEE,IAAI;qBAAEF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYtC,KAAKyB,SAASM;iBAAQ;;QAC1H;QACApB;IACF;AACF"}

package/dist/esm/search-error.d.ts

@@ -0,0 +1 @@
+export declare function searchError(err: Error, terms: string, scope?: string): Error;

package/dist/esm/search-error.js

@@ -0,0 +1,25 @@
+import { SenseError } from './errors.js';
+// FTS5 reads these as operators, so a bare term containing one is a syntax error or --
+// worse -- a column filter: `end-to-end` parses as a filter on column `to`, and SQLite
+// reports `no such column: to`, which is a true statement about the parse and a false
+// one about the input. Both field reports on 0.6.0 misdiagnosed that message.
+const FTS5_OPERATORS = /[-'"/.:^*()]/;
+export function searchError(err, terms, scope) {
+    var _terms_match;
+    var _exec;
+    const message = err.message;
+    if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;
+    const suspects = ((_terms_match = terms.match(/\S+/g)) !== null && _terms_match !== void 0 ? _terms_match : []).filter((t)=>!t.startsWith('"') && FTS5_OPERATORS.test(t));
+    // Blame the terms only when the failing token actually came from one -- a typo'd column
+    // in --where (or the tree's default scope) raises "no such column" through this same
+    // statement, and naming a term for it would state a false fact about the input.
+    const col = (_exec = /no such column: (\S+)/.exec(message)) === null || _exec === void 0 ? void 0 : _exec[1];
+    const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t)=>t.split(/[^\p{L}\p{N}]+/u).includes(col));
+    if (fromTerms && suspects.length > 0) {
+        return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t)=>`\`${t}\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '"${suspects[0]}"'. Searchable columns are title, summary, text.`);
+    }
+    if (col !== undefined && scope !== undefined) {
+        return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense query "SELECT name FROM pragma_table_info('frontmatter')".`);
+    }
+    return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense query (list them with pragma_table_info('frontmatter')).`);
+}

package/dist/esm/search-error.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/search-error.ts"],"sourcesContent":["import { SenseError } from './errors.ts';\n\n// FTS5 reads these as operators, so a bare term containing one is a syntax error or --\n// worse -- a column filter: `end-to-end` parses as a filter on column `to`, and SQLite\n// reports `no such column: to`, which is a true statement about the parse and a false\n// one about the input. Both field reports on 0.6.0 misdiagnosed that message.\nconst FTS5_OPERATORS = /[-'\"/.:^*()]/;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_OPERATORS.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where (or the tree's default scope) raises \"no such column\" through this same\n  // statement, and naming a term for it would state a false fact about the input.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense query \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense query (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","FTS5_OPERATORS","searchError","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","map","join"],"mappings":"AAAA,SAASA,UAAU,QAAQ,cAAc;AAEzC,uFAAuF;AACvF,uFAAuF;AACvF,sFAAsF;AACtF,8EAA8E;AAC9E,MAAMC,iBAAiB;AAEvB,OAAO,SAASC,YAAYC,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAIN;IANZ,MAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,MAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQX,eAAeM,IAAI,CAACI;IACrG,wFAAwF;IACxF,qFAAqF;IACrF,gFAAgF;IAChF,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,CAACP,IAAMA,EAAEQ,KAAK,CAAC,mBAAmBC,QAAQ,CAACP;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIjB,WAAW,iBAAiB,GAAGM,QAAQ,uBAAuB,EAAEE,SAASa,GAAG,CAAC,CAACV,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEW,IAAI,CAAC,MAAM,gFAAgF,EAAEd,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIhB,WAAW,iBAAiB,GAAGM,QAAQ,yBAAyB,EAAED,MAAM,mHAAmH,CAAC;IACzM;IACA,OAAO,IAAIL,WAAW,iBAAiB,GAAGM,QAAQ,8JAA8J,CAAC;AACnN"}

package/dist/esm/verbs.js

@@ -4,6 +4,7 @@ import { SenseError } from './errors.js';
 import { semanticCandidates } from './features/embed.js';
 import { linkEdges } from './features/index.js';
 import { personalizedRank } from './graph.js';
+import { searchError } from './search-error.js';
 // The three layer verbs: mapTree (orient), find (locate), peek (structure).
 // Each returns data; cli.ts renders. All of them degrade when a feature is off.
 const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0)';
@@ -13,17 +14,26 @@ const RRF_K = 60;
 // Semantic expansion is per-query opt-in: without opts.semantic the result is
 // byte-for-byte independent of the embed feature.
 export async function find(db, cfg, terms, opts = {}) {
-    var _opts_k, _hits_get, _chunkLines_get;
+    var _opts_k, _opts_where, _hits_get, _chunkLines_get, _chunkSimilarity_get;
+    var _cfg_defaults_find, _cfg_defaults;
     const k = (_opts_k = opts.k) !== null && _opts_k !== void 0 ? _opts_k : 10;
     const fetch = Math.max(k * 3, 30);
     // Terms pass verbatim to FTS5 MATCH: bare words AND-join, operators are the caller's.
     // Invalid syntax propagates as an error, zero matches return zero -- no silent rewrites.
     // --where applies inside the candidate query (a post-filter over the top-N would drop
     // matches ranked past the pool) and again on the final select for link-derived rows.
-    const whereJoin = opts.where ? `JOIN frontmatter f ON f."path" = content.path` : '';
-    const whereCond = opts.where ? `AND (${opts.where})` : '';
+    // An explicit --where replaces the tree's declared default rather than ANDing with it,
+    // so a caller can always widen back to the whole tree.
+    const scope = (_opts_where = opts.where) !== null && _opts_where !== void 0 ? _opts_where : (_cfg_defaults = cfg.defaults) === null || _cfg_defaults === void 0 ? void 0 : (_cfg_defaults_find = _cfg_defaults.find) === null || _cfg_defaults_find === void 0 ? void 0 : _cfg_defaults_find.where;
+    const whereJoin = scope ? `JOIN frontmatter f ON f."path" = content.path` : '';
+    const whereCond = scope ? `AND (${scope})` : '';
     const matchSql = `SELECT content.path AS path, snippet(content, -1, '«', '»', '…', 10) AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;
-    const matchRows = db.prepare(matchSql).all(terms);
+    let matchRows;
+    try {
+        matchRows = db.prepare(matchSql).all(terms);
+    } catch (err) {
+        throw searchError(err, terms, scope);
+    }
     const hits = new Map(matchRows.map((r)=>[
             r.path,
             r.hit
@@ -66,10 +76,12 @@ export async function find(db, cfg, terms, opts = {}) {
     // Vector expansion, invoked only: a third RRF list at the swept flat-region constants
     // (weight 1, pool = fetch). Each row carries its best chunk's line range.
     const chunkLines = new Map();
+    const chunkSimilarity = new Map();
     if (opts.semantic) {
         const vec = await semanticCandidates(db, cfg, terms, fetch);
-        vec.forEach(({ path, lines }, i)=>{
+        vec.forEach(({ path, lines, similarity }, i)=>{
             chunkLines.set(path, lines);
+            chunkSimilarity.set(path, similarity);
             const existing = candidates.get(path);
             if (existing) {
                 existing.score += 1 / (RRF_K + i);
@@ -83,11 +95,11 @@ export async function find(db, cfg, terms, opts = {}) {
         });
     }
     db.exec('DROP TABLE IF EXISTS _find');
-    db.exec('CREATE TEMP TABLE _find ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT)');
-    const insert = db.prepare('INSERT INTO _find ("path", score, via, hit, lines) VALUES (?, ?, ?, ?, ?)');
-    for (const [path, c] of candidates)insert.run(path, c.score, c.via, (_hits_get = hits.get(path)) !== null && _hits_get !== void 0 ? _hits_get : null, (_chunkLines_get = chunkLines.get(path)) !== null && _chunkLines_get !== void 0 ? _chunkLines_get : null);
-    const where = opts.where ? `WHERE ${opts.where}` : '';
-    const linesCol = opts.semantic ? ', _find.lines' : '';
+    db.exec('CREATE TEMP TABLE _find ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');
+    const insert = db.prepare('INSERT INTO _find ("path", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');
+    for (const [path, c] of candidates)insert.run(path, c.score, c.via, (_hits_get = hits.get(path)) !== null && _hits_get !== void 0 ? _hits_get : null, (_chunkLines_get = chunkLines.get(path)) !== null && _chunkLines_get !== void 0 ? _chunkLines_get : null, (_chunkSimilarity_get = chunkSimilarity.get(path)) !== null && _chunkSimilarity_get !== void 0 ? _chunkSimilarity_get : null);
+    const where = scope ? `WHERE ${scope}` : '';
+    const linesCol = opts.semantic ? ', _find.lines, _find.similarity' : '';
     return db.prepare(`SELECT f."path" AS path, content.title, content.summary, _find.hit, _find.via, round(_find.score, 4) AS score${linesCol}
        FROM _find JOIN frontmatter f ON f."path" = _find."path" JOIN content ON content.path = _find."path"
        ${where} ORDER BY _find.score DESC LIMIT ?`).all(k);
@@ -103,10 +115,15 @@ export function mapTree(db, cfg) {
     const docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM("_size"), 0) AS bytes FROM frontmatter').get();
     const columns = db.prepare('PRAGMA table_info(frontmatter)').all().map((r)=>r.name).filter((name)=>!INTERNAL_COLUMNS.has(name));
     const allFields = columns.map((name)=>{
-        const { n } = db.prepare(`SELECT COUNT("${name.split('"').join('""')}") AS n FROM frontmatter`).get();
+        const quoted = `"${name.split('"').join('""')}"`;
+        // Observed storage class, not a declared one: these columns are added dynamically and
+        // SQLite types per value, so a field can be text in most notes and numeric in a few.
+        // Listing every distinct type makes both the type and any drift visible.
+        const { n, types } = db.prepare(`SELECT COUNT(${quoted}) AS n, GROUP_CONCAT(DISTINCT typeof(${quoted})) AS types FROM frontmatter WHERE ${quoted} IS NOT NULL`).get();
         return {
             field: name,
-            coverage: n
+            coverage: n,
+            type: types !== null && types !== void 0 ? types : ''
         };
     }).sort((a, b)=>b.coverage - a.coverage);
     const fields = allFields.slice(0, 20);

package/dist/esm/verbs.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/verbs.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, FeatureName } from './config.ts';\nimport { featureEnabled, featureStates } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { semanticCandidates } from './features/embed.ts';\nimport { linkEdges } from './features/index.ts';\nimport { personalizedRank } from './graph.ts';\nimport type { Row } from './output.ts';\n\n// The three layer verbs: mapTree (orient), find (locate), peek (structure).\n// Each returns data; cli.ts renders. All of them degrade when a feature is off.\n\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0)';\nconst RRF_K = 60;\n\nexport interface FindOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  semantic?: boolean; // invoke vector expansion (requires features.embed); rows gain via 'vector' and a lines column\n}\n\n// Layer 1: BM25 + link-graph expansion, fused by reciprocal rank. `via` says which\n// signal produced each row so the agent knows what evidence it is trusting.\n// Semantic expansion is per-query opt-in: without opts.semantic the result is\n// byte-for-byte independent of the embed feature.\nexport async function find(db: DatabaseSync, cfg: Config, terms: string, opts: FindOptions = {}): Promise<Row[]> {\n  const k = opts.k ?? 10;\n  const fetch = Math.max(k * 3, 30);\n\n  // Terms pass verbatim to FTS5 MATCH: bare words AND-join, operators are the caller's.\n  // Invalid syntax propagates as an error, zero matches return zero -- no silent rewrites.\n  // --where applies inside the candidate query (a post-filter over the top-N would drop\n  // matches ranked past the pool) and again on the final select for link-derived rows.\n  const whereJoin = opts.where ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = opts.where ? `AND (${opts.where})` : '';\n  const matchSql = `SELECT content.path AS path, snippet(content, -1, '«', '»', '…', 10) AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const matchRows = db.prepare(matchSql).all(terms) as Array<{ path: string; hit: string }>;\n\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  const candidates = new Map<string, { score: number; via: string }>();\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: 1 / (RRF_K + i), via: 'match' });\n  });\n\n  const edges = featureEnabled(cfg, 'links') && matchRows.length > 0 ? linkEdges(db) : [];\n  if (edges.length > 0) {\n    // `linked` gates the label only, not the score: PPR restart mass gives every seed a\n    // nonzero rank even without an incident edge, which is not link evidence — but dropping\n    // that mass from the score list reweights fusion toward connectivity and measurably\n    // wrecks ranking on link-dense corpora (FEVER hit@10 0.997 -> 0.907; fusion-tuning.md).\n    const linked = new Set(edges.flat());\n    const nodes = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n    const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n    const ranked = [...personalizedRank(nodes, edges, seeds)]\n      .filter(([, score]) => score > 1e-9)\n      .sort((a, b) => b[1] - a[1])\n      .slice(0, fetch);\n    ranked.forEach(([path], i) => {\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        if (linked.has(path)) existing.via = 'match+link';\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'link' });\n      }\n    });\n  }\n\n  // Vector expansion, invoked only: a third RRF list at the swept flat-region constants\n  // (weight 1, pool = fetch). Each row carries its best chunk's line range.\n  const chunkLines = new Map<string, string>();\n  if (opts.semantic) {\n    const vec = await semanticCandidates(db, cfg, terms, fetch);\n    vec.forEach(({ path, lines }, i) => {\n      chunkLines.set(path, lines);\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        existing.via = `${existing.via}+vector`;\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'vector' });\n      }\n    });\n  }\n\n  db.exec('DROP TABLE IF EXISTS _find');\n  db.exec('CREATE TEMP TABLE _find (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT)');\n  const insert = db.prepare('INSERT INTO _find (\"path\", score, via, hit, lines) VALUES (?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null);\n\n  const where = opts.where ? `WHERE ${opts.where}` : '';\n  const linesCol = opts.semantic ? ', _find.lines' : '';\n  return db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _find.hit, _find.via, round(_find.score, 4) AS score${linesCol}\n       FROM _find JOIN frontmatter f ON f.\"path\" = _find.\"path\" JOIN content ON content.path = _find.\"path\"\n       ${where} ORDER BY _find.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n}\n\nexport interface TreeMap {\n  docs: { count: number; bytes: number };\n  fields: Row[]; // top 20 by coverage; fieldsTotal carries the real count\n  fieldsTotal: number;\n  features: { on: FeatureName[]; off: FeatureName[] };\n  hubs: Row[];\n  recent: Row[];\n}\n\nconst INTERNAL_COLUMNS = new Set(['path', '_mtime', '_size', '_rank']);\n\n// Layer 0: what is this tree. Fixed-size output regardless of tree size.\nexport function mapTree(db: DatabaseSync, cfg: Config): TreeMap {\n  const docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM(\"_size\"), 0) AS bytes FROM frontmatter').get() as { count: number; bytes: number };\n\n  const columns = (db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>).map((r) => r.name).filter((name) => !INTERNAL_COLUMNS.has(name));\n  const allFields = columns\n    .map((name) => {\n      const { n } = db.prepare(`SELECT COUNT(\"${name.split('\"').join('\"\"')}\") AS n FROM frontmatter`).get() as { n: number };\n      return { field: name, coverage: n };\n    })\n    .sort((a, b) => (b.coverage as number) - (a.coverage as number)) as Row[];\n  const fields = allFields.slice(0, 20);\n\n  const hubs = featureEnabled(cfg, 'rank') ? (db.prepare(`SELECT f.\"path\" AS path, round(f.\"_rank\" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f.\"path\" WHERE f.\"_rank\" IS NOT NULL ORDER BY f.\"_rank\" DESC LIMIT 8`).all() as Row[]) : [];\n\n  const recent = db.prepare(`SELECT \"path\", datetime(\"_mtime\" / 1000, 'unixepoch') AS modified FROM frontmatter ORDER BY \"_mtime\" DESC LIMIT 5`).all() as Row[];\n\n  return { docs, fields, fieldsTotal: allFields.length, features: featureStates(cfg), hubs, recent };\n}\n\nexport interface Peek {\n  path: string;\n  tokens: number;\n  frontmatter: Row;\n  sections: Row[];\n  outbound: string[];\n  backlinks: string[];\n  unresolved: string[];\n  // Totals before truncation: a hub can have thousands of backlinks, and peek's whole\n  // point is bounded output. Query the links table directly for the full list.\n  outboundTotal: number;\n  backlinksTotal: number;\n  unresolvedTotal: number;\n  off: FeatureName[]; // disabled features whose blocks are omitted (not empty)\n}\n\nconst PEEK_LINK_LIMIT = 20;\n\n// Layer 2: everything about one note except its prose -- frontmatter, outline with line\n// ranges + token estimates (so the follow-up Read is a range, not the file), links both ways.\nexport function peek(db: DatabaseSync, cfg: Config, pathArg: string): Peek {\n  const paths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  let path = paths.find((p) => p === pathArg);\n  if (!path) {\n    const base = posix.basename(pathArg).replace(/\\.md$/i, '').toLowerCase();\n    const matches = paths.filter((p) => posix.basename(p).replace(/\\.md$/i, '').toLowerCase() === base);\n    if (matches.length === 1) path = matches[0];\n    else if (matches.length > 1) throw new SenseError('NOTE_AMBIGUOUS', `\"${pathArg}\" is ambiguous: ${matches.join(', ')}`);\n    else throw new SenseError('NOTE_NOT_FOUND', `no note matches \"${pathArg}\"`);\n  }\n\n  const row = db.prepare('SELECT * FROM frontmatter WHERE \"path\" = ?').get(path) as Row;\n  const frontmatter: Row = {};\n  for (const [key, value] of Object.entries(row)) {\n    if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;\n  }\n\n  const sections = featureEnabled(cfg, 'sections') ? (db.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE \"path\" = ? ORDER BY idx').all(path) as Row[]) : [];\n\n  let outbound: string[] = [];\n  let backlinks: string[] = [];\n  let unresolved: string[] = [];\n  let backlinksTotal = 0;\n  if (featureEnabled(cfg, 'links')) {\n    const out = db.prepare('SELECT target, dst FROM links WHERE src = ? ORDER BY target').all(path) as Array<{ target: string; dst: string | null }>;\n    outbound = [...new Set(out.filter((l) => l.dst !== null).map((l) => l.dst as string))];\n    unresolved = out.filter((l) => l.dst === null).map((l) => l.target);\n    backlinksTotal = (db.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ?').get(path) as { n: number }).n;\n    backlinks = (db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? ORDER BY src LIMIT ?').all(path, PEEK_LINK_LIMIT) as Array<{ src: string }>).map((r) => r.src);\n  }\n\n  return {\n    path,\n    tokens: Math.ceil(((row._size as number) ?? 0) / 4),\n    frontmatter,\n    sections,\n    outbound: outbound.slice(0, PEEK_LINK_LIMIT),\n    backlinks,\n    unresolved: unresolved.slice(0, PEEK_LINK_LIMIT),\n    outboundTotal: outbound.length,\n    backlinksTotal,\n    unresolvedTotal: unresolved.length,\n    off: (['sections', 'links'] as FeatureName[]).filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n"],"names":["posix","featureEnabled","featureStates","SenseError","semanticCandidates","linkEdges","personalizedRank","WEIGHTED_BM25","RRF_K","find","db","cfg","terms","opts","hits","chunkLines","k","fetch","Math","max","whereJoin","where","whereCond","matchSql","matchRows","prepare","all","Map","map","r","path","hit","candidates","forEach","i","set","score","via","edges","length","linked","Set","flat","nodes","seeds","ranked","filter","sort","a","b","slice","existing","get","has","semantic","vec","lines","exec","insert","c","run","linesCol","INTERNAL_COLUMNS","mapTree","docs","columns","name","allFields","n","split","join","field","coverage","fields","hubs","recent","fieldsTotal","features","PEEK_LINK_LIMIT","peek","pathArg","row","paths","p","base","basename","replace","toLowerCase","matches","frontmatter","key","value","Object","entries","sections","outbound","backlinks","unresolved","backlinksTotal","out","l","dst","target","src","tokens","ceil","_size","outboundTotal","unresolvedTotal","off"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAGpC,SAASC,cAAc,EAAEC,aAAa,QAAQ,cAAc;AAC5D,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,kBAAkB,QAAQ,sBAAsB;AACzD,SAASC,SAAS,QAAQ,sBAAsB;AAChD,SAASC,gBAAgB,QAAQ,aAAa;AAG9C,4EAA4E;AAC5E,gFAAgF;AAEhF,MAAMC,gBAAgB;AACtB,MAAMC,QAAQ;AAQd,mFAAmF;AACnF,4EAA4E;AAC5E,8EAA8E;AAC9E,kDAAkD;AAClD,OAAO,eAAeC,KAAKC,EAAgB,EAAEC,GAAW,EAAEC,KAAa,EAAEC,OAAoB,CAAC,CAAC;QACnFA,SA8D2DC,WAAwBC;IA9D7F,MAAMC,KAAIH,UAAAA,KAAKG,CAAC,cAANH,qBAAAA,UAAU;IACpB,MAAMI,QAAQC,KAAKC,GAAG,CAACH,IAAI,GAAG;IAE9B,sFAAsF;IACtF,yFAAyF;IACzF,sFAAsF;IACtF,qFAAqF;IACrF,MAAMI,YAAYP,KAAKQ,KAAK,GAAG,CAAC,6CAA6C,CAAC,GAAG;IACjF,MAAMC,YAAYT,KAAKQ,KAAK,GAAG,CAAC,KAAK,EAAER,KAAKQ,KAAK,CAAC,CAAC,CAAC,GAAG;IACvD,MAAME,WAAW,CAAC,yFAAyF,EAAEH,UAAU,uBAAuB,EAAEE,UAAU,UAAU,EAAEf,cAAc,OAAO,EAAEU,OAAO;IACpM,MAAMO,YAAYd,GAAGe,OAAO,CAACF,UAAUG,GAAG,CAACd;IAE3C,MAAME,OAAO,IAAIa,IAAIH,UAAUI,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEC,IAAI;YAAED,EAAEE,GAAG;SAAC;IACzD,MAAMC,aAAa,IAAIL;IACvBH,UAAUS,OAAO,CAAC,CAACJ,GAAGK;QACpBF,WAAWG,GAAG,CAACN,EAAEC,IAAI,EAAE;YAAEM,OAAO,IAAK5B,CAAAA,QAAQ0B,CAAAA;YAAIG,KAAK;QAAQ;IAChE;IAEA,MAAMC,QAAQrC,eAAeU,KAAK,YAAYa,UAAUe,MAAM,GAAG,IAAIlC,UAAUK,MAAM,EAAE;IACvF,IAAI4B,MAAMC,MAAM,GAAG,GAAG;QACpB,oFAAoF;QACpF,wFAAwF;QACxF,oFAAoF;QACpF,wFAAwF;QACxF,MAAMC,SAAS,IAAIC,IAAIH,MAAMI,IAAI;QACjC,MAAMC,QAAQ,AAACjC,GAAGe,OAAO,CAAC,kCAAkCC,GAAG,GAA+BE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;QAC/G,MAAMc,QAAQ,IAAIjB,IAAIH,UAAUI,GAAG,CAAC,CAACC,GAAGK,IAAM;gBAACL,EAAEC,IAAI;gBAAE,IAAKI,CAAAA,IAAI,CAAA;aAAG;QACnE,MAAMW,SAAS;eAAIvC,iBAAiBqC,OAAOL,OAAOM;SAAO,CACtDE,MAAM,CAAC,CAAC,GAAGV,MAAM,GAAKA,QAAQ,MAC9BW,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGjC;QACZ4B,OAAOZ,OAAO,CAAC,CAAC,CAACH,KAAK,EAAEI;YACtB,MAAMiB,WAAWnB,WAAWoB,GAAG,CAACtB;YAChC,IAAIqB,UAAU;gBACZA,SAASf,KAAK,IAAI,IAAK5B,CAAAA,QAAQ0B,CAAAA;gBAC/B,IAAIM,OAAOa,GAAG,CAACvB,OAAOqB,SAASd,GAAG,GAAG;YACvC,OAAO;gBACLL,WAAWG,GAAG,CAACL,MAAM;oBAAEM,OAAO,IAAK5B,CAAAA,QAAQ0B,CAAAA;oBAAIG,KAAK;gBAAO;YAC7D;QACF;IACF;IAEA,sFAAsF;IACtF,0EAA0E;IAC1E,MAAMtB,aAAa,IAAIY;IACvB,IAAId,KAAKyC,QAAQ,EAAE;QACjB,MAAMC,MAAM,MAAMnD,mBAAmBM,IAAIC,KAAKC,OAAOK;QACrDsC,IAAItB,OAAO,CAAC,CAAC,EAAEH,IAAI,EAAE0B,KAAK,EAAE,EAAEtB;YAC5BnB,WAAWoB,GAAG,CAACL,MAAM0B;YACrB,MAAML,WAAWnB,WAAWoB,GAAG,CAACtB;YAChC,IAAIqB,UAAU;gBACZA,SAASf,KAAK,IAAI,IAAK5B,CAAAA,QAAQ0B,CAAAA;gBAC/BiB,SAASd,GAAG,GAAG,GAAGc,SAASd,GAAG,CAAC,OAAO,CAAC;YACzC,OAAO;gBACLL,WAAWG,GAAG,CAACL,MAAM;oBAAEM,OAAO,IAAK5B,CAAAA,QAAQ0B,CAAAA;oBAAIG,KAAK;gBAAS;YAC/D;QACF;IACF;IAEA3B,GAAG+C,IAAI,CAAC;IACR/C,GAAG+C,IAAI,CAAC;IACR,MAAMC,SAAShD,GAAGe,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACK,MAAM6B,EAAE,IAAI3B,WAAY0B,OAAOE,GAAG,CAAC9B,MAAM6B,EAAEvB,KAAK,EAAEuB,EAAEtB,GAAG,GAAEvB,YAAAA,KAAKsC,GAAG,CAACtB,mBAAThB,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWqC,GAAG,CAACtB,mBAAff,6BAAAA,kBAAwB;IAErH,MAAMM,QAAQR,KAAKQ,KAAK,GAAG,CAAC,MAAM,EAAER,KAAKQ,KAAK,EAAE,GAAG;IACnD,MAAMwC,WAAWhD,KAAKyC,QAAQ,GAAG,kBAAkB;IACnD,OAAO5C,GACJe,OAAO,CACN,CAAC,6GAA6G,EAAEoC,SAAS;;OAExH,EAAExC,MAAM,kCAAkC,CAAC,EAE7CK,GAAG,CAACV;AACT;AAWA,MAAM8C,mBAAmB,IAAIrB,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAQ;AAErE,yEAAyE;AACzE,OAAO,SAASsB,QAAQrD,EAAgB,EAAEC,GAAW;IACnD,MAAMqD,OAAOtD,GAAGe,OAAO,CAAC,iFAAiF2B,GAAG;IAE5G,MAAMa,UAAU,AAACvD,GAAGe,OAAO,CAAC,kCAAkCC,GAAG,GAA+BE,GAAG,CAAC,CAACC,IAAMA,EAAEqC,IAAI,EAAEpB,MAAM,CAAC,CAACoB,OAAS,CAACJ,iBAAiBT,GAAG,CAACa;IAC1J,MAAMC,YAAYF,QACfrC,GAAG,CAAC,CAACsC;QACJ,MAAM,EAAEE,CAAC,EAAE,GAAG1D,GAAGe,OAAO,CAAC,CAAC,cAAc,EAAEyC,KAAKG,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,wBAAwB,CAAC,EAAElB,GAAG;QACnG,OAAO;YAAEmB,OAAOL;YAAMM,UAAUJ;QAAE;IACpC,GACCrB,IAAI,CAAC,CAACC,GAAGC,IAAM,AAACA,EAAEuB,QAAQ,GAAexB,EAAEwB,QAAQ;IACtD,MAAMC,SAASN,UAAUjB,KAAK,CAAC,GAAG;IAElC,MAAMwB,OAAOzE,eAAeU,KAAK,UAAWD,GAAGe,OAAO,CAAC,CAAC,gMAAgM,CAAC,EAAEC,GAAG,KAAe,EAAE;IAE/Q,MAAMiD,SAASjE,GAAGe,OAAO,CAAC,CAAC,iHAAiH,CAAC,EAAEC,GAAG;IAElJ,OAAO;QAAEsC;QAAMS;QAAQG,aAAaT,UAAU5B,MAAM;QAAEsC,UAAU3E,cAAcS;QAAM+D;QAAMC;IAAO;AACnG;AAkBA,MAAMG,kBAAkB;AAExB,wFAAwF;AACxF,8FAA8F;AAC9F,OAAO,SAASC,KAAKrE,EAAgB,EAAEC,GAAW,EAAEqE,OAAe;QAiC3CC;IAhCtB,MAAMC,QAAQ,AAACxE,GAAGe,OAAO,CAAC,kCAAkCC,GAAG,GAA+BE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAC/G,IAAIA,OAAOoD,MAAMzE,IAAI,CAAC,CAAC0E,IAAMA,MAAMH;IACnC,IAAI,CAAClD,MAAM;QACT,MAAMsD,OAAOpF,MAAMqF,QAAQ,CAACL,SAASM,OAAO,CAAC,UAAU,IAAIC,WAAW;QACtE,MAAMC,UAAUN,MAAMpC,MAAM,CAAC,CAACqC,IAAMnF,MAAMqF,QAAQ,CAACF,GAAGG,OAAO,CAAC,UAAU,IAAIC,WAAW,OAAOH;QAC9F,IAAII,QAAQjD,MAAM,KAAK,GAAGT,OAAO0D,OAAO,CAAC,EAAE;aACtC,IAAIA,QAAQjD,MAAM,GAAG,GAAG,MAAM,IAAIpC,WAAW,kBAAkB,CAAC,CAAC,EAAE6E,QAAQ,gBAAgB,EAAEQ,QAAQlB,IAAI,CAAC,OAAO;aACjH,MAAM,IAAInE,WAAW,kBAAkB,CAAC,iBAAiB,EAAE6E,QAAQ,CAAC,CAAC;IAC5E;IAEA,MAAMC,MAAMvE,GAAGe,OAAO,CAAC,8CAA8C2B,GAAG,CAACtB;IACzE,MAAM2D,cAAmB,CAAC;IAC1B,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAOC,OAAO,CAACZ,KAAM;QAC9C,IAAI,CAACnB,iBAAiBT,GAAG,CAACqC,QAAQC,UAAU,MAAMF,WAAW,CAACC,IAAI,GAAGC;IACvE;IAEA,MAAMG,WAAW7F,eAAeU,KAAK,cAAeD,GAAGe,OAAO,CAAC,mGAAmGC,GAAG,CAACI,QAAkB,EAAE;IAE1L,IAAIiE,WAAqB,EAAE;IAC3B,IAAIC,YAAsB,EAAE;IAC5B,IAAIC,aAAuB,EAAE;IAC7B,IAAIC,iBAAiB;IACrB,IAAIjG,eAAeU,KAAK,UAAU;QAChC,MAAMwF,MAAMzF,GAAGe,OAAO,CAAC,+DAA+DC,GAAG,CAACI;QAC1FiE,WAAW;eAAI,IAAItD,IAAI0D,IAAIrD,MAAM,CAAC,CAACsD,IAAMA,EAAEC,GAAG,KAAK,MAAMzE,GAAG,CAAC,CAACwE,IAAMA,EAAEC,GAAG;SAAa;QACtFJ,aAAaE,IAAIrD,MAAM,CAAC,CAACsD,IAAMA,EAAEC,GAAG,KAAK,MAAMzE,GAAG,CAAC,CAACwE,IAAMA,EAAEE,MAAM;QAClEJ,iBAAiB,AAACxF,GAAGe,OAAO,CAAC,4DAA4D2B,GAAG,CAACtB,MAAwBsC,CAAC;QACtH4B,YAAY,AAACtF,GAAGe,OAAO,CAAC,qEAAqEC,GAAG,CAACI,MAAMgD,iBAA4ClD,GAAG,CAAC,CAACC,IAAMA,EAAE0E,GAAG;IACrK;IAEA,OAAO;QACLzE;QACA0E,QAAQtF,KAAKuF,IAAI,CAAC,EAAExB,aAAAA,IAAIyB,KAAK,cAATzB,wBAAAA,aAAwB,KAAK;QACjDQ;QACAK;QACAC,UAAUA,SAAS7C,KAAK,CAAC,GAAG4B;QAC5BkB;QACAC,YAAYA,WAAW/C,KAAK,CAAC,GAAG4B;QAChC6B,eAAeZ,SAASxD,MAAM;QAC9B2D;QACAU,iBAAiBX,WAAW1D,MAAM;QAClCsE,KAAK,AAAC;YAAC;YAAY;SAAQ,CAAmB/D,MAAM,CAAC,CAACoB,OAAS,CAACjE,eAAeU,KAAKuD;IACtF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/verbs.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, FeatureName } from './config.ts';\nimport { featureEnabled, featureStates } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { semanticCandidates } from './features/embed.ts';\nimport { linkEdges } from './features/index.ts';\nimport { personalizedRank } from './graph.ts';\nimport type { Row } from './output.ts';\nimport { searchError } from './search-error.ts';\n\n// The three layer verbs: mapTree (orient), find (locate), peek (structure).\n// Each returns data; cli.ts renders. All of them degrade when a feature is off.\n\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0)';\nconst RRF_K = 60;\n\nexport interface FindOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n  semantic?: boolean; // invoke vector expansion (requires features.embed); rows gain via 'vector' and a lines column\n}\n\n// Layer 1: BM25 + link-graph expansion, fused by reciprocal rank. `via` says which\n// signal produced each row so the agent knows what evidence it is trusting.\n// Semantic expansion is per-query opt-in: without opts.semantic the result is\n// byte-for-byte independent of the embed feature.\nexport async function find(db: DatabaseSync, cfg: Config, terms: string, opts: FindOptions = {}): Promise<Row[]> {\n  const k = opts.k ?? 10;\n  const fetch = Math.max(k * 3, 30);\n\n  // Terms pass verbatim to FTS5 MATCH: bare words AND-join, operators are the caller's.\n  // Invalid syntax propagates as an error, zero matches return zero -- no silent rewrites.\n  // --where applies inside the candidate query (a post-filter over the top-N would drop\n  // matches ranked past the pool) and again on the final select for link-derived rows.\n  // An explicit --where replaces the tree's declared default rather than ANDing with it,\n  // so a caller can always widen back to the whole tree.\n  const scope = opts.where ?? cfg.defaults?.find?.where;\n  const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n  const whereCond = scope ? `AND (${scope})` : '';\n  const matchSql = `SELECT content.path AS path, snippet(content, -1, '«', '»', '…', 10) AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  let matchRows: Array<{ path: string; hit: string }>;\n  try {\n    matchRows = db.prepare(matchSql).all(terms) as Array<{ path: string; hit: string }>;\n  } catch (err) {\n    throw searchError(err as Error, terms, scope);\n  }\n\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  const candidates = new Map<string, { score: number; via: string }>();\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: 1 / (RRF_K + i), via: 'match' });\n  });\n\n  const edges = featureEnabled(cfg, 'links') && matchRows.length > 0 ? linkEdges(db) : [];\n  if (edges.length > 0) {\n    // `linked` gates the label only, not the score: PPR restart mass gives every seed a\n    // nonzero rank even without an incident edge, which is not link evidence — but dropping\n    // that mass from the score list reweights fusion toward connectivity and measurably\n    // wrecks ranking on link-dense corpora (FEVER hit@10 0.997 -> 0.907; fusion-tuning.md).\n    const linked = new Set(edges.flat());\n    const nodes = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n    const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n    const ranked = [...personalizedRank(nodes, edges, seeds)]\n      .filter(([, score]) => score > 1e-9)\n      .sort((a, b) => b[1] - a[1])\n      .slice(0, fetch);\n    ranked.forEach(([path], i) => {\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        if (linked.has(path)) existing.via = 'match+link';\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'link' });\n      }\n    });\n  }\n\n  // Vector expansion, invoked only: a third RRF list at the swept flat-region constants\n  // (weight 1, pool = fetch). Each row carries its best chunk's line range.\n  const chunkLines = new Map<string, string>();\n  const chunkSimilarity = new Map<string, number>();\n  if (opts.semantic) {\n    const vec = await semanticCandidates(db, cfg, terms, fetch);\n    vec.forEach(({ path, lines, similarity }, i) => {\n      chunkLines.set(path, lines);\n      chunkSimilarity.set(path, similarity);\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        existing.via = `${existing.via}+vector`;\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'vector' });\n      }\n    });\n  }\n\n  db.exec('DROP TABLE IF EXISTS _find');\n  db.exec('CREATE TEMP TABLE _find (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');\n  const insert = db.prepare('INSERT INTO _find (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null);\n\n  const where = scope ? `WHERE ${scope}` : '';\n  const linesCol = opts.semantic ? ', _find.lines, _find.similarity' : '';\n  return db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _find.hit, _find.via, round(_find.score, 4) AS score${linesCol}\n       FROM _find JOIN frontmatter f ON f.\"path\" = _find.\"path\" JOIN content ON content.path = _find.\"path\"\n       ${where} ORDER BY _find.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n}\n\nexport interface TreeMap {\n  docs: { count: number; bytes: number };\n  fields: Row[]; // top 20 by coverage; fieldsTotal carries the real count\n  fieldsTotal: number;\n  features: { on: FeatureName[]; off: FeatureName[] };\n  hubs: Row[];\n  recent: Row[];\n}\n\nconst INTERNAL_COLUMNS = new Set(['path', '_mtime', '_size', '_rank']);\n\n// Layer 0: what is this tree. Fixed-size output regardless of tree size.\nexport function mapTree(db: DatabaseSync, cfg: Config): TreeMap {\n  const docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM(\"_size\"), 0) AS bytes FROM frontmatter').get() as { count: number; bytes: number };\n\n  const columns = (db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>).map((r) => r.name).filter((name) => !INTERNAL_COLUMNS.has(name));\n  const allFields = columns\n    .map((name) => {\n      const quoted = `\"${name.split('\"').join('\"\"')}\"`;\n      // Observed storage class, not a declared one: these columns are added dynamically and\n      // SQLite types per value, so a field can be text in most notes and numeric in a few.\n      // Listing every distinct type makes both the type and any drift visible.\n      const { n, types } = db.prepare(`SELECT COUNT(${quoted}) AS n, GROUP_CONCAT(DISTINCT typeof(${quoted})) AS types FROM frontmatter WHERE ${quoted} IS NOT NULL`).get() as { n: number; types: string | null };\n      return { field: name, coverage: n, type: types ?? '' };\n    })\n    .sort((a, b) => (b.coverage as number) - (a.coverage as number)) as Row[];\n  const fields = allFields.slice(0, 20);\n\n  const hubs = featureEnabled(cfg, 'rank') ? (db.prepare(`SELECT f.\"path\" AS path, round(f.\"_rank\" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f.\"path\" WHERE f.\"_rank\" IS NOT NULL ORDER BY f.\"_rank\" DESC LIMIT 8`).all() as Row[]) : [];\n\n  const recent = db.prepare(`SELECT \"path\", datetime(\"_mtime\" / 1000, 'unixepoch') AS modified FROM frontmatter ORDER BY \"_mtime\" DESC LIMIT 5`).all() as Row[];\n\n  return { docs, fields, fieldsTotal: allFields.length, features: featureStates(cfg), hubs, recent };\n}\n\nexport interface Peek {\n  path: string;\n  tokens: number;\n  frontmatter: Row;\n  sections: Row[];\n  outbound: string[];\n  backlinks: string[];\n  unresolved: string[];\n  // Totals before truncation: a hub can have thousands of backlinks, and peek's whole\n  // point is bounded output. Query the links table directly for the full list.\n  outboundTotal: number;\n  backlinksTotal: number;\n  unresolvedTotal: number;\n  off: FeatureName[]; // disabled features whose blocks are omitted (not empty)\n}\n\nconst PEEK_LINK_LIMIT = 20;\n\n// Layer 2: everything about one note except its prose -- frontmatter, outline with line\n// ranges + token estimates (so the follow-up Read is a range, not the file), links both ways.\nexport function peek(db: DatabaseSync, cfg: Config, pathArg: string): Peek {\n  const paths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  let path = paths.find((p) => p === pathArg);\n  if (!path) {\n    const base = posix.basename(pathArg).replace(/\\.md$/i, '').toLowerCase();\n    const matches = paths.filter((p) => posix.basename(p).replace(/\\.md$/i, '').toLowerCase() === base);\n    if (matches.length === 1) path = matches[0];\n    else if (matches.length > 1) throw new SenseError('NOTE_AMBIGUOUS', `\"${pathArg}\" is ambiguous: ${matches.join(', ')}`);\n    else throw new SenseError('NOTE_NOT_FOUND', `no note matches \"${pathArg}\"`);\n  }\n\n  const row = db.prepare('SELECT * FROM frontmatter WHERE \"path\" = ?').get(path) as Row;\n  const frontmatter: Row = {};\n  for (const [key, value] of Object.entries(row)) {\n    if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;\n  }\n\n  const sections = featureEnabled(cfg, 'sections') ? (db.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE \"path\" = ? ORDER BY idx').all(path) as Row[]) : [];\n\n  let outbound: string[] = [];\n  let backlinks: string[] = [];\n  let unresolved: string[] = [];\n  let backlinksTotal = 0;\n  if (featureEnabled(cfg, 'links')) {\n    const out = db.prepare('SELECT target, dst FROM links WHERE src = ? ORDER BY target').all(path) as Array<{ target: string; dst: string | null }>;\n    outbound = [...new Set(out.filter((l) => l.dst !== null).map((l) => l.dst as string))];\n    unresolved = out.filter((l) => l.dst === null).map((l) => l.target);\n    backlinksTotal = (db.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ?').get(path) as { n: number }).n;\n    backlinks = (db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? ORDER BY src LIMIT ?').all(path, PEEK_LINK_LIMIT) as Array<{ src: string }>).map((r) => r.src);\n  }\n\n  return {\n    path,\n    tokens: Math.ceil(((row._size as number) ?? 0) / 4),\n    frontmatter,\n    sections,\n    outbound: outbound.slice(0, PEEK_LINK_LIMIT),\n    backlinks,\n    unresolved: unresolved.slice(0, PEEK_LINK_LIMIT),\n    outboundTotal: outbound.length,\n    backlinksTotal,\n    unresolvedTotal: unresolved.length,\n    off: (['sections', 'links'] as FeatureName[]).filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n"],"names":["posix","featureEnabled","featureStates","SenseError","semanticCandidates","linkEdges","personalizedRank","searchError","WEIGHTED_BM25","RRF_K","find","db","cfg","terms","opts","hits","chunkLines","chunkSimilarity","k","fetch","Math","max","scope","where","defaults","whereJoin","whereCond","matchSql","matchRows","prepare","all","err","Map","map","r","path","hit","candidates","forEach","i","set","score","via","edges","length","linked","Set","flat","nodes","seeds","ranked","filter","sort","a","b","slice","existing","get","has","semantic","vec","lines","similarity","exec","insert","c","run","linesCol","INTERNAL_COLUMNS","mapTree","docs","columns","name","allFields","quoted","split","join","n","types","field","coverage","type","fields","hubs","recent","fieldsTotal","features","PEEK_LINK_LIMIT","peek","pathArg","row","paths","p","base","basename","replace","toLowerCase","matches","frontmatter","key","value","Object","entries","sections","outbound","backlinks","unresolved","backlinksTotal","out","l","dst","target","src","tokens","ceil","_size","outboundTotal","unresolvedTotal","off"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAGpC,SAASC,cAAc,EAAEC,aAAa,QAAQ,cAAc;AAC5D,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,kBAAkB,QAAQ,sBAAsB;AACzD,SAASC,SAAS,QAAQ,sBAAsB;AAChD,SAASC,gBAAgB,QAAQ,aAAa;AAE9C,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,4EAA4E;AAC5E,gFAAgF;AAEhF,MAAMC,gBAAgB;AACtB,MAAMC,QAAQ;AAQd,mFAAmF;AACnF,4EAA4E;AAC5E,8EAA8E;AAC9E,kDAAkD;AAClD,OAAO,eAAeC,KAAKC,EAAgB,EAAEC,GAAW,EAAEC,KAAa,EAAEC,OAAoB,CAAC,CAAC;QACnFA,SASIA,aA+DuDC,WAAwBC,iBAA8BC;QA/D/FL,oBAAAA;IAT5B,MAAMM,KAAIJ,UAAAA,KAAKI,CAAC,cAANJ,qBAAAA,UAAU;IACpB,MAAMK,QAAQC,KAAKC,GAAG,CAACH,IAAI,GAAG;IAE9B,sFAAsF;IACtF,yFAAyF;IACzF,sFAAsF;IACtF,qFAAqF;IACrF,uFAAuF;IACvF,uDAAuD;IACvD,MAAMI,SAAQR,cAAAA,KAAKS,KAAK,cAAVT,yBAAAA,eAAcF,gBAAAA,IAAIY,QAAQ,cAAZZ,qCAAAA,qBAAAA,cAAcF,IAAI,cAAlBE,yCAAAA,mBAAoBW,KAAK;IACrD,MAAME,YAAYH,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAMI,YAAYJ,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,MAAMK,WAAW,CAAC,yFAAyF,EAAEF,UAAU,uBAAuB,EAAEC,UAAU,UAAU,EAAElB,cAAc,OAAO,EAAEW,OAAO;IACpM,IAAIS;IACJ,IAAI;QACFA,YAAYjB,GAAGkB,OAAO,CAACF,UAAUG,GAAG,CAACjB;IACvC,EAAE,OAAOkB,KAAK;QACZ,MAAMxB,YAAYwB,KAAclB,OAAOS;IACzC;IAEA,MAAMP,OAAO,IAAIiB,IAAIJ,UAAUK,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEC,IAAI;YAAED,EAAEE,GAAG;SAAC;IACzD,MAAMC,aAAa,IAAIL;IACvBJ,UAAUU,OAAO,CAAC,CAACJ,GAAGK;QACpBF,WAAWG,GAAG,CAACN,EAAEC,IAAI,EAAE;YAAEM,OAAO,IAAKhC,CAAAA,QAAQ8B,CAAAA;YAAIG,KAAK;QAAQ;IAChE;IAEA,MAAMC,QAAQ1C,eAAeW,KAAK,YAAYgB,UAAUgB,MAAM,GAAG,IAAIvC,UAAUM,MAAM,EAAE;IACvF,IAAIgC,MAAMC,MAAM,GAAG,GAAG;QACpB,oFAAoF;QACpF,wFAAwF;QACxF,oFAAoF;QACpF,wFAAwF;QACxF,MAAMC,SAAS,IAAIC,IAAIH,MAAMI,IAAI;QACjC,MAAMC,QAAQ,AAACrC,GAAGkB,OAAO,CAAC,kCAAkCC,GAAG,GAA+BG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;QAC/G,MAAMc,QAAQ,IAAIjB,IAAIJ,UAAUK,GAAG,CAAC,CAACC,GAAGK,IAAM;gBAACL,EAAEC,IAAI;gBAAE,IAAKI,CAAAA,IAAI,CAAA;aAAG;QACnE,MAAMW,SAAS;eAAI5C,iBAAiB0C,OAAOL,OAAOM;SAAO,CACtDE,MAAM,CAAC,CAAC,GAAGV,MAAM,GAAKA,QAAQ,MAC9BW,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAGpC;QACZ+B,OAAOZ,OAAO,CAAC,CAAC,CAACH,KAAK,EAAEI;YACtB,MAAMiB,WAAWnB,WAAWoB,GAAG,CAACtB;YAChC,IAAIqB,UAAU;gBACZA,SAASf,KAAK,IAAI,IAAKhC,CAAAA,QAAQ8B,CAAAA;gBAC/B,IAAIM,OAAOa,GAAG,CAACvB,OAAOqB,SAASd,GAAG,GAAG;YACvC,OAAO;gBACLL,WAAWG,GAAG,CAACL,MAAM;oBAAEM,OAAO,IAAKhC,CAAAA,QAAQ8B,CAAAA;oBAAIG,KAAK;gBAAO;YAC7D;QACF;IACF;IAEA,sFAAsF;IACtF,0EAA0E;IAC1E,MAAM1B,aAAa,IAAIgB;IACvB,MAAMf,kBAAkB,IAAIe;IAC5B,IAAIlB,KAAK6C,QAAQ,EAAE;QACjB,MAAMC,MAAM,MAAMxD,mBAAmBO,IAAIC,KAAKC,OAAOM;QACrDyC,IAAItB,OAAO,CAAC,CAAC,EAAEH,IAAI,EAAE0B,KAAK,EAAEC,UAAU,EAAE,EAAEvB;YACxCvB,WAAWwB,GAAG,CAACL,MAAM0B;YACrB5C,gBAAgBuB,GAAG,CAACL,MAAM2B;YAC1B,MAAMN,WAAWnB,WAAWoB,GAAG,CAACtB;YAChC,IAAIqB,UAAU;gBACZA,SAASf,KAAK,IAAI,IAAKhC,CAAAA,QAAQ8B,CAAAA;gBAC/BiB,SAASd,GAAG,GAAG,GAAGc,SAASd,GAAG,CAAC,OAAO,CAAC;YACzC,OAAO;gBACLL,WAAWG,GAAG,CAACL,MAAM;oBAAEM,OAAO,IAAKhC,CAAAA,QAAQ8B,CAAAA;oBAAIG,KAAK;gBAAS;YAC/D;QACF;IACF;IAEA/B,GAAGoD,IAAI,CAAC;IACRpD,GAAGoD,IAAI,CAAC;IACR,MAAMC,SAASrD,GAAGkB,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACM,MAAM8B,EAAE,IAAI5B,WAAY2B,OAAOE,GAAG,CAAC/B,MAAM8B,EAAExB,KAAK,EAAEwB,EAAEvB,GAAG,GAAE3B,YAAAA,KAAK0C,GAAG,CAACtB,mBAATpB,uBAAAA,YAAkB,OAAMC,kBAAAA,WAAWyC,GAAG,CAACtB,mBAAfnB,6BAAAA,kBAAwB,OAAMC,uBAAAA,gBAAgBwC,GAAG,CAACtB,mBAApBlB,kCAAAA,uBAA6B;IAExJ,MAAMM,QAAQD,QAAQ,CAAC,MAAM,EAAEA,OAAO,GAAG;IACzC,MAAM6C,WAAWrD,KAAK6C,QAAQ,GAAG,oCAAoC;IACrE,OAAOhD,GACJkB,OAAO,CACN,CAAC,6GAA6G,EAAEsC,SAAS;;OAExH,EAAE5C,MAAM,kCAAkC,CAAC,EAE7CO,GAAG,CAACZ;AACT;AAWA,MAAMkD,mBAAmB,IAAItB,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAQ;AAErE,yEAAyE;AACzE,OAAO,SAASuB,QAAQ1D,EAAgB,EAAEC,GAAW;IACnD,MAAM0D,OAAO3D,GAAGkB,OAAO,CAAC,iFAAiF4B,GAAG;IAE5G,MAAMc,UAAU,AAAC5D,GAAGkB,OAAO,CAAC,kCAAkCC,GAAG,GAA+BG,GAAG,CAAC,CAACC,IAAMA,EAAEsC,IAAI,EAAErB,MAAM,CAAC,CAACqB,OAAS,CAACJ,iBAAiBV,GAAG,CAACc;IAC1J,MAAMC,YAAYF,QACftC,GAAG,CAAC,CAACuC;QACJ,MAAME,SAAS,CAAC,CAAC,EAAEF,KAAKG,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,sFAAsF;QACtF,qFAAqF;QACrF,yEAAyE;QACzE,MAAM,EAAEC,CAAC,EAAEC,KAAK,EAAE,GAAGnE,GAAGkB,OAAO,CAAC,CAAC,aAAa,EAAE6C,OAAO,qCAAqC,EAAEA,OAAO,mCAAmC,EAAEA,OAAO,YAAY,CAAC,EAAEjB,GAAG;QACnK,OAAO;YAAEsB,OAAOP;YAAMQ,UAAUH;YAAGI,IAAI,EAAEH,kBAAAA,mBAAAA,QAAS;QAAG;IACvD,GACC1B,IAAI,CAAC,CAACC,GAAGC,IAAM,AAACA,EAAE0B,QAAQ,GAAe3B,EAAE2B,QAAQ;IACtD,MAAME,SAAST,UAAUlB,KAAK,CAAC,GAAG;IAElC,MAAM4B,OAAOlF,eAAeW,KAAK,UAAWD,GAAGkB,OAAO,CAAC,CAAC,gMAAgM,CAAC,EAAEC,GAAG,KAAe,EAAE;IAE/Q,MAAMsD,SAASzE,GAAGkB,OAAO,CAAC,CAAC,iHAAiH,CAAC,EAAEC,GAAG;IAElJ,OAAO;QAAEwC;QAAMY;QAAQG,aAAaZ,UAAU7B,MAAM;QAAE0C,UAAUpF,cAAcU;QAAMuE;QAAMC;IAAO;AACnG;AAkBA,MAAMG,kBAAkB;AAExB,wFAAwF;AACxF,8FAA8F;AAC9F,OAAO,SAASC,KAAK7E,EAAgB,EAAEC,GAAW,EAAE6E,OAAe;QAiC3CC;IAhCtB,MAAMC,QAAQ,AAAChF,GAAGkB,OAAO,CAAC,kCAAkCC,GAAG,GAA+BG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAC/G,IAAIA,OAAOwD,MAAMjF,IAAI,CAAC,CAACkF,IAAMA,MAAMH;IACnC,IAAI,CAACtD,MAAM;QACT,MAAM0D,OAAO7F,MAAM8F,QAAQ,CAACL,SAASM,OAAO,CAAC,UAAU,IAAIC,WAAW;QACtE,MAAMC,UAAUN,MAAMxC,MAAM,CAAC,CAACyC,IAAM5F,MAAM8F,QAAQ,CAACF,GAAGG,OAAO,CAAC,UAAU,IAAIC,WAAW,OAAOH;QAC9F,IAAII,QAAQrD,MAAM,KAAK,GAAGT,OAAO8D,OAAO,CAAC,EAAE;aACtC,IAAIA,QAAQrD,MAAM,GAAG,GAAG,MAAM,IAAIzC,WAAW,kBAAkB,CAAC,CAAC,EAAEsF,QAAQ,gBAAgB,EAAEQ,QAAQrB,IAAI,CAAC,OAAO;aACjH,MAAM,IAAIzE,WAAW,kBAAkB,CAAC,iBAAiB,EAAEsF,QAAQ,CAAC,CAAC;IAC5E;IAEA,MAAMC,MAAM/E,GAAGkB,OAAO,CAAC,8CAA8C4B,GAAG,CAACtB;IACzE,MAAM+D,cAAmB,CAAC;IAC1B,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAOC,OAAO,CAACZ,KAAM;QAC9C,IAAI,CAACtB,iBAAiBV,GAAG,CAACyC,QAAQC,UAAU,MAAMF,WAAW,CAACC,IAAI,GAAGC;IACvE;IAEA,MAAMG,WAAWtG,eAAeW,KAAK,cAAeD,GAAGkB,OAAO,CAAC,mGAAmGC,GAAG,CAACK,QAAkB,EAAE;IAE1L,IAAIqE,WAAqB,EAAE;IAC3B,IAAIC,YAAsB,EAAE;IAC5B,IAAIC,aAAuB,EAAE;IAC7B,IAAIC,iBAAiB;IACrB,IAAI1G,eAAeW,KAAK,UAAU;QAChC,MAAMgG,MAAMjG,GAAGkB,OAAO,CAAC,+DAA+DC,GAAG,CAACK;QAC1FqE,WAAW;eAAI,IAAI1D,IAAI8D,IAAIzD,MAAM,CAAC,CAAC0D,IAAMA,EAAEC,GAAG,KAAK,MAAM7E,GAAG,CAAC,CAAC4E,IAAMA,EAAEC,GAAG;SAAa;QACtFJ,aAAaE,IAAIzD,MAAM,CAAC,CAAC0D,IAAMA,EAAEC,GAAG,KAAK,MAAM7E,GAAG,CAAC,CAAC4E,IAAMA,EAAEE,MAAM;QAClEJ,iBAAiB,AAAChG,GAAGkB,OAAO,CAAC,4DAA4D4B,GAAG,CAACtB,MAAwB0C,CAAC;QACtH4B,YAAY,AAAC9F,GAAGkB,OAAO,CAAC,qEAAqEC,GAAG,CAACK,MAAMoD,iBAA4CtD,GAAG,CAAC,CAACC,IAAMA,EAAE8E,GAAG;IACrK;IAEA,OAAO;QACL7E;QACA8E,QAAQ7F,KAAK8F,IAAI,CAAC,EAAExB,aAAAA,IAAIyB,KAAK,cAATzB,wBAAAA,aAAwB,KAAK;QACjDQ;QACAK;QACAC,UAAUA,SAASjD,KAAK,CAAC,GAAGgC;QAC5BkB;QACAC,YAAYA,WAAWnD,KAAK,CAAC,GAAGgC;QAChC6B,eAAeZ,SAAS5D,MAAM;QAC9B+D;QACAU,iBAAiBX,WAAW9D,MAAM;QAClC0E,KAAK,AAAC;YAAC;YAAY;SAAQ,CAAmBnE,MAAM,CAAC,CAACqB,OAAS,CAACvE,eAAeW,KAAK4D;IACtF;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.6.0",
+  "version": "0.7.1",
   "description": "Query a knowledge base you build with an agent: filter notes by frontmatter, then search inside them",
   "keywords": [
     "markdown",

package/schema.json

@@ -64,9 +64,32 @@
         }
       }
     },
+    "defaults": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Tree-declared defaults for the verbs.",
+      "properties": {
+        "find": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Default scope for `find`. Use when a tree holds a layer that should stay queryable but out of ordinary search -- generated output, or unverified source extractions that must not be cited as conclusions.",
+          "properties": {
+            "where": {
+              "type": "string",
+              "description": "SQL condition against frontmatter alias `f`, e.g. \"f.type != 'raw'\". Applied when `find` runs without --where; an explicit --where replaces it (so `--where \"1=1\"` searches the whole tree). Reported by `sense status`."
+            }
+          }
+        }
+      }
+    },
+    "checks": {
+      "type": "object",
+      "description": "Assertions over saved queries, for trees whose queries encode invariants: `\"checks\": { \"dead-links\": \"empty\" }` makes `sense check` fail (exit 1) when that query returns rows. Without an entry, `check` reports zero-row queries as `ok, but returns 0 rows` -- informational, since an ordinary query returning nothing is sometimes a bug and sometimes just an empty result. Keys must name saved queries.",
+      "additionalProperties": { "type": "string", "enum": ["empty"] }
+    },
     "queries": {
       "type": "object",
-      "description": "Named SQL queries runnable as `sense <name> [params...]`. Tables: `frontmatter` (one row per file, one column per discovered frontmatter key, plus `path`/`_mtime`/`_size`/`_rank`), `content` (FTS5: `title`, `summary`, `text`, `path`), `links` (`src`, `target`, `dst`), and `sections` (`path`, `idx`, `level`, `heading`, `start_line`, `end_line`, `tokens`). `?` placeholders bind to CLI positional args in order. `has(field, value)`: array membership on a JSON-array field, substring match on a string (so has(f.status, 'active') also matches 'inactive'), false on NULL. Exact matches: `=` for scalars, `EXISTS (SELECT 1 FROM json_each(f.tags) WHERE value = ?)` for array members. Canonical query: `SELECT f.path, content.title, content.summary, snippet(content, -1, '«', '»', '…', 10) AS hit FROM frontmatter f JOIN content ON content.path = f.path WHERE content MATCH ? ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10`. Reserved frontmatter keys: `path`, `_mtime`, `_size`, `_rank`, `content`, `links`, `sections`. Reserved query names (unreachable as subcommands): `init`, `query`, `find`, `map`, `peek`, `watch`, `status`, `rebuild`.",
+      "description": "Named SQL queries runnable as `sense <name> [params...]`. Tables: `frontmatter` (one row per file, one column per discovered frontmatter key, plus `path`/`_mtime`/`_size`/`_rank`), `content` (FTS5: `title`, `summary`, `text`, `path`), `links` (`src`, `target`, `dst`), and `sections` (`path`, `idx`, `level`, `heading`, `start_line`, `end_line`, `tokens`). `?` placeholders bind to CLI positional args in order. `has(field, value)`: array membership on a JSON-array field, substring match on a string (so has(f.status, 'active') also matches 'inactive'), false on NULL. Exact matches: `=` for scalars, `EXISTS (SELECT 1 FROM json_each(f.tags) WHERE value = ?)` for array members. Canonical query: `SELECT f.path, content.title, content.summary, snippet(content, -1, '«', '»', '…', 10) AS hit FROM frontmatter f JOIN content ON content.path = f.path WHERE content MATCH ? ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10`. Reserved frontmatter keys: `path`, `_mtime`, `_size`, `_rank`, `content`, `links`, `sections`. Reserved query names (unreachable as subcommands): `init`, `query`, `find`, `map`, `peek`, `watch`, `status`, `rebuild`, `check`.",
       "additionalProperties": { "type": "string" }
     }
   }

package/skills/sense/SKILL.md

@@ -45,7 +45,7 @@ sense peek notes/pricing-model.md               # a unique basename also works
 sense map
 sense query "<sql>" [params...]                 # ad-hoc SQL; ? binds positional args, count-checked
 sense <name> [params...]                        # named query from sense.config.json
-sense --list | status | rebuild
+sense --list | status | rebuild | check
 ```
 
 - Terms pass verbatim to FTS5 MATCH. Bare words AND-join — one absent word means zero rows —
@@ -63,7 +63,31 @@ sense --list | status | rebuild
   `match` (terms hit), `link` (connected to notes that hit), `match+link` (both). With
   `--semantic`, `vector` joins the composition and rows gain a `lines` column pointing at the
   best-matching section, a direct `Read` range.
-- `--where` takes a frontmatter condition against alias `f`, e.g. `"f.status = 'active' AND has(f.tags, 'x')"`.
+- `--where` takes any SQL condition against frontmatter alias `f` — not only field equality:
+  `"f.status = 'active' AND has(f.tags, 'x')"`, `"f.path NOT LIKE 'generated/%'"`,
+  `"f.created >= datetime(?)"`. A tree can declare a default scope in `sense.config.json`
+  (`defaults.find.where`); an explicit `--where` replaces it, so `--where "1=1"` searches
+  everything. `sense status` prints the active default.
+- `score` is a rank-fusion value: it ranks rows within one result set and is not comparable
+  across queries, not a relevance magnitude — a perfect lexical hit and a weak vector-only
+  hit can both read ~0.017, because the number encodes how many signals fired and at what
+  rank. With `--semantic`, rows carry `similarity`: the cosine (-1 to 1) of the query
+  against that file's best-matching chunk — the same chunk the `lines` range points at.
+  It orders vector evidence within a result set; its absolute range depends on the corpus.
+  Measured: on thousands of notes, unrelated queries ~0.2 and genuine matches ~0.6; on a
+  few dozen notes the ranges compress and can overlap, because even a nonsense query has a
+  moderately near neighbour somewhere. Compare similarities within a result set rather than
+  against a fixed cutoff carried between trees.
+- Lexical `find` returns 0 rows when nothing matches, so it answers "is this in the tree at
+  all". `--semantic` always returns up to `k` rows — nearest-neighbour search has a nearest
+  neighbour for any input — so absence is a lexical question; `similarity` and the snippet
+  are the evidence for judging whether a vector row is a real hit.
+- `sense check` prepares every saved query (catching syntax and unknown-column errors), runs
+  the ones taking no parameters, and prints row counts: a saved query returning 0 rows looks
+  the same as a true empty result until something distinguishes them. For queries that
+  encode invariants (a dead-link list, an unsupported-claims list — rows are violations),
+  `checks: { "<name>": "empty" }` in the config inverts the meaning: `check` fails when the
+  query returns rows, making it usable as a test suite rather than a linter.
 
 ## SQL
 
@@ -87,6 +111,10 @@ sense query "SELECT j.value, COUNT(*) n FROM frontmatter, json_each(frontmatter.
   `snippet(content, -1, '«', '»', '…', 10)`.
 - Select `content.title`/`content.summary` (always exist, empty when absent) rather than
   `f.title`/`f.summary` (discovered columns — error on trees that never declare them).
+- Frontmatter values keep their YAML type: strings are TEXT, whole numbers and booleans are
+  INTEGER (`true` stores as 1, so `WHERE flag = 1` matches and `WHERE flag = 'true'` matches
+  nothing), fractions are REAL, lists and maps are JSON text. `map` prints the observed type
+  per field, and a field showing two types (`integer,text`) has drifted across notes.
 - `has(field, value)`: array membership on JSON-array fields, substring on strings, false on NULL
   — the `includes()` convention. Substring means `has(f.status, 'active')` also matches
   `inactive`; exact scalar match is `f.status = ?`, deliberate substring is `LIKE`, exact array

package/skills/sense-setup/SKILL.md

@@ -31,6 +31,16 @@ installing, configuring features, and the design decisions a tree owner faces.
   Face id or local path), `type` (`static` pure-JS built-in, or `api` for any
   OpenAI-compatible `/v1/embeddings` endpoint — Ollama, LM Studio, llama.cpp,
   hosted), `url`, `key` (env var name). A local `model` path is fully offline.
+- The two types differ in what they match, not only in cost: `static` is a
+  context-free distilled model that handles paraphrase and reworded concepts
+  (a query like "delegating without micromanaging" can reach a note that uses
+  neither word); tight near-synonyms and domain jargon ("heart attack" for
+  "myocardial infarction") are where it misses and an `api` transformer model
+  tends to succeed. The gain concentrates where the searcher's vocabulary
+  differs from the notes' — measured in the retrieval eval (BENCHMARKING.md,
+  "Retrieval quality"): on a vocabulary-gap corpus semantic expansion adds
+  recall; where vocabulary overlaps, BM25 plus link expansion already answers
+  most queries and vectors mostly reorder.
 - Enabling `embed` changes no default `find` result — expansion runs only when
   a query passes `--semantic`. Invoking `--semantic` on a tree without `embed`
   is an error naming the config key.
@@ -46,10 +56,18 @@ do, and every consequence is listed so the choice can be made deliberately.
 
 - **Frontmatter fields.** Columns are discovered per tree — whatever keys notes
   declare become queryable. Consistent fields across notes make SQL filters
-  and named queries possible (`WHERE status = 'active'`); inconsistent fields
-  still work but produce sparse columns that filter less of the tree. Reserved
-  keys (dropped with a warning): `path`, `_mtime`, `_size`, `_rank`, `content`,
-  `links`, `sections`.
+  and named queries possible (`WHERE status = 'active'`). Reserved keys
+  (dropped with a warning): `path`, `_mtime`, `_size`, `_rank`, `content`,
+  `links`, `sections`. Values keep their YAML type: strings TEXT, whole numbers
+  and booleans INTEGER (`true` is 1), fractions REAL, lists and maps JSON text;
+  `map` prints the observed type per field.
+- **What a note omits is also a filter.** A layer that deliberately carries none
+  of the fields the saved views filter on is excluded from all of them without
+  any view naming the layer; only a view filtering solely on a field the layer
+  does share needs an explicit condition (`AND type != 'raw'`). Sparse fields cut
+  both ways: less of the tree filters when you want breadth, and exactly this
+  separation when layers differ in authority — source extractions vs.
+  conclusions, generated output vs. notes.
 - **Dates.** `datetime()` comparisons work for dates written as ISO 8601 —
   the only format it parses. A tree that mixes date formats can store them,
   but can't compare them in SQL.