sensemaking 0.2.1 → 0.3.1

package/dist/esm/graph.js

@@ -0,0 +1,79 @@
+// Link-graph math: PageRank at reconcile, personalized PageRank at query time. Pure JS, no deps.
+// Power iteration. `personal` biases both teleport and dangling mass toward seed nodes;
+// without it this is plain PageRank (uniform base). Returns scores summing to ~1.
+export function pagerank(nodes, edges, opts = {}) {
+    var _opts_damping, _opts_iterations;
+    const damping = (_opts_damping = opts.damping) !== null && _opts_damping !== void 0 ? _opts_damping : 0.85;
+    const iterations = (_opts_iterations = opts.iterations) !== null && _opts_iterations !== void 0 ? _opts_iterations : 30;
+    const n = nodes.length;
+    if (n === 0) return new Map();
+    const index = new Map(nodes.map((node, i)=>[
+            node,
+            i
+        ]));
+    const out = nodes.map(()=>[]);
+    for (const [src, dst] of edges){
+        const si = index.get(src);
+        const di = index.get(dst);
+        if (si !== undefined && di !== undefined) out[si].push(di);
+    }
+    let base;
+    if (opts.personal) {
+        base = nodes.map(()=>0);
+        let total = 0;
+        for (const [node, weight] of opts.personal){
+            const i = index.get(node);
+            if (i !== undefined && weight > 0) {
+                base[i] = weight;
+                total += weight;
+            }
+        }
+        if (total === 0) return new Map(nodes.map((node)=>[
+                node,
+                0
+            ]));
+        base = base.map((b)=>b / total);
+    } else {
+        base = nodes.map(()=>1 / n);
+    }
+    let rank = [
+        ...base
+    ];
+    for(let iter = 0; iter < iterations; iter++){
+        const next = base.map((b)=>(1 - damping) * b);
+        let dangling = 0;
+        for(let i = 0; i < n; i++){
+            if (out[i].length === 0) {
+                dangling += rank[i];
+                continue;
+            }
+            const share = damping * rank[i] / out[i].length;
+            for (const j of out[i])next[j] += share;
+        }
+        for(let i = 0; i < n; i++)next[i] += damping * dangling * base[i];
+        rank = next;
+    }
+    return new Map(nodes.map((node, i)=>[
+            node,
+            rank[i]
+        ]));
+}
+// Connective expansion: walk the graph (undirected -- backlinks relate as much as links)
+// from a seed result set. Answers "what is linked to the notes that matched".
+export function personalizedRank(nodes, edges, seeds) {
+    const undirected = [];
+    for (const [src, dst] of edges){
+        undirected.push([
+            src,
+            dst
+        ], [
+            dst,
+            src
+        ]);
+    }
+    return pagerank(nodes, undirected, {
+        damping: 0.7,
+        iterations: 15,
+        personal: seeds
+    });
+}

package/dist/esm/graph.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/graph.ts"],"sourcesContent":["// Link-graph math: PageRank at reconcile, personalized PageRank at query time. Pure JS, no deps.\n\nexport type Edge = [src: string, dst: string];\n\n// Power iteration. `personal` biases both teleport and dangling mass toward seed nodes;\n// without it this is plain PageRank (uniform base). Returns scores summing to ~1.\nexport function pagerank(nodes: string[], edges: Edge[], opts: { damping?: number; iterations?: number; personal?: Map<string, number> } = {}): Map<string, number> {\n  const damping = opts.damping ?? 0.85;\n  const iterations = opts.iterations ?? 30;\n  const n = nodes.length;\n  if (n === 0) return new Map();\n\n  const index = new Map(nodes.map((node, i) => [node, i]));\n  const out: number[][] = nodes.map(() => []);\n  for (const [src, dst] of edges) {\n    const si = index.get(src);\n    const di = index.get(dst);\n    if (si !== undefined && di !== undefined) out[si].push(di);\n  }\n\n  let base: number[];\n  if (opts.personal) {\n    base = nodes.map(() => 0);\n    let total = 0;\n    for (const [node, weight] of opts.personal) {\n      const i = index.get(node);\n      if (i !== undefined && weight > 0) {\n        base[i] = weight;\n        total += weight;\n      }\n    }\n    if (total === 0) return new Map(nodes.map((node) => [node, 0]));\n    base = base.map((b) => b / total);\n  } else {\n    base = nodes.map(() => 1 / n);\n  }\n\n  let rank = [...base];\n  for (let iter = 0; iter < iterations; iter++) {\n    const next = base.map((b) => (1 - damping) * b);\n    let dangling = 0;\n    for (let i = 0; i < n; i++) {\n      if (out[i].length === 0) {\n        dangling += rank[i];\n        continue;\n      }\n      const share = (damping * rank[i]) / out[i].length;\n      for (const j of out[i]) next[j] += share;\n    }\n    for (let i = 0; i < n; i++) next[i] += damping * dangling * base[i];\n    rank = next;\n  }\n  return new Map(nodes.map((node, i) => [node, rank[i]]));\n}\n\n// Connective expansion: walk the graph (undirected -- backlinks relate as much as links)\n// from a seed result set. Answers \"what is linked to the notes that matched\".\nexport function personalizedRank(nodes: string[], edges: Edge[], seeds: Map<string, number>): Map<string, number> {\n  const undirected: Edge[] = [];\n  for (const [src, dst] of edges) {\n    undirected.push([src, dst], [dst, src]);\n  }\n  return pagerank(nodes, undirected, { damping: 0.7, iterations: 15, personal: seeds });\n}\n"],"names":["pagerank","nodes","edges","opts","damping","iterations","n","length","Map","index","map","node","i","out","src","dst","si","get","di","undefined","push","base","personal","total","weight","b","rank","iter","next","dangling","share","j","personalizedRank","seeds","undirected"],"mappings":"AAAA,iGAAiG;AAIjG,wFAAwF;AACxF,kFAAkF;AAClF,OAAO,SAASA,SAASC,KAAe,EAAEC,KAAa,EAAEC,OAAkF,CAAC,CAAC;QAC3HA,eACGA;IADnB,MAAMC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAgB;IAChC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmB;IACtC,MAAMG,IAAIL,MAAMM,MAAM;IACtB,IAAID,MAAM,GAAG,OAAO,IAAIE;IAExB,MAAMC,QAAQ,IAAID,IAAIP,MAAMS,GAAG,CAAC,CAACC,MAAMC,IAAM;YAACD;YAAMC;SAAE;IACtD,MAAMC,MAAkBZ,MAAMS,GAAG,CAAC,IAAM,EAAE;IAC1C,KAAK,MAAM,CAACI,KAAKC,IAAI,IAAIb,MAAO;QAC9B,MAAMc,KAAKP,MAAMQ,GAAG,CAACH;QACrB,MAAMI,KAAKT,MAAMQ,GAAG,CAACF;QACrB,IAAIC,OAAOG,aAAaD,OAAOC,WAAWN,GAAG,CAACG,GAAG,CAACI,IAAI,CAACF;IACzD;IAEA,IAAIG;IACJ,IAAIlB,KAAKmB,QAAQ,EAAE;QACjBD,OAAOpB,MAAMS,GAAG,CAAC,IAAM;QACvB,IAAIa,QAAQ;QACZ,KAAK,MAAM,CAACZ,MAAMa,OAAO,IAAIrB,KAAKmB,QAAQ,CAAE;YAC1C,MAAMV,IAAIH,MAAMQ,GAAG,CAACN;YACpB,IAAIC,MAAMO,aAAaK,SAAS,GAAG;gBACjCH,IAAI,CAACT,EAAE,GAAGY;gBACVD,SAASC;YACX;QACF;QACA,IAAID,UAAU,GAAG,OAAO,IAAIf,IAAIP,MAAMS,GAAG,CAAC,CAACC,OAAS;gBAACA;gBAAM;aAAE;QAC7DU,OAAOA,KAAKX,GAAG,CAAC,CAACe,IAAMA,IAAIF;IAC7B,OAAO;QACLF,OAAOpB,MAAMS,GAAG,CAAC,IAAM,IAAIJ;IAC7B;IAEA,IAAIoB,OAAO;WAAIL;KAAK;IACpB,IAAK,IAAIM,OAAO,GAAGA,OAAOtB,YAAYsB,OAAQ;QAC5C,MAAMC,OAAOP,KAAKX,GAAG,CAAC,CAACe,IAAM,AAAC,CAAA,IAAIrB,OAAM,IAAKqB;QAC7C,IAAII,WAAW;QACf,IAAK,IAAIjB,IAAI,GAAGA,IAAIN,GAAGM,IAAK;YAC1B,IAAIC,GAAG,CAACD,EAAE,CAACL,MAAM,KAAK,GAAG;gBACvBsB,YAAYH,IAAI,CAACd,EAAE;gBACnB;YACF;YACA,MAAMkB,QAAQ,AAAC1B,UAAUsB,IAAI,CAACd,EAAE,GAAIC,GAAG,CAACD,EAAE,CAACL,MAAM;YACjD,KAAK,MAAMwB,KAAKlB,GAAG,CAACD,EAAE,CAAEgB,IAAI,CAACG,EAAE,IAAID;QACrC;QACA,IAAK,IAAIlB,IAAI,GAAGA,IAAIN,GAAGM,IAAKgB,IAAI,CAAChB,EAAE,IAAIR,UAAUyB,WAAWR,IAAI,CAACT,EAAE;QACnEc,OAAOE;IACT;IACA,OAAO,IAAIpB,IAAIP,MAAMS,GAAG,CAAC,CAACC,MAAMC,IAAM;YAACD;YAAMe,IAAI,CAACd,EAAE;SAAC;AACvD;AAEA,yFAAyF;AACzF,8EAA8E;AAC9E,OAAO,SAASoB,iBAAiB/B,KAAe,EAAEC,KAAa,EAAE+B,KAA0B;IACzF,MAAMC,aAAqB,EAAE;IAC7B,KAAK,MAAM,CAACpB,KAAKC,IAAI,IAAIb,MAAO;QAC9BgC,WAAWd,IAAI,CAAC;YAACN;YAAKC;SAAI,EAAE;YAACA;YAAKD;SAAI;IACxC;IACA,OAAOd,SAASC,OAAOiC,YAAY;QAAE9B,SAAS;QAAKC,YAAY;QAAIiB,UAAUW;IAAM;AACrF"}

package/dist/esm/index.d.ts

@@ -1,12 +1,19 @@
-export type { Config, ResolvedConfig } from './config.js';
-export { CONFIG_FILENAME, findConfigPath, initConfig, loadConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
+export type { Config, FeatureName, ResolvedConfig } from './config.js';
+export { CONFIG_FILENAME, enabledFeatures, FEATURE_NAMES, featureEnabled, findConfigPath, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export type { OpenResult } from './db.js';
 export { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.js';
 export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
+export { activeFeatures, FEATURES, linkEdges } from './features/index.js';
+export type { Section } from './features/sections.js';
+export type { Feature } from './features/types.js';
+export type { Edge } from './graph.js';
+export { pagerank, personalizedRank } from './graph.js';
 export type { Row } from './output.js';
 export { printRows } from './output.js';
 export type { FileStat, ParsedDoc } from './scan.js';
 export { listFiles, parseFile } from './scan.js';
+export type { FindOptions, Peek, VaultMap } from './verbs.js';
+export { find, peek, vaultMap } from './verbs.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js

@@ -1,7 +1,10 @@
 // Public library API.
-export { CONFIG_FILENAME, findConfigPath, initConfig, loadConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
+export { CONFIG_FILENAME, enabledFeatures, FEATURE_NAMES, featureEnabled, findConfigPath, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.js';
 export { SenseError } from './errors.js';
+export { activeFeatures, FEATURES, linkEdges } from './features/index.js';
+export { pagerank, personalizedRank } from './graph.js';
 export { printRows } from './output.js';
 export { listFiles, parseFile } from './scan.js';
+export { find, peek, vaultMap } from './verbs.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API.\n\nexport type { Config, ResolvedConfig } from './config.ts';\nexport { CONFIG_FILENAME, findConfigPath, initConfig, loadConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\n\nexport type { OpenResult } from './db.ts';\nexport { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.ts';\n\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\n\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { FileStat, ParsedDoc } from './scan.ts';\nexport { listFiles, parseFile } from './scan.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CONFIG_FILENAME","findConfigPath","initConfig","loadConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","DB_FILENAME","docCount","getMeta","open","rebuild","reconcile","setMeta","SenseError","printRows","listFiles","parseFile","runWatch"],"mappings":"AAAA,sBAAsB;AAGtB,SAASA,eAAe,EAAEC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAG3H,SAASC,WAAW,EAAEC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,UAAU;AAG5F,SAASC,UAAU,QAAQ,cAAc;AAGzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAGjD,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API.\n\nexport type { Config, FeatureName, ResolvedConfig } from './config.ts';\nexport { CONFIG_FILENAME, enabledFeatures, FEATURE_NAMES, featureEnabled, findConfigPath, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\n\nexport type { OpenResult } from './db.ts';\nexport { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport { activeFeatures, FEATURES, linkEdges } from './features/index.ts';\nexport type { Section } from './features/sections.ts';\nexport type { Feature } from './features/types.ts';\n\nexport type { Edge } from './graph.ts';\nexport { pagerank, personalizedRank } from './graph.ts';\n\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { FileStat, ParsedDoc } from './scan.ts';\nexport { listFiles, parseFile } from './scan.ts';\n\nexport type { FindOptions, Peek, VaultMap } from './verbs.ts';\nexport { find, peek, vaultMap } from './verbs.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CONFIG_FILENAME","enabledFeatures","FEATURE_NAMES","featureEnabled","findConfigPath","initConfig","loadConfig","migrateConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","DB_FILENAME","docCount","getMeta","open","rebuild","reconcile","setMeta","SenseError","activeFeatures","FEATURES","linkEdges","pagerank","personalizedRank","printRows","listFiles","parseFile","find","peek","vaultMap","runWatch"],"mappings":"AAAA,sBAAsB;AAGtB,SAASA,eAAe,EAAEC,eAAe,EAAEC,aAAa,EAAEC,cAAc,EAAEC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAG1L,SAASC,WAAW,EAAEC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,UAAU;AAE5F,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,EAAEC,QAAQ,EAAEC,SAAS,QAAQ,sBAAsB;AAK1E,SAASC,QAAQ,EAAEC,gBAAgB,QAAQ,aAAa;AAGxD,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAGjD,SAASC,IAAI,EAAEC,IAAI,EAAEC,QAAQ,QAAQ,aAAa;AAGlD,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/output.d.ts

@@ -1,2 +1,24 @@
 export type Row = Record<string, unknown>;
 export declare function printRows(rows: Row[], format: 'table' | 'json'): void;
+export declare function renderMap(result: {
+    docs: {
+        count: number;
+        bytes: number;
+    };
+    fields: Row[];
+    fieldsTotal: number;
+    hubs: Row[];
+    recent: Row[];
+}): string;
+export declare function renderPeek(result: {
+    path: string;
+    tokens: number;
+    frontmatter: Row;
+    sections: Row[];
+    outbound: string[];
+    backlinks: string[];
+    unresolved: string[];
+    outboundTotal: number;
+    backlinksTotal: number;
+    unresolvedTotal: number;
+}): string;

package/dist/esm/output.js

@@ -25,3 +25,32 @@ function render(rows, format) {
         ...cells.map(formatRow)
     ].join('\n');
 }
+// Text renderers for the layer verbs; cli.ts prints what these return.
+export function renderMap(result) {
+    const parts = [
+        `docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)\n`,
+        render(result.fields, 'table')
+    ];
+    if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);
+    if (result.hubs.length > 0) parts.push('\nhubs (by link rank):', render(result.hubs, 'table'));
+    parts.push('\nrecent:', render(result.recent, 'table'));
+    return parts.join('\n');
+}
+export function renderPeek(result) {
+    const lines = [
+        `${result.path}  (~${result.tokens} tokens)`
+    ];
+    for (const [key, value] of Object.entries(result.frontmatter))lines.push(`  ${key}: ${value}`);
+    if (result.sections.length > 0) {
+        lines.push('', 'sections:');
+        for (const s of result.sections)lines.push(`  ${'#'.repeat(s.level)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);
+    }
+    const linkLine = (label, shown, total)=>{
+        if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);
+    };
+    if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');
+    linkLine('links out', result.outbound, result.outboundTotal);
+    linkLine('unresolved', result.unresolved, result.unresolvedTotal);
+    linkLine('backlinks', result.backlinks, result.backlinksTotal);
+    return lines.join('\n');
+}

package/dist/esm/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = render(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\nfunction render(rows: Row[], format: 'table' | 'json'): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const widths = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","render","console","log","length","warn","Math","round","JSON","stringify","columns","Object","keys","cells","map","row","col","widths","i","max","formatRow","values","padEnd","join","w","repeat"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,OAAOH,MAAMC;IAC9BG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,SAASH,OAAOH,IAAW,EAAEC,MAAwB;IACnD,IAAIA,WAAW,QAAQ,OAAOS,KAAKC,SAAS,CAACX,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMM,UAAUC,OAAOC,IAAI,CAACd,IAAI,CAAC,EAAE;IACnC,MAAMe,QAAQf,KAAKgB,GAAG,CAAC,CAACC,MAAQL,QAAQI,GAAG,CAAC,CAACE,MAAQvB,KAAKsB,GAAG,CAACC,IAAI;IAClE,MAAMC,SAASP,QAAQI,GAAG,CAAC,CAACE,KAAKE,IAAMZ,KAAKa,GAAG,CAACH,IAAIZ,MAAM,KAAKS,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAACd,MAAM;IAE/F,MAAMgB,YAAY,CAACC,SAAqBA,OAAOP,GAAG,CAAC,CAACpB,OAAOwB,IAAMxB,MAAM4B,MAAM,CAACL,MAAM,CAACC,EAAE,GAAGK,IAAI,CAAC;IAE/F,OAAO;QAACH,UAAUV;QAAUO,OAAOH,GAAG,CAAC,CAACU,IAAM,IAAIC,MAAM,CAACD,IAAID,IAAI,CAAC;WAAUV,MAAMC,GAAG,CAACM;KAAW,CAACG,IAAI,CAAC;AACzG"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = render(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\nfunction render(rows: Row[], format: 'table' | 'json'): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const widths = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)\\n`, render(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', render(result.hubs, 'table'));\n  parts.push('\\nrecent:', render(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","render","console","log","length","warn","Math","round","JSON","stringify","columns","Object","keys","cells","map","row","col","widths","i","max","formatRow","values","padEnd","join","w","repeat","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","linkLine","label","shown","total","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,OAAOH,MAAMC;IAC9BG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,SAASH,OAAOH,IAAW,EAAEC,MAAwB;IACnD,IAAIA,WAAW,QAAQ,OAAOS,KAAKC,SAAS,CAACX,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMM,UAAUC,OAAOC,IAAI,CAACd,IAAI,CAAC,EAAE;IACnC,MAAMe,QAAQf,KAAKgB,GAAG,CAAC,CAACC,MAAQL,QAAQI,GAAG,CAAC,CAACE,MAAQvB,KAAKsB,GAAG,CAACC,IAAI;IAClE,MAAMC,SAASP,QAAQI,GAAG,CAAC,CAACE,KAAKE,IAAMZ,KAAKa,GAAG,CAACH,IAAIZ,MAAM,KAAKS,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAACd,MAAM;IAE/F,MAAMgB,YAAY,CAACC,SAAqBA,OAAOP,GAAG,CAAC,CAACpB,OAAOwB,IAAMxB,MAAM4B,MAAM,CAACL,MAAM,CAACC,EAAE,GAAGK,IAAI,CAAC;IAE/F,OAAO;QAACH,UAAUV;QAAUO,OAAOH,GAAG,CAAC,CAACU,IAAM,IAAIC,MAAM,CAACD,IAAID,IAAI,CAAC;WAAUV,MAAMC,GAAG,CAACM;KAAW,CAACG,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,OAAO,SAASG,UAAUC,MAAkH;IAC1I,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAExB,KAAKC,KAAK,CAACoB,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,MAAM,CAAC;QAAE9B,OAAO0B,OAAOK,MAAM,EAAE;KAAS;IAC3H,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC5B,MAAM,EAAEwB,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC5B,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIuB,OAAOQ,IAAI,CAAC/B,MAAM,GAAG,GAAGwB,MAAMM,IAAI,CAAC,0BAA0BjC,OAAO0B,OAAOQ,IAAI,EAAE;IACrFP,MAAMM,IAAI,CAAC,aAAajC,OAAO0B,OAAOS,MAAM,EAAE;IAC9C,OAAOR,MAAML,IAAI,CAAC;AACpB;AAEA,OAAO,SAASc,WAAWV,MAAkN;IAC3O,MAAMW,QAAQ;QAAC,GAAGX,OAAOY,IAAI,CAAC,IAAI,EAAEZ,OAAOa,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAK/C,MAAM,IAAIiB,OAAO+B,OAAO,CAACf,OAAOgB,WAAW,EAAGL,MAAMJ,IAAI,CAAC,CAAC,EAAE,EAAEO,IAAI,EAAE,EAAE/C,OAAO;IAC9F,IAAIiC,OAAOiB,QAAQ,CAACxC,MAAM,GAAG,GAAG;QAC9BkC,MAAMJ,IAAI,CAAC,IAAI;QACf,KAAK,MAAMW,KAAKlB,OAAOiB,QAAQ,CAAEN,MAAMJ,IAAI,CAAC,CAAC,EAAE,EAAE,IAAIT,MAAM,CAACoB,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;IAChJ;IACA,MAAMU,WAAW,CAACC,OAAeC,OAAiBC;QAChD,IAAIA,QAAQ,GAAGf,MAAMJ,IAAI,CAAC,GAAGiB,MAAM,EAAE,EAAEE,MAAM,GAAG,EAAED,MAAM7B,IAAI,CAAC,QAAQ8B,QAAQD,MAAMhD,MAAM,GAAG,CAAC,GAAG,EAAEiD,QAAQD,MAAMhD,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAIuB,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGlB,MAAMJ,IAAI,CAAC;IAC1FgB,SAAS,aAAavB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;IAC3DJ,SAAS,cAAcvB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;IAChEL,SAAS,aAAavB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC7D,OAAOlB,MAAMf,IAAI,CAAC;AACpB"}

package/dist/esm/scan.d.ts

@@ -1,4 +1,5 @@
 import type { Config } from './config.js';
+import type { Feature } from './features/types.js';
 export interface FileStat {
     relPath: string;
     absPath: string;
@@ -16,8 +17,9 @@ export interface ParsedDoc {
         summary: string;
         text: string;
     };
+    extracted: Record<string, unknown>;
 }
-export declare function parseFile(file: FileStat): {
+export declare function parseFile(file: FileStat, extractors?: Feature[]): {
     doc: ParsedDoc;
     warnings: string[];
 };

package/dist/esm/scan.js

@@ -1,15 +1,18 @@
 import { readFileSync, statSync } from 'node:fs';
 import { join } from 'node:path';
 import fastGlob from 'fast-glob';
-import matter from 'gray-matter';
 import removeMarkdown from 'remove-markdown';
+import { parseDocument } from 'yaml';
 // Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
-// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the `content` FTS5 table.
+// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).
 const RESERVED_COLUMNS = new Set([
     'path',
     '_mtime',
     '_size',
-    'content'
+    '_rank',
+    'content',
+    'links',
+    'sections'
 ]);
 function normalizeText(value) {
     if (value === null || value === undefined) return '';
@@ -40,15 +43,54 @@ export function listFiles(cfg, baseDir) {
 }
 function mapValue(value) {
     if (value === null || value === undefined) return null;
-    if (value instanceof Date) return value.toISOString();
     if (typeof value === 'boolean') return value ? 1 : 0;
     if (typeof value === 'string' || typeof value === 'number') return value;
     return JSON.stringify(value);
 }
-export function parseFile(file) {
+// The delimiter split is all this package used gray-matter for.
+function splitFrontmatter(raw) {
+    const open = raw.match(/^---\r?\n/);
+    if (!open) return {
+        fm: null,
+        body: raw
+    };
+    const rest = raw.slice(open[0].length);
+    const close = rest.match(/^---\r?(\n|$)/m);
+    if (!close || close.index === undefined) return {
+        fm: null,
+        body: raw
+    };
+    return {
+        fm: rest.slice(0, close.index),
+        body: rest.slice(close.index + close[0].length)
+    };
+}
+// Lenient by design: parseDocument collects syntax errors as data and still yields values,
+// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.
+function parseFrontmatter(relPath, fm, warnings) {
+    const doc = parseDocument(fm);
+    if (doc.errors.length > 0) {
+        warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\n')[0]}); parsed leniently`);
+    }
+    let data;
+    try {
+        data = doc.toJS();
+    } catch (err) {
+        warnings.push(`warning: ${relPath} has unparseable frontmatter (${err.message.split('\n')[0]}); indexing without it`);
+        return {};
+    }
+    if (data === null || data === undefined) return {};
+    if (typeof data !== 'object' || Array.isArray(data)) {
+        warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);
+        return {};
+    }
+    return data;
+}
+export function parseFile(file, extractors = []) {
     const raw = readFileSync(file.absPath, 'utf8');
-    const { data, content } = matter(raw);
     const warnings = [];
+    const { fm, body: content } = splitFrontmatter(raw);
+    const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);
     const mapped = {};
     for (const key of Object.keys(data)){
         if (RESERVED_COLUMNS.has(key)) {
@@ -69,7 +111,14 @@ export function parseFile(file) {
                 title: normalizeText(data.title),
                 summary: normalizeText(data.summary),
                 text: stripText(content)
-            }
+            },
+            extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
+                var _f_extract;
+                return [
+                    f.name,
+                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content)
+                ];
+            }))
         },
         warnings
     };

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport matter from 'gray-matter';\nimport removeMarkdown from 'remove-markdown';\nimport type { Config } from './config.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the `content` FTS5 table.\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', 'content']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n}\n\nfunction mapValue(value: unknown): string | number | null {\n  if (value === null || value === undefined) return null;\n  if (value instanceof Date) return value.toISOString();\n  if (typeof value === 'boolean') return value ? 1 : 0;\n  if (typeof value === 'string' || typeof value === 'number') return value;\n  return JSON.stringify(value);\n}\n\nexport function parseFile(file: FileStat): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const { data, content } = matter(raw);\n\n  const warnings: string[] = [];\n  const mapped: Record<string, string | number | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search: {\n        // title/summary are plain YAML strings -- whitespace-collapse only;\n        // the prose gets the full markdown strip.\n        title: normalizeText(data.title),\n        summary: normalizeText(data.summary),\n        text: stripText(content),\n      },\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","matter","removeMarkdown","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","Date","toISOString","JSON","stringify","parseFile","file","raw","data","content","warnings","mapped","key","Object","keys","has","push","doc","search","title","summary","text"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,YAAY,cAAc;AACjC,OAAOC,oBAAoB,kBAAkB;AAG7C,6EAA6E;AAE7E,qHAAqH;AACrH,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAU;AAEvE,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBX,eAAeU;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AASA,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWlB,SAASmB,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAU3B,KAAKkB,SAASQ;QAC9B,MAAME,KAAK7B,SAAS4B;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAWA,SAASC,SAASxB,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAID,iBAAiByB,MAAM,OAAOzB,MAAM0B,WAAW;IACnD,IAAI,OAAO1B,UAAU,WAAW,OAAOA,QAAQ,IAAI;IACnD,IAAI,OAAOA,UAAU,YAAY,OAAOA,UAAU,UAAU,OAAOA;IACnE,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,OAAO,SAAS6B,UAAUC,IAAc;IACtC,MAAMC,MAAMxC,aAAauC,KAAKV,OAAO,EAAE;IACvC,MAAM,EAAEY,IAAI,EAAEC,OAAO,EAAE,GAAGtC,OAAOoC;IAEjC,MAAMG,WAAqB,EAAE;IAC7B,MAAMC,SAAiD,CAAC;IAExD,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAInC,iBAAiB0C,GAAG,CAACH,MAAM;YAC7BF,SAASM,IAAI,CAAC,CAAC,SAAS,EAAEV,KAAKX,OAAO,CAAC,8BAA8B,EAAEiB,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGZ,SAASQ,IAAI,CAACI,IAAI;IAClC;IAEA,OAAO;QACLK,KAAK;YACHtB,SAASW,KAAKX,OAAO;YACrBG,SAASQ,KAAKR,OAAO;YACrBC,MAAMO,KAAKP,IAAI;YACfS,MAAMG;YACNO,QAAQ;gBACN,oEAAoE;gBACpE,0CAA0C;gBAC1CC,OAAO5C,cAAciC,KAAKW,KAAK;gBAC/BC,SAAS7C,cAAciC,KAAKY,OAAO;gBACnCC,MAAMxC,UAAU4B;YAClB;QACF;QACAC;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nfunction mapValue(value: unknown): string | number | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return value ? 1 : 0;\n  if (typeof value === 'string' || typeof value === 'number') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search: {\n        // title/summary are plain YAML strings -- whitespace-collapse only;\n        // the prose gets the full markdown strip.\n        title: normalizeText(data.title),\n        summary: normalizeText(data.summary),\n        text: stripText(content),\n      },\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","keys","has","search","title","summary","text","extracted","fromEntries","filter","f","extract","name"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAIrC,6EAA6E;AAE7E,8IAA8I;AAC9I,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAErG,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBZ,eAAeW;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AASA,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWlB,SAASmB,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAU3B,KAAKkB,SAASQ;QAC9B,MAAME,KAAK7B,SAAS4B;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAaA,SAASC,SAASxB,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOA,QAAQ,IAAI;IACnD,IAAI,OAAOA,UAAU,YAAY,OAAOA,UAAU,UAAU,OAAOA;IACnE,OAAOyB,KAAKC,SAAS,CAAC1B;AACxB;AAEA,gEAAgE;AAChE,SAAS2B,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKpC,WAAW,OAAO;QAAE8B,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBnB,OAAe,EAAEY,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM5C,cAAcmC;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEvB,QAAQ,iBAAiB,EAAEqB,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEvB,QAAQ,8BAA8B,EAAE,AAAC4B,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS5C,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO4C,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEvB,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAO0B;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMrC,aAAa4D,KAAK/B,OAAO,EAAE;IACvC,MAAMmB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAKhC,OAAO,EAAEY,IAAIQ;IACnE,MAAMe,SAAiD,CAAC;IAExD,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACZ,MAAO;QACnC,IAAIhD,iBAAiB6D,GAAG,CAACH,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAKhC,OAAO,CAAC,8BAA8B,EAAEoC,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAG/B,SAASqB,IAAI,CAACU,IAAI;IAClC;IAEA,OAAO;QACLf,KAAK;YACHrB,SAASgC,KAAKhC,OAAO;YACrBG,SAAS6B,KAAK7B,OAAO;YACrBC,MAAM4B,KAAK5B,IAAI;YACfsB,MAAMS;YACNK,QAAQ;gBACN,oEAAoE;gBACpE,0CAA0C;gBAC1CC,OAAO7D,cAAc8C,KAAKe,KAAK;gBAC/BC,SAAS9D,cAAc8C,KAAKgB,OAAO;gBACnCC,MAAMzD,UAAUgD;YAClB;YACAU,WAAWP,OAAOQ,WAAW,CAACZ,WAAWa,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEjD,GAAG,CAAC,CAACgD;oBAAeA;uBAAT;oBAACA,EAAEE,IAAI;qBAAEF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYtC,KAAKyB;iBAAS;;QAClH;QACAd;IACF;AACF"}

package/dist/esm/verbs.d.ts

@@ -0,0 +1,32 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Config } from './config.js';
+import type { Row } from './output.js';
+export interface FindOptions {
+    k?: number;
+    where?: string;
+}
+export declare function find(db: DatabaseSync, cfg: Config, terms: string, opts?: FindOptions): Row[];
+export interface VaultMap {
+    docs: {
+        count: number;
+        bytes: number;
+    };
+    fields: Row[];
+    fieldsTotal: number;
+    hubs: Row[];
+    recent: Row[];
+}
+export declare function vaultMap(db: DatabaseSync, cfg: Config): VaultMap;
+export interface Peek {
+    path: string;
+    tokens: number;
+    frontmatter: Row;
+    sections: Row[];
+    outbound: string[];
+    backlinks: string[];
+    unresolved: string[];
+    outboundTotal: number;
+    backlinksTotal: number;
+    unresolvedTotal: number;
+}
+export declare function peek(db: DatabaseSync, cfg: Config, pathArg: string): Peek;

package/dist/esm/verbs.js

@@ -0,0 +1,135 @@
+import posix from 'node:path/posix';
+import { featureEnabled } from './config.js';
+import { SenseError } from './errors.js';
+import { linkEdges } from './features/index.js';
+import { personalizedRank } from './graph.js';
+// The three layer verbs: vaultMap (orient), find (locate), peek (structure).
+// Each returns data; cli.ts renders. All of them degrade when a feature is off.
+const WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0)';
+const RRF_K = 60;
+// Layer 1: BM25 + link-graph expansion, fused by reciprocal rank. `via` says which
+// signal produced each row so the agent knows what evidence it is trusting.
+export function find(db, cfg, terms, opts = {}) {
+    var _opts_k, _hits_get;
+    const k = (_opts_k = opts.k) !== null && _opts_k !== void 0 ? _opts_k : 10;
+    const fetch = Math.max(k * 3, 30);
+    // Terms pass verbatim to FTS5 MATCH: bare words AND-join, operators are the caller's.
+    // Invalid syntax propagates as an error, zero matches return zero -- no silent rewrites.
+    const matchSql = `SELECT content.path AS path, snippet(content, -1, '«', '»', '…', 10) AS hit FROM content WHERE content MATCH ? ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;
+    const matchRows = db.prepare(matchSql).all(terms);
+    const hits = new Map(matchRows.map((r)=>[
+            r.path,
+            r.hit
+        ]));
+    const candidates = new Map();
+    matchRows.forEach((r, i)=>{
+        candidates.set(r.path, {
+            score: 1 / (RRF_K + i),
+            via: 'match'
+        });
+    });
+    if (featureEnabled(cfg, 'links') && matchRows.length > 0) {
+        const nodes = db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path);
+        const seeds = new Map(matchRows.map((r, i)=>[
+                r.path,
+                1 / (i + 1)
+            ]));
+        const ranked = [
+            ...personalizedRank(nodes, linkEdges(db), seeds)
+        ].filter(([, score])=>score > 1e-9).sort((a, b)=>b[1] - a[1]).slice(0, fetch);
+        ranked.forEach(([path], i)=>{
+            const existing = candidates.get(path);
+            if (existing) {
+                existing.score += 1 / (RRF_K + i);
+                existing.via = 'match+link';
+            } else {
+                candidates.set(path, {
+                    score: 1 / (RRF_K + i),
+                    via: 'link'
+                });
+            }
+        });
+    }
+    db.exec('CREATE TEMP TABLE IF NOT EXISTS _find ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT)');
+    db.exec('DELETE FROM _find');
+    const insert = db.prepare('INSERT INTO _find ("path", score, via, hit) VALUES (?, ?, ?, ?)');
+    for (const [path, c] of candidates)insert.run(path, c.score, c.via, (_hits_get = hits.get(path)) !== null && _hits_get !== void 0 ? _hits_get : null);
+    const where = opts.where ? `WHERE ${opts.where}` : '';
+    return db.prepare(`SELECT f."path" AS path, content.title, content.summary, _find.hit, _find.via, round(_find.score, 4) AS score
+       FROM _find JOIN frontmatter f ON f."path" = _find."path" JOIN content ON content.path = _find."path"
+       ${where} ORDER BY _find.score DESC LIMIT ?`).all(k);
+}
+const INTERNAL_COLUMNS = new Set([
+    'path',
+    '_mtime',
+    '_size',
+    '_rank'
+]);
+// Layer 0: what is this vault. Fixed-size output regardless of vault size.
+export function vaultMap(db, cfg) {
+    const docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM("_size"), 0) AS bytes FROM frontmatter').get();
+    const columns = db.prepare('PRAGMA table_info(frontmatter)').all().map((r)=>r.name).filter((name)=>!INTERNAL_COLUMNS.has(name));
+    const allFields = columns.map((name)=>{
+        const { n } = db.prepare(`SELECT COUNT("${name.split('"').join('""')}") AS n FROM frontmatter`).get();
+        return {
+            field: name,
+            coverage: n
+        };
+    }).sort((a, b)=>b.coverage - a.coverage);
+    const fields = allFields.slice(0, 20);
+    const hubs = featureEnabled(cfg, 'rank') ? db.prepare(`SELECT f."path" AS path, round(f."_rank" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f."path" WHERE f."_rank" IS NOT NULL ORDER BY f."_rank" DESC LIMIT 8`).all() : [];
+    const recent = db.prepare(`SELECT "path", datetime("_mtime" / 1000, 'unixepoch') AS modified FROM frontmatter ORDER BY "_mtime" DESC LIMIT 5`).all();
+    return {
+        docs,
+        fields,
+        fieldsTotal: allFields.length,
+        hubs,
+        recent
+    };
+}
+const PEEK_LINK_LIMIT = 20;
+// Layer 2: everything about one note except its prose -- frontmatter, outline with line
+// ranges + token estimates (so the follow-up Read is a range, not the file), links both ways.
+export function peek(db, cfg, pathArg) {
+    var _row__size;
+    const paths = db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path);
+    let path = paths.find((p)=>p === pathArg);
+    if (!path) {
+        const base = posix.basename(pathArg).replace(/\.md$/i, '').toLowerCase();
+        const matches = paths.filter((p)=>posix.basename(p).replace(/\.md$/i, '').toLowerCase() === base);
+        if (matches.length === 1) path = matches[0];
+        else if (matches.length > 1) throw new SenseError('NOTE_AMBIGUOUS', `"${pathArg}" is ambiguous: ${matches.join(', ')}`);
+        else throw new SenseError('NOTE_NOT_FOUND', `no note matches "${pathArg}"`);
+    }
+    const row = db.prepare('SELECT * FROM frontmatter WHERE "path" = ?').get(path);
+    const frontmatter = {};
+    for (const [key, value] of Object.entries(row)){
+        if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;
+    }
+    const sections = featureEnabled(cfg, 'sections') ? db.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE "path" = ? ORDER BY idx').all(path) : [];
+    let outbound = [];
+    let backlinks = [];
+    let unresolved = [];
+    let backlinksTotal = 0;
+    if (featureEnabled(cfg, 'links')) {
+        const out = db.prepare('SELECT target, dst FROM links WHERE src = ? ORDER BY target').all(path);
+        outbound = [
+            ...new Set(out.filter((l)=>l.dst !== null).map((l)=>l.dst))
+        ];
+        unresolved = out.filter((l)=>l.dst === null).map((l)=>l.target);
+        backlinksTotal = db.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ?').get(path).n;
+        backlinks = db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? ORDER BY src LIMIT ?').all(path, PEEK_LINK_LIMIT).map((r)=>r.src);
+    }
+    return {
+        path,
+        tokens: Math.ceil(((_row__size = row._size) !== null && _row__size !== void 0 ? _row__size : 0) / 4),
+        frontmatter,
+        sections,
+        outbound: outbound.slice(0, PEEK_LINK_LIMIT),
+        backlinks,
+        unresolved: unresolved.slice(0, PEEK_LINK_LIMIT),
+        outboundTotal: outbound.length,
+        backlinksTotal,
+        unresolvedTotal: unresolved.length
+    };
+}

package/dist/esm/verbs.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/verbs.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from './config.ts';\nimport { featureEnabled } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { linkEdges } from './features/index.ts';\nimport { personalizedRank } from './graph.ts';\nimport type { Row } from './output.ts';\n\n// The three layer verbs: vaultMap (orient), find (locate), peek (structure).\n// Each returns data; cli.ts renders. All of them degrade when a feature is off.\n\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0)';\nconst RRF_K = 60;\n\nexport interface FindOptions {\n  k?: number;\n  where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n}\n\n// Layer 1: BM25 + link-graph expansion, fused by reciprocal rank. `via` says which\n// signal produced each row so the agent knows what evidence it is trusting.\nexport function find(db: DatabaseSync, cfg: Config, terms: string, opts: FindOptions = {}): Row[] {\n  const k = opts.k ?? 10;\n  const fetch = Math.max(k * 3, 30);\n\n  // Terms pass verbatim to FTS5 MATCH: bare words AND-join, operators are the caller's.\n  // Invalid syntax propagates as an error, zero matches return zero -- no silent rewrites.\n  const matchSql = `SELECT content.path AS path, snippet(content, -1, '«', '»', '…', 10) AS hit FROM content WHERE content MATCH ? ORDER BY ${WEIGHTED_BM25} LIMIT ${fetch}`;\n  const matchRows = db.prepare(matchSql).all(terms) as Array<{ path: string; hit: string }>;\n\n  const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n  const candidates = new Map<string, { score: number; via: string }>();\n  matchRows.forEach((r, i) => {\n    candidates.set(r.path, { score: 1 / (RRF_K + i), via: 'match' });\n  });\n\n  if (featureEnabled(cfg, 'links') && matchRows.length > 0) {\n    const nodes = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n    const seeds = new Map(matchRows.map((r, i) => [r.path, 1 / (i + 1)]));\n    const ranked = [...personalizedRank(nodes, linkEdges(db), seeds)]\n      .filter(([, score]) => score > 1e-9)\n      .sort((a, b) => b[1] - a[1])\n      .slice(0, fetch);\n    ranked.forEach(([path], i) => {\n      const existing = candidates.get(path);\n      if (existing) {\n        existing.score += 1 / (RRF_K + i);\n        existing.via = 'match+link';\n      } else {\n        candidates.set(path, { score: 1 / (RRF_K + i), via: 'link' });\n      }\n    });\n  }\n\n  db.exec('CREATE TEMP TABLE IF NOT EXISTS _find (\"path\" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT)');\n  db.exec('DELETE FROM _find');\n  const insert = db.prepare('INSERT INTO _find (\"path\", score, via, hit) VALUES (?, ?, ?, ?)');\n  for (const [path, c] of candidates) insert.run(path, c.score, c.via, hits.get(path) ?? null);\n\n  const where = opts.where ? `WHERE ${opts.where}` : '';\n  return db\n    .prepare(\n      `SELECT f.\"path\" AS path, content.title, content.summary, _find.hit, _find.via, round(_find.score, 4) AS score\n       FROM _find JOIN frontmatter f ON f.\"path\" = _find.\"path\" JOIN content ON content.path = _find.\"path\"\n       ${where} ORDER BY _find.score DESC LIMIT ?`\n    )\n    .all(k) as Row[];\n}\n\nexport interface VaultMap {\n  docs: { count: number; bytes: number };\n  fields: Row[]; // top 20 by coverage; fieldsTotal carries the real count\n  fieldsTotal: number;\n  hubs: Row[];\n  recent: Row[];\n}\n\nconst INTERNAL_COLUMNS = new Set(['path', '_mtime', '_size', '_rank']);\n\n// Layer 0: what is this vault. Fixed-size output regardless of vault size.\nexport function vaultMap(db: DatabaseSync, cfg: Config): VaultMap {\n  const docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM(\"_size\"), 0) AS bytes FROM frontmatter').get() as { count: number; bytes: number };\n\n  const columns = (db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>).map((r) => r.name).filter((name) => !INTERNAL_COLUMNS.has(name));\n  const allFields = columns\n    .map((name) => {\n      const { n } = db.prepare(`SELECT COUNT(\"${name.split('\"').join('\"\"')}\") AS n FROM frontmatter`).get() as { n: number };\n      return { field: name, coverage: n };\n    })\n    .sort((a, b) => (b.coverage as number) - (a.coverage as number)) as Row[];\n  const fields = allFields.slice(0, 20);\n\n  const hubs = featureEnabled(cfg, 'rank') ? (db.prepare(`SELECT f.\"path\" AS path, round(f.\"_rank\" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f.\"path\" WHERE f.\"_rank\" IS NOT NULL ORDER BY f.\"_rank\" DESC LIMIT 8`).all() as Row[]) : [];\n\n  const recent = db.prepare(`SELECT \"path\", datetime(\"_mtime\" / 1000, 'unixepoch') AS modified FROM frontmatter ORDER BY \"_mtime\" DESC LIMIT 5`).all() as Row[];\n\n  return { docs, fields, fieldsTotal: allFields.length, hubs, recent };\n}\n\nexport interface Peek {\n  path: string;\n  tokens: number;\n  frontmatter: Row;\n  sections: Row[];\n  outbound: string[];\n  backlinks: string[];\n  unresolved: string[];\n  // Totals before truncation: a hub can have thousands of backlinks, and peek's whole\n  // point is bounded output. Query the links table directly for the full list.\n  outboundTotal: number;\n  backlinksTotal: number;\n  unresolvedTotal: number;\n}\n\nconst PEEK_LINK_LIMIT = 20;\n\n// Layer 2: everything about one note except its prose -- frontmatter, outline with line\n// ranges + token estimates (so the follow-up Read is a range, not the file), links both ways.\nexport function peek(db: DatabaseSync, cfg: Config, pathArg: string): Peek {\n  const paths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  let path = paths.find((p) => p === pathArg);\n  if (!path) {\n    const base = posix.basename(pathArg).replace(/\\.md$/i, '').toLowerCase();\n    const matches = paths.filter((p) => posix.basename(p).replace(/\\.md$/i, '').toLowerCase() === base);\n    if (matches.length === 1) path = matches[0];\n    else if (matches.length > 1) throw new SenseError('NOTE_AMBIGUOUS', `\"${pathArg}\" is ambiguous: ${matches.join(', ')}`);\n    else throw new SenseError('NOTE_NOT_FOUND', `no note matches \"${pathArg}\"`);\n  }\n\n  const row = db.prepare('SELECT * FROM frontmatter WHERE \"path\" = ?').get(path) as Row;\n  const frontmatter: Row = {};\n  for (const [key, value] of Object.entries(row)) {\n    if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;\n  }\n\n  const sections = featureEnabled(cfg, 'sections') ? (db.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE \"path\" = ? ORDER BY idx').all(path) as Row[]) : [];\n\n  let outbound: string[] = [];\n  let backlinks: string[] = [];\n  let unresolved: string[] = [];\n  let backlinksTotal = 0;\n  if (featureEnabled(cfg, 'links')) {\n    const out = db.prepare('SELECT target, dst FROM links WHERE src = ? ORDER BY target').all(path) as Array<{ target: string; dst: string | null }>;\n    outbound = [...new Set(out.filter((l) => l.dst !== null).map((l) => l.dst as string))];\n    unresolved = out.filter((l) => l.dst === null).map((l) => l.target);\n    backlinksTotal = (db.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ?').get(path) as { n: number }).n;\n    backlinks = (db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? ORDER BY src LIMIT ?').all(path, PEEK_LINK_LIMIT) as Array<{ src: string }>).map((r) => r.src);\n  }\n\n  return {\n    path,\n    tokens: Math.ceil(((row._size as number) ?? 0) / 4),\n    frontmatter,\n    sections,\n    outbound: outbound.slice(0, PEEK_LINK_LIMIT),\n    backlinks,\n    unresolved: unresolved.slice(0, PEEK_LINK_LIMIT),\n    outboundTotal: outbound.length,\n    backlinksTotal,\n    unresolvedTotal: unresolved.length,\n  };\n}\n"],"names":["posix","featureEnabled","SenseError","linkEdges","personalizedRank","WEIGHTED_BM25","RRF_K","find","db","cfg","terms","opts","hits","k","fetch","Math","max","matchSql","matchRows","prepare","all","Map","map","r","path","hit","candidates","forEach","i","set","score","via","length","nodes","seeds","ranked","filter","sort","a","b","slice","existing","get","exec","insert","c","run","where","INTERNAL_COLUMNS","Set","vaultMap","docs","columns","name","has","allFields","n","split","join","field","coverage","fields","hubs","recent","fieldsTotal","PEEK_LINK_LIMIT","peek","pathArg","row","paths","p","base","basename","replace","toLowerCase","matches","frontmatter","key","value","Object","entries","sections","outbound","backlinks","unresolved","backlinksTotal","out","l","dst","target","src","tokens","ceil","_size","outboundTotal","unresolvedTotal"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAGpC,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,sBAAsB;AAChD,SAASC,gBAAgB,QAAQ,aAAa;AAG9C,6EAA6E;AAC7E,gFAAgF;AAEhF,MAAMC,gBAAgB;AACtB,MAAMC,QAAQ;AAOd,mFAAmF;AACnF,4EAA4E;AAC5E,OAAO,SAASC,KAAKC,EAAgB,EAAEC,GAAW,EAAEC,KAAa,EAAEC,OAAoB,CAAC,CAAC;QAC7EA,SAmC2DC;IAnCrE,MAAMC,KAAIF,UAAAA,KAAKE,CAAC,cAANF,qBAAAA,UAAU;IACpB,MAAMG,QAAQC,KAAKC,GAAG,CAACH,IAAI,GAAG;IAE9B,sFAAsF;IACtF,yFAAyF;IACzF,MAAMI,WAAW,CAAC,wHAAwH,EAAEZ,cAAc,OAAO,EAAES,OAAO;IAC1K,MAAMI,YAAYV,GAAGW,OAAO,CAACF,UAAUG,GAAG,CAACV;IAE3C,MAAME,OAAO,IAAIS,IAAIH,UAAUI,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEC,IAAI;YAAED,EAAEE,GAAG;SAAC;IACzD,MAAMC,aAAa,IAAIL;IACvBH,UAAUS,OAAO,CAAC,CAACJ,GAAGK;QACpBF,WAAWG,GAAG,CAACN,EAAEC,IAAI,EAAE;YAAEM,OAAO,IAAKxB,CAAAA,QAAQsB,CAAAA;YAAIG,KAAK;QAAQ;IAChE;IAEA,IAAI9B,eAAeQ,KAAK,YAAYS,UAAUc,MAAM,GAAG,GAAG;QACxD,MAAMC,QAAQ,AAACzB,GAAGW,OAAO,CAAC,kCAAkCC,GAAG,GAA+BE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;QAC/G,MAAMU,QAAQ,IAAIb,IAAIH,UAAUI,GAAG,CAAC,CAACC,GAAGK,IAAM;gBAACL,EAAEC,IAAI;gBAAE,IAAKI,CAAAA,IAAI,CAAA;aAAG;QACnE,MAAMO,SAAS;eAAI/B,iBAAiB6B,OAAO9B,UAAUK,KAAK0B;SAAO,CAC9DE,MAAM,CAAC,CAAC,GAAGN,MAAM,GAAKA,QAAQ,MAC9BO,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BE,KAAK,CAAC,GAAG1B;QACZqB,OAAOR,OAAO,CAAC,CAAC,CAACH,KAAK,EAAEI;YACtB,MAAMa,WAAWf,WAAWgB,GAAG,CAAClB;YAChC,IAAIiB,UAAU;gBACZA,SAASX,KAAK,IAAI,IAAKxB,CAAAA,QAAQsB,CAAAA;gBAC/Ba,SAASV,GAAG,GAAG;YACjB,OAAO;gBACLL,WAAWG,GAAG,CAACL,MAAM;oBAAEM,OAAO,IAAKxB,CAAAA,QAAQsB,CAAAA;oBAAIG,KAAK;gBAAO;YAC7D;QACF;IACF;IAEAvB,GAAGmC,IAAI,CAAC;IACRnC,GAAGmC,IAAI,CAAC;IACR,MAAMC,SAASpC,GAAGW,OAAO,CAAC;IAC1B,KAAK,MAAM,CAACK,MAAMqB,EAAE,IAAInB,WAAYkB,OAAOE,GAAG,CAACtB,MAAMqB,EAAEf,KAAK,EAAEe,EAAEd,GAAG,GAAEnB,YAAAA,KAAK8B,GAAG,CAAClB,mBAATZ,uBAAAA,YAAkB;IAEvF,MAAMmC,QAAQpC,KAAKoC,KAAK,GAAG,CAAC,MAAM,EAAEpC,KAAKoC,KAAK,EAAE,GAAG;IACnD,OAAOvC,GACJW,OAAO,CACN,CAAC;;OAEA,EAAE4B,MAAM,kCAAkC,CAAC,EAE7C3B,GAAG,CAACP;AACT;AAUA,MAAMmC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAQ;AAErE,2EAA2E;AAC3E,OAAO,SAASC,SAAS1C,EAAgB,EAAEC,GAAW;IACpD,MAAM0C,OAAO3C,GAAGW,OAAO,CAAC,iFAAiFuB,GAAG;IAE5G,MAAMU,UAAU,AAAC5C,GAAGW,OAAO,CAAC,kCAAkCC,GAAG,GAA+BE,GAAG,CAAC,CAACC,IAAMA,EAAE8B,IAAI,EAAEjB,MAAM,CAAC,CAACiB,OAAS,CAACL,iBAAiBM,GAAG,CAACD;IAC1J,MAAME,YAAYH,QACf9B,GAAG,CAAC,CAAC+B;QACJ,MAAM,EAAEG,CAAC,EAAE,GAAGhD,GAAGW,OAAO,CAAC,CAAC,cAAc,EAAEkC,KAAKI,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,wBAAwB,CAAC,EAAEhB,GAAG;QACnG,OAAO;YAAEiB,OAAON;YAAMO,UAAUJ;QAAE;IACpC,GACCnB,IAAI,CAAC,CAACC,GAAGC,IAAM,AAACA,EAAEqB,QAAQ,GAAetB,EAAEsB,QAAQ;IACtD,MAAMC,SAASN,UAAUf,KAAK,CAAC,GAAG;IAElC,MAAMsB,OAAO7D,eAAeQ,KAAK,UAAWD,GAAGW,OAAO,CAAC,CAAC,gMAAgM,CAAC,EAAEC,GAAG,KAAe,EAAE;IAE/Q,MAAM2C,SAASvD,GAAGW,OAAO,CAAC,CAAC,iHAAiH,CAAC,EAAEC,GAAG;IAElJ,OAAO;QAAE+B;QAAMU;QAAQG,aAAaT,UAAUvB,MAAM;QAAE8B;QAAMC;IAAO;AACrE;AAiBA,MAAME,kBAAkB;AAExB,wFAAwF;AACxF,8FAA8F;AAC9F,OAAO,SAASC,KAAK1D,EAAgB,EAAEC,GAAW,EAAE0D,OAAe;QAiC3CC;IAhCtB,MAAMC,QAAQ,AAAC7D,GAAGW,OAAO,CAAC,kCAAkCC,GAAG,GAA+BE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAC/G,IAAIA,OAAO6C,MAAM9D,IAAI,CAAC,CAAC+D,IAAMA,MAAMH;IACnC,IAAI,CAAC3C,MAAM;QACT,MAAM+C,OAAOvE,MAAMwE,QAAQ,CAACL,SAASM,OAAO,CAAC,UAAU,IAAIC,WAAW;QACtE,MAAMC,UAAUN,MAAMjC,MAAM,CAAC,CAACkC,IAAMtE,MAAMwE,QAAQ,CAACF,GAAGG,OAAO,CAAC,UAAU,IAAIC,WAAW,OAAOH;QAC9F,IAAII,QAAQ3C,MAAM,KAAK,GAAGR,OAAOmD,OAAO,CAAC,EAAE;aACtC,IAAIA,QAAQ3C,MAAM,GAAG,GAAG,MAAM,IAAI9B,WAAW,kBAAkB,CAAC,CAAC,EAAEiE,QAAQ,gBAAgB,EAAEQ,QAAQjB,IAAI,CAAC,OAAO;aACjH,MAAM,IAAIxD,WAAW,kBAAkB,CAAC,iBAAiB,EAAEiE,QAAQ,CAAC,CAAC;IAC5E;IAEA,MAAMC,MAAM5D,GAAGW,OAAO,CAAC,8CAA8CuB,GAAG,CAAClB;IACzE,MAAMoD,cAAmB,CAAC;IAC1B,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAOC,OAAO,CAACZ,KAAM;QAC9C,IAAI,CAACpB,iBAAiBM,GAAG,CAACuB,QAAQC,UAAU,MAAMF,WAAW,CAACC,IAAI,GAAGC;IACvE;IAEA,MAAMG,WAAWhF,eAAeQ,KAAK,cAAeD,GAAGW,OAAO,CAAC,mGAAmGC,GAAG,CAACI,QAAkB,EAAE;IAE1L,IAAI0D,WAAqB,EAAE;IAC3B,IAAIC,YAAsB,EAAE;IAC5B,IAAIC,aAAuB,EAAE;IAC7B,IAAIC,iBAAiB;IACrB,IAAIpF,eAAeQ,KAAK,UAAU;QAChC,MAAM6E,MAAM9E,GAAGW,OAAO,CAAC,+DAA+DC,GAAG,CAACI;QAC1F0D,WAAW;eAAI,IAAIjC,IAAIqC,IAAIlD,MAAM,CAAC,CAACmD,IAAMA,EAAEC,GAAG,KAAK,MAAMlE,GAAG,CAAC,CAACiE,IAAMA,EAAEC,GAAG;SAAa;QACtFJ,aAAaE,IAAIlD,MAAM,CAAC,CAACmD,IAAMA,EAAEC,GAAG,KAAK,MAAMlE,GAAG,CAAC,CAACiE,IAAMA,EAAEE,MAAM;QAClEJ,iBAAiB,AAAC7E,GAAGW,OAAO,CAAC,4DAA4DuB,GAAG,CAAClB,MAAwBgC,CAAC;QACtH2B,YAAY,AAAC3E,GAAGW,OAAO,CAAC,qEAAqEC,GAAG,CAACI,MAAMyC,iBAA4C3C,GAAG,CAAC,CAACC,IAAMA,EAAEmE,GAAG;IACrK;IAEA,OAAO;QACLlE;QACAmE,QAAQ5E,KAAK6E,IAAI,CAAC,EAAExB,aAAAA,IAAIyB,KAAK,cAATzB,wBAAAA,aAAwB,KAAK;QACjDQ;QACAK;QACAC,UAAUA,SAAS1C,KAAK,CAAC,GAAGyB;QAC5BkB;QACAC,YAAYA,WAAW5C,KAAK,CAAC,GAAGyB;QAChC6B,eAAeZ,SAASlD,MAAM;QAC9BqD;QACAU,iBAAiBX,WAAWpD,MAAM;IACpC;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "Query a knowledge base you build with an agent: filter notes by frontmatter, then search inside them",
   "keywords": [
     "markdown",
@@ -48,8 +48,8 @@
   },
   "dependencies": {
     "fast-glob": "^3.3.3",
-    "gray-matter": "^4.0.3",
-    "remove-markdown": "^0.6.4"
+    "remove-markdown": "^0.6.4",
+    "yaml": "^2.9.0"
   },
   "devDependencies": {
     "@types/mocha": "*",
@@ -59,6 +59,6 @@
     "tsds-config": "*"
   },
   "engines": {
-    "node": ">=22.13"
+    "node": ">=22.16"
   }
 }