sensemaking 0.12.2 → 0.13.0

package/dist/esm/features/links.js

@@ -1,21 +1,31 @@
 import posix from 'node:path/posix';
-// links(src, target, target_base, dst): target as written, target_base its baseKey
+// links(src, target, target_base, dst, embed): target as written, target_base its baseKey
 // (indexed, for the incremental resolve below), dst the resolved path or NULL (a
-// queryable dead link).
+// queryable dead link), embed whether it's `![[x]]`/`![](x.md)` rather than `[[x]]`/`[](x.md)`
+// -- Obsidian's grain: a target both linked and embedded in the same note is two rows.
 // Wikilinks ([[target]], [[target#anchor|alias]], embeds) plus relative markdown links to .md files.
 function extract(_raw, body) {
-    const targets = new Set();
+    const seen = new Set();
+    const results = [];
+    const add = (target, embed)=>{
+        const key = `${target}\0${embed ? '1' : '0'}`;
+        if (seen.has(key)) return;
+        seen.add(key);
+        results.push({
+            target,
+            embed
+        });
+    };
     for (const m of body.matchAll(/\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|[^\]]*)?\]\]/g)){
+        var _m_index;
         const target = m[1].trim();
-        if (target) targets.add(target);
+        if (target) add(target, body[((_m_index = m.index) !== null && _m_index !== void 0 ? _m_index : 0) - 1] === '!');
     }
-    for (const m of body.matchAll(/\]\(([^)]+\.md)(?:#[^)]*)?\)/g)){
-        const target = m[1].trim();
-        if (target && !/^[a-z]+:\/\//i.test(target)) targets.add(target);
+    for (const m of body.matchAll(/(!)?\[[^\]]*\]\(([^)]+\.md)(?:#[^)]*)?\)/g)){
+        const target = m[2].trim();
+        if (target && !/^[a-z]+:\/\//i.test(target)) add(target, m[1] === '!');
     }
-    return [
-        ...targets
-    ];
+    return results;
 }
 function cleanTarget(target) {
     return target.replace(/\\/g, '/').replace(/^\.\//, '');
@@ -52,7 +62,9 @@ function buildByBase(files) {
     }
     return byBase;
 }
-// Applies resolveTarget to `rows`, writing only rows whose dst actually changes.
+// Applies resolveTarget to `rows`, writing only rows whose dst actually changes. The UPDATE
+// keys on (src, target): a link/embed sibling pair shares one resolution, so either row's
+// mismatch rewrites both.
 function resolveRows(db, rows, pathSet, byBase) {
     const update = db.prepare('UPDATE links SET dst = ? WHERE src = ? AND target = ?');
     let changed = false;
@@ -70,7 +82,7 @@ function resolveRows(db, rows, pathSet, byBase) {
 function resolveAll(db, files) {
     const pathSet = new Set(files.map((f)=>f.relPath));
     const byBase = buildByBase(files);
-    const rows = db.prepare('SELECT src, target, dst FROM links').all();
+    const rows = db.prepare('SELECT src, target, dst, embed FROM links').all();
     return resolveRows(db, rows, pathSet, byBase);
 }
 // Re-resolve only what this reconcile could have touched: re-stored sources, links whose
@@ -85,7 +97,7 @@ function resolveIncremental(db, delta) {
     // \0 can't appear in a path or target, so the key never collides; a space can.
     const candidates = new Map();
     const collect = (rows)=>{
-        for (const row of rows)candidates.set(`${row.src}\0${row.target}`, row);
+        for (const row of rows)candidates.set(`${row.src}\0${row.target}\0${row.embed}`, row);
     };
     // Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable
     // limit (SQLITE_MAX_VARIABLE_NUMBER, 32766).
@@ -93,7 +105,7 @@ function resolveIncremental(db, delta) {
         for(let i = 0; i < keys.length; i += 500){
             const chunk = keys.slice(i, i + 500);
             const placeholders = chunk.map(()=>'?').join(', ');
-            collect(db.prepare(`SELECT src, target, dst FROM links WHERE ${column} IN (${placeholders})`).all(...chunk));
+            collect(db.prepare(`SELECT src, target, dst, embed FROM links WHERE ${column} IN (${placeholders})`).all(...chunk));
         }
     };
     collectIn('src', delta.reparsed);
@@ -105,9 +117,10 @@ function resolveIncremental(db, delta) {
         ...candidates.values()
     ], pathSet, byBase);
 }
-// Resolved edges, for rank and for search's graph expansion.
+// Resolved edges, for rank and for search's graph expansion. DISTINCT: a target both linked
+// and embedded is two rows sharing one edge, which must not double its weight.
 export function linkEdges(db) {
-    return db.prepare('SELECT src, dst FROM links WHERE dst IS NOT NULL').all().map((r)=>[
+    return db.prepare('SELECT DISTINCT src, dst FROM links WHERE dst IS NOT NULL').all().map((r)=>[
             r.src,
             r.dst
         ]);
@@ -136,7 +149,7 @@ function vanishedSet(delta) {
 export const links = {
     name: 'links',
     schema (db) {
-        db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, PRIMARY KEY (src, target))');
+        db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, embed INTEGER, PRIMARY KEY (src, target, embed))');
         db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');
         db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');
     },
@@ -151,19 +164,21 @@ export const links = {
     },
     store (db, path, extracted, delta) {
         const targets = extracted;
-        // Stale rows (targets the new parse no longer contains) are real edge removals.
+        // Stale rows (target/embed pairs the new parse no longer contains) are real edge removals.
+        // \0 can't appear in a target, so the composite key never collides.
         let stale;
         if (targets.length === 0) {
             stale = db.prepare('DELETE FROM links WHERE src = ?').run(path);
         } else {
             const placeholders = targets.map(()=>'?').join(', ');
-            stale = db.prepare(`DELETE FROM links WHERE src = ? AND target NOT IN (${placeholders})`).run(path, ...targets);
+            const keys = targets.map((t)=>`${t.target}\0${t.embed ? '1' : '0'}`);
+            stale = db.prepare(`DELETE FROM links WHERE src = ? AND (target || char(0) || embed) NOT IN (${placeholders})`).run(path, ...keys);
         }
         if (Number(stale.changes) > 0) recordRemoved(delta, path);
         // Upsert preserves dst on surviving rows, so an unchanged link resolves to the same
         // value and reports no change; only genuinely new rows start at NULL.
-        const insert = db.prepare('INSERT INTO links (src, target, target_base, dst) VALUES (?, ?, ?, NULL) ON CONFLICT(src, target) DO UPDATE SET target_base = excluded.target_base');
-        for (const target of targets)insert.run(path, target, baseKey(cleanTarget(target)));
+        const insert = db.prepare('INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base');
+        for (const { target, embed } of targets)insert.run(path, target, baseKey(cleanTarget(target)), embed ? 1 : 0);
     },
     afterReconcile (db, delta) {
         var _ref;

package/dist/esm/features/links.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { FileStat } from '../scan.ts';\nimport type { Feature, ReconcileDelta } from './types.ts';\n\n// links(src, target, target_base, dst): target as written, target_base its baseKey\n// (indexed, for the incremental resolve below), dst the resolved path or NULL (a\n// queryable dead link).\n\n// Wikilinks ([[target]], [[target#anchor|alias]], embeds) plus relative markdown links to .md files.\nfunction extract(_raw: string, body: string): string[] {\n  const targets = new Set<string>();\n  for (const m of body.matchAll(/\\[\\[([^\\]|#]+)(?:#[^\\]|]*)?(?:\\|[^\\]]*)?\\]\\]/g)) {\n    const target = m[1].trim();\n    if (target) targets.add(target);\n  }\n  for (const m of body.matchAll(/\\]\\(([^)]+\\.md)(?:#[^)]*)?\\)/g)) {\n    const target = m[1].trim();\n    if (target && !/^[a-z]+:\\/\\//i.test(target)) targets.add(target);\n  }\n  return [...targets];\n}\n\nfunction cleanTarget(target: string): string {\n  return target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n}\n\nfunction baseKey(path: string): string {\n  return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking\n// note's directory, then basename match (lexicographically first on ties).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n  const clean = cleanTarget(target);\n  const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n  for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n    if (pathSet.has(candidate)) return candidate;\n  }\n  return byBase.get(baseKey(clean))?.[0] ?? null;\n}\n\nfunction buildByBase(files: FileStat[]): Map<string, string[]> {\n  const byBase = new Map<string, string[]>();\n  for (const path of files.map((f) => f.relPath).sort()) {\n    const key = baseKey(path);\n    const list = byBase.get(key);\n    if (list) list.push(path);\n    else byBase.set(key, [path]);\n  }\n  return byBase;\n}\n\ninterface LinkRow {\n  src: string;\n  target: string;\n  dst: string | null;\n}\n\n// Applies resolveTarget to `rows`, writing only rows whose dst actually changes.\nfunction resolveRows(db: DatabaseSync, rows: LinkRow[], pathSet: Set<string>, byBase: Map<string, string[]>): boolean {\n  const update = db.prepare('UPDATE links SET dst = ? WHERE src = ? AND target = ?');\n  let changed = false;\n  for (const row of rows) {\n    const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n    if (dst !== row.dst) {\n      update.run(dst, row.src, row.target);\n      changed = true;\n    }\n  }\n  return changed;\n}\n\n// A new or deleted file can change any note's resolution, so re-resolve the whole table.\n// Fallback for cold builds and large deltas -- see afterReconcile's threshold.\nfunction resolveAll(db: DatabaseSync, files: FileStat[]): boolean {\n  const pathSet = new Set(files.map((f) => f.relPath));\n  const byBase = buildByBase(files);\n  const rows = db.prepare('SELECT src, target, dst FROM links').all() as unknown as LinkRow[];\n  return resolveRows(db, rows, pathSet, byBase);\n}\n\n// Re-resolve only what this reconcile could have touched: re-stored sources, links whose\n// target basename matches an added/vanished file, and links whose dst just vanished. All\n// three are indexed lookups, so this never reads the whole table.\nfunction resolveIncremental(db: DatabaseSync, delta: ReconcileDelta): boolean {\n  const pathSet = new Set(delta.files.map((f) => f.relPath));\n  const byBase = buildByBase(delta.files);\n\n  const changedBasenames = new Set<string>();\n  for (const p of delta.added) changedBasenames.add(baseKey(p));\n  for (const p of delta.vanished) changedBasenames.add(baseKey(p));\n\n  // \\0 can't appear in a path or target, so the key never collides; a space can.\n  const candidates = new Map<string, LinkRow>();\n  const collect = (rows: LinkRow[]) => {\n    for (const row of rows) candidates.set(`${row.src}\\0${row.target}`, row);\n  };\n  // Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable\n  // limit (SQLITE_MAX_VARIABLE_NUMBER, 32766).\n  const collectIn = (column: string, keys: string[]) => {\n    for (let i = 0; i < keys.length; i += 500) {\n      const chunk = keys.slice(i, i + 500);\n      const placeholders = chunk.map(() => '?').join(', ');\n      collect(db.prepare(`SELECT src, target, dst FROM links WHERE ${column} IN (${placeholders})`).all(...chunk) as unknown as LinkRow[]);\n    }\n  };\n\n  collectIn('src', delta.reparsed);\n  collectIn('target_base', [...changedBasenames]);\n  collectIn('dst', delta.vanished);\n\n  return resolveRows(db, [...candidates.values()], pathSet, byBase);\n}\n\n// Resolved edges, for rank and for search's graph expansion.\nexport function linkEdges(db: DatabaseSync): [string, string][] {\n  return (db.prepare('SELECT src, dst FROM links WHERE dst IS NOT NULL').all() as Array<{ src: string; dst: string }>).map((r) => [r.src, r.dst]);\n}\n\n// remove() has already deleted the rows by the time afterReconcile runs, so it records here\n// whether they carried edges. Keyed on the delta object, so the state dies with the reconcile.\nconst removedWithLinks = new WeakMap<ReconcileDelta, Set<string>>();\n\nfunction recordRemoved(delta: ReconcileDelta, path: string): void {\n  let set = removedWithLinks.get(delta);\n  if (!set) {\n    set = new Set();\n    removedWithLinks.set(delta, set);\n  }\n  set.add(path);\n}\n\n// remove() runs per path; the vanished list is an array, so cache the Set per delta.\nconst vanishedSets = new WeakMap<ReconcileDelta, Set<string>>();\nfunction vanishedSet(delta: ReconcileDelta): Set<string> {\n  let set = vanishedSets.get(delta);\n  if (!set) {\n    set = new Set(delta.vanished);\n    vanishedSets.set(delta, set);\n  }\n  return set;\n}\n\nexport const links: Feature = {\n  name: 'links',\n  schema(db) {\n    db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, PRIMARY KEY (src, target))');\n    db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n    db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');\n  },\n  extract,\n  // Reparsed docs are diffed in store() below instead of wiped here: deleting and\n  // re-inserting resets dst to NULL, which made every touch-only reparse read as an edge\n  // change and recompute PageRank -- measured as most of the remaining update cost.\n  remove(db, path, delta) {\n    if (!vanishedSet(delta).has(path)) return;\n    const result = db.prepare('DELETE FROM links WHERE src = ?').run(path);\n    if (Number(result.changes) > 0) recordRemoved(delta, path);\n  },\n  store(db, path, extracted, delta) {\n    const targets = extracted as string[];\n    // Stale rows (targets the new parse no longer contains) are real edge removals.\n    let stale: ReturnType<ReturnType<DatabaseSync['prepare']>['run']>;\n    if (targets.length === 0) {\n      stale = db.prepare('DELETE FROM links WHERE src = ?').run(path);\n    } else {\n      const placeholders = targets.map(() => '?').join(', ');\n      stale = db.prepare(`DELETE FROM links WHERE src = ? AND target NOT IN (${placeholders})`).run(path, ...targets);\n    }\n    if (Number(stale.changes) > 0) recordRemoved(delta, path);\n    // Upsert preserves dst on surviving rows, so an unchanged link resolves to the same\n    // value and reports no change; only genuinely new rows start at NULL.\n    const insert = db.prepare('INSERT INTO links (src, target, target_base, dst) VALUES (?, ?, ?, NULL) ON CONFLICT(src, target) DO UPDATE SET target_base = excluded.target_base');\n    for (const target of targets) insert.run(path, target, baseKey(cleanTarget(target)));\n  },\n  afterReconcile(db, delta) {\n    // Any deleted rows that carried edges mean the edge set shrank -- whether the file\n    // vanished or was reparsed down to fewer/no links. dstChanged alone misses the\n    // reparsed-to-zero-links case (no surviving rows to re-resolve).\n    const removeHadLinks = (removedWithLinks.get(delta)?.size ?? 0) > 0;\n\n    const churn = delta.reparsed.length + delta.vanished.length;\n    // Past this share of the tree, a full pass is the only one guaranteed to match\n    // resolveTarget's ambiguity rules. A cold build clears the threshold on its own.\n    const large = delta.files.length === 0 || churn > 0.2 * delta.files.length;\n\n    const dstChanged = large ? resolveAll(db, delta.files) : resolveIncremental(db, delta);\n    delta.linksChanged = dstChanged || removeHadLinks;\n  },\n};\n"],"names":["posix","extract","_raw","body","targets","Set","m","matchAll","target","trim","add","test","cleanTarget","replace","baseKey","path","basename","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","has","get","buildByBase","files","Map","map","f","relPath","sort","key","list","push","set","resolveRows","db","rows","update","prepare","changed","row","dst","run","resolveAll","all","resolveIncremental","delta","changedBasenames","p","added","vanished","candidates","collect","collectIn","column","keys","i","length","chunk","slice","placeholders","reparsed","values","linkEdges","r","removedWithLinks","WeakMap","recordRemoved","vanishedSets","vanishedSet","links","name","schema","exec","remove","result","Number","changes","store","extracted","stale","insert","afterReconcile","removeHadLinks","size","churn","large","dstChanged","linksChanged"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAKpC,mFAAmF;AACnF,iFAAiF;AACjF,wBAAwB;AAExB,qGAAqG;AACrG,SAASC,QAAQC,IAAY,EAAEC,IAAY;IACzC,MAAMC,UAAU,IAAIC;IACpB,KAAK,MAAMC,KAAKH,KAAKI,QAAQ,CAAC,iDAAkD;QAC9E,MAAMC,SAASF,CAAC,CAAC,EAAE,CAACG,IAAI;QACxB,IAAID,QAAQJ,QAAQM,GAAG,CAACF;IAC1B;IACA,KAAK,MAAMF,KAAKH,KAAKI,QAAQ,CAAC,iCAAkC;QAC9D,MAAMC,SAASF,CAAC,CAAC,EAAE,CAACG,IAAI;QACxB,IAAID,UAAU,CAAC,gBAAgBG,IAAI,CAACH,SAASJ,QAAQM,GAAG,CAACF;IAC3D;IACA,OAAO;WAAIJ;KAAQ;AACrB;AAEA,SAASQ,YAAYJ,MAAc;IACjC,OAAOA,OAAOK,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;AACrD;AAEA,SAASC,QAAQC,IAAY;IAC3B,OAAOf,MAAMgB,QAAQ,CAACD,MAAMF,OAAO,CAAC,UAAU,IAAII,WAAW;AAC/D;AAEA,uFAAuF;AACvF,2EAA2E;AAC3E,SAASC,cAAcC,GAAW,EAAEX,MAAc,EAAEY,OAAoB,EAAEC,MAA6B;;QAM9FA;IALP,MAAMC,QAAQV,YAAYJ;IAC1B,MAAMe,UAAUvB,MAAMwB,SAAS,CAACxB,MAAMyB,IAAI,CAACzB,MAAM0B,OAAO,CAACP,MAAMG;IAC/D,KAAK,MAAMK,aAAa;QAACL;QAAO,GAAGA,MAAM,GAAG,CAAC;QAAEC;QAAS,GAAGA,QAAQ,GAAG,CAAC;KAAC,CAAE;QACxE,IAAIH,QAAQQ,GAAG,CAACD,YAAY,OAAOA;IACrC;IACA,gBAAON,cAAAA,OAAOQ,GAAG,CAACf,QAAQQ,qBAAnBD,kCAAAA,WAA4B,CAAC,EAAE,uCAAI;AAC5C;AAEA,SAASS,YAAYC,KAAiB;IACpC,MAAMV,SAAS,IAAIW;IACnB,KAAK,MAAMjB,QAAQgB,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,IAAI,GAAI;QACrD,MAAMC,MAAMvB,QAAQC;QACpB,MAAMuB,OAAOjB,OAAOQ,GAAG,CAACQ;QACxB,IAAIC,MAAMA,KAAKC,IAAI,CAACxB;aACfM,OAAOmB,GAAG,CAACH,KAAK;YAACtB;SAAK;IAC7B;IACA,OAAOM;AACT;AAQA,iFAAiF;AACjF,SAASoB,YAAYC,EAAgB,EAAEC,IAAe,EAAEvB,OAAoB,EAAEC,MAA6B;IACzG,MAAMuB,SAASF,GAAGG,OAAO,CAAC;IAC1B,IAAIC,UAAU;IACd,KAAK,MAAMC,OAAOJ,KAAM;QACtB,MAAMK,MAAM9B,cAAc6B,IAAI5B,GAAG,EAAE4B,IAAIvC,MAAM,EAAEY,SAASC;QACxD,IAAI2B,QAAQD,IAAIC,GAAG,EAAE;YACnBJ,OAAOK,GAAG,CAACD,KAAKD,IAAI5B,GAAG,EAAE4B,IAAIvC,MAAM;YACnCsC,UAAU;QACZ;IACF;IACA,OAAOA;AACT;AAEA,yFAAyF;AACzF,+EAA+E;AAC/E,SAASI,WAAWR,EAAgB,EAAEX,KAAiB;IACrD,MAAMX,UAAU,IAAIf,IAAI0B,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAClD,MAAMd,SAASS,YAAYC;IAC3B,MAAMY,OAAOD,GAAGG,OAAO,CAAC,sCAAsCM,GAAG;IACjE,OAAOV,YAAYC,IAAIC,MAAMvB,SAASC;AACxC;AAEA,yFAAyF;AACzF,yFAAyF;AACzF,kEAAkE;AAClE,SAAS+B,mBAAmBV,EAAgB,EAAEW,KAAqB;IACjE,MAAMjC,UAAU,IAAIf,IAAIgD,MAAMtB,KAAK,CAACE,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACxD,MAAMd,SAASS,YAAYuB,MAAMtB,KAAK;IAEtC,MAAMuB,mBAAmB,IAAIjD;IAC7B,KAAK,MAAMkD,KAAKF,MAAMG,KAAK,CAAEF,iBAAiB5C,GAAG,CAACI,QAAQyC;IAC1D,KAAK,MAAMA,KAAKF,MAAMI,QAAQ,CAAEH,iBAAiB5C,GAAG,CAACI,QAAQyC;IAE7D,+EAA+E;IAC/E,MAAMG,aAAa,IAAI1B;IACvB,MAAM2B,UAAU,CAAChB;QACf,KAAK,MAAMI,OAAOJ,KAAMe,WAAWlB,GAAG,CAAC,GAAGO,IAAI5B,GAAG,CAAC,EAAE,EAAE4B,IAAIvC,MAAM,EAAE,EAAEuC;IACtE;IACA,qFAAqF;IACrF,6CAA6C;IAC7C,MAAMa,YAAY,CAACC,QAAgBC;QACjC,IAAK,IAAIC,IAAI,GAAGA,IAAID,KAAKE,MAAM,EAAED,KAAK,IAAK;YACzC,MAAME,QAAQH,KAAKI,KAAK,CAACH,GAAGA,IAAI;YAChC,MAAMI,eAAeF,MAAMhC,GAAG,CAAC,IAAM,KAAKR,IAAI,CAAC;YAC/CkC,QAAQjB,GAAGG,OAAO,CAAC,CAAC,yCAAyC,EAAEgB,OAAO,KAAK,EAAEM,aAAa,CAAC,CAAC,EAAEhB,GAAG,IAAIc;QACvG;IACF;IAEAL,UAAU,OAAOP,MAAMe,QAAQ;IAC/BR,UAAU,eAAe;WAAIN;KAAiB;IAC9CM,UAAU,OAAOP,MAAMI,QAAQ;IAE/B,OAAOhB,YAAYC,IAAI;WAAIgB,WAAWW,MAAM;KAAG,EAAEjD,SAASC;AAC5D;AAEA,6DAA6D;AAC7D,OAAO,SAASiD,UAAU5B,EAAgB;IACxC,OAAO,AAACA,GAAGG,OAAO,CAAC,oDAAoDM,GAAG,GAA2ClB,GAAG,CAAC,CAACsC,IAAM;YAACA,EAAEpD,GAAG;YAAEoD,EAAEvB,GAAG;SAAC;AAChJ;AAEA,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMwB,mBAAmB,IAAIC;AAE7B,SAASC,cAAcrB,KAAqB,EAAEtC,IAAY;IACxD,IAAIyB,MAAMgC,iBAAiB3C,GAAG,CAACwB;IAC/B,IAAI,CAACb,KAAK;QACRA,MAAM,IAAInC;QACVmE,iBAAiBhC,GAAG,CAACa,OAAOb;IAC9B;IACAA,IAAI9B,GAAG,CAACK;AACV;AAEA,qFAAqF;AACrF,MAAM4D,eAAe,IAAIF;AACzB,SAASG,YAAYvB,KAAqB;IACxC,IAAIb,MAAMmC,aAAa9C,GAAG,CAACwB;IAC3B,IAAI,CAACb,KAAK;QACRA,MAAM,IAAInC,IAAIgD,MAAMI,QAAQ;QAC5BkB,aAAanC,GAAG,CAACa,OAAOb;IAC1B;IACA,OAAOA;AACT;AAEA,OAAO,MAAMqC,QAAiB;IAC5BC,MAAM;IACNC,QAAOrC,EAAE;QACPA,GAAGsC,IAAI,CAAC;QACRtC,GAAGsC,IAAI,CAAC;QACRtC,GAAGsC,IAAI,CAAC;IACV;IACA/E;IACA,gFAAgF;IAChF,uFAAuF;IACvF,kFAAkF;IAClFgF,QAAOvC,EAAE,EAAE3B,IAAI,EAAEsC,KAAK;QACpB,IAAI,CAACuB,YAAYvB,OAAOzB,GAAG,CAACb,OAAO;QACnC,MAAMmE,SAASxC,GAAGG,OAAO,CAAC,mCAAmCI,GAAG,CAAClC;QACjE,IAAIoE,OAAOD,OAAOE,OAAO,IAAI,GAAGV,cAAcrB,OAAOtC;IACvD;IACAsE,OAAM3C,EAAE,EAAE3B,IAAI,EAAEuE,SAAS,EAAEjC,KAAK;QAC9B,MAAMjD,UAAUkF;QAChB,gFAAgF;QAChF,IAAIC;QACJ,IAAInF,QAAQ4D,MAAM,KAAK,GAAG;YACxBuB,QAAQ7C,GAAGG,OAAO,CAAC,mCAAmCI,GAAG,CAAClC;QAC5D,OAAO;YACL,MAAMoD,eAAe/D,QAAQ6B,GAAG,CAAC,IAAM,KAAKR,IAAI,CAAC;YACjD8D,QAAQ7C,GAAGG,OAAO,CAAC,CAAC,mDAAmD,EAAEsB,aAAa,CAAC,CAAC,EAAElB,GAAG,CAAClC,SAASX;QACzG;QACA,IAAI+E,OAAOI,MAAMH,OAAO,IAAI,GAAGV,cAAcrB,OAAOtC;QACpD,oFAAoF;QACpF,sEAAsE;QACtE,MAAMyE,SAAS9C,GAAGG,OAAO,CAAC;QAC1B,KAAK,MAAMrC,UAAUJ,QAASoF,OAAOvC,GAAG,CAAClC,MAAMP,QAAQM,QAAQF,YAAYJ;IAC7E;IACAiF,gBAAe/C,EAAE,EAAEW,KAAK;;YAIEmB;QAHxB,mFAAmF;QACnF,+EAA+E;QAC/E,iEAAiE;QACjE,MAAMkB,iBAAiB,UAAClB,wBAAAA,iBAAiB3C,GAAG,CAACwB,oBAArBmB,4CAAAA,sBAA6BmB,IAAI,uCAAI,KAAK;QAElE,MAAMC,QAAQvC,MAAMe,QAAQ,CAACJ,MAAM,GAAGX,MAAMI,QAAQ,CAACO,MAAM;QAC3D,+EAA+E;QAC/E,iFAAiF;QACjF,MAAM6B,QAAQxC,MAAMtB,KAAK,CAACiC,MAAM,KAAK,KAAK4B,QAAQ,MAAMvC,MAAMtB,KAAK,CAACiC,MAAM;QAE1E,MAAM8B,aAAaD,QAAQ3C,WAAWR,IAAIW,MAAMtB,KAAK,IAAIqB,mBAAmBV,IAAIW;QAChFA,MAAM0C,YAAY,GAAGD,cAAcJ;IACrC;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { FileStat } from '../scan.ts';\nimport type { Feature, ReconcileDelta } from './types.ts';\n\n// links(src, target, target_base, dst, embed): target as written, target_base its baseKey\n// (indexed, for the incremental resolve below), dst the resolved path or NULL (a\n// queryable dead link), embed whether it's `![[x]]`/`![](x.md)` rather than `[[x]]`/`[](x.md)`\n// -- Obsidian's grain: a target both linked and embedded in the same note is two rows.\n\n// Wikilinks ([[target]], [[target#anchor|alias]], embeds) plus relative markdown links to .md files.\nfunction extract(_raw: string, body: string): Array<{ target: string; embed: boolean }> {\n  const seen = new Set<string>();\n  const results: Array<{ target: string; embed: boolean }> = [];\n  const add = (target: string, embed: boolean) => {\n    const key = `${target}\\0${embed ? '1' : '0'}`;\n    if (seen.has(key)) return;\n    seen.add(key);\n    results.push({ target, embed });\n  };\n  for (const m of body.matchAll(/\\[\\[([^\\]|#]+)(?:#[^\\]|]*)?(?:\\|[^\\]]*)?\\]\\]/g)) {\n    const target = m[1].trim();\n    if (target) add(target, body[(m.index ?? 0) - 1] === '!');\n  }\n  for (const m of body.matchAll(/(!)?\\[[^\\]]*\\]\\(([^)]+\\.md)(?:#[^)]*)?\\)/g)) {\n    const target = m[2].trim();\n    if (target && !/^[a-z]+:\\/\\//i.test(target)) add(target, m[1] === '!');\n  }\n  return results;\n}\n\nfunction cleanTarget(target: string): string {\n  return target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n}\n\nfunction baseKey(path: string): string {\n  return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking\n// note's directory, then basename match (lexicographically first on ties).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n  const clean = cleanTarget(target);\n  const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n  for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n    if (pathSet.has(candidate)) return candidate;\n  }\n  return byBase.get(baseKey(clean))?.[0] ?? null;\n}\n\nfunction buildByBase(files: FileStat[]): Map<string, string[]> {\n  const byBase = new Map<string, string[]>();\n  for (const path of files.map((f) => f.relPath).sort()) {\n    const key = baseKey(path);\n    const list = byBase.get(key);\n    if (list) list.push(path);\n    else byBase.set(key, [path]);\n  }\n  return byBase;\n}\n\ninterface LinkRow {\n  src: string;\n  target: string;\n  dst: string | null;\n  embed: number;\n}\n\n// Applies resolveTarget to `rows`, writing only rows whose dst actually changes. The UPDATE\n// keys on (src, target): a link/embed sibling pair shares one resolution, so either row's\n// mismatch rewrites both.\nfunction resolveRows(db: DatabaseSync, rows: LinkRow[], pathSet: Set<string>, byBase: Map<string, string[]>): boolean {\n  const update = db.prepare('UPDATE links SET dst = ? WHERE src = ? AND target = ?');\n  let changed = false;\n  for (const row of rows) {\n    const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n    if (dst !== row.dst) {\n      update.run(dst, row.src, row.target);\n      changed = true;\n    }\n  }\n  return changed;\n}\n\n// A new or deleted file can change any note's resolution, so re-resolve the whole table.\n// Fallback for cold builds and large deltas -- see afterReconcile's threshold.\nfunction resolveAll(db: DatabaseSync, files: FileStat[]): boolean {\n  const pathSet = new Set(files.map((f) => f.relPath));\n  const byBase = buildByBase(files);\n  const rows = db.prepare('SELECT src, target, dst, embed FROM links').all() as unknown as LinkRow[];\n  return resolveRows(db, rows, pathSet, byBase);\n}\n\n// Re-resolve only what this reconcile could have touched: re-stored sources, links whose\n// target basename matches an added/vanished file, and links whose dst just vanished. All\n// three are indexed lookups, so this never reads the whole table.\nfunction resolveIncremental(db: DatabaseSync, delta: ReconcileDelta): boolean {\n  const pathSet = new Set(delta.files.map((f) => f.relPath));\n  const byBase = buildByBase(delta.files);\n\n  const changedBasenames = new Set<string>();\n  for (const p of delta.added) changedBasenames.add(baseKey(p));\n  for (const p of delta.vanished) changedBasenames.add(baseKey(p));\n\n  // \\0 can't appear in a path or target, so the key never collides; a space can.\n  const candidates = new Map<string, LinkRow>();\n  const collect = (rows: LinkRow[]) => {\n    for (const row of rows) candidates.set(`${row.src}\\0${row.target}\\0${row.embed}`, row);\n  };\n  // Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable\n  // limit (SQLITE_MAX_VARIABLE_NUMBER, 32766).\n  const collectIn = (column: string, keys: string[]) => {\n    for (let i = 0; i < keys.length; i += 500) {\n      const chunk = keys.slice(i, i + 500);\n      const placeholders = chunk.map(() => '?').join(', ');\n      collect(db.prepare(`SELECT src, target, dst, embed FROM links WHERE ${column} IN (${placeholders})`).all(...chunk) as unknown as LinkRow[]);\n    }\n  };\n\n  collectIn('src', delta.reparsed);\n  collectIn('target_base', [...changedBasenames]);\n  collectIn('dst', delta.vanished);\n\n  return resolveRows(db, [...candidates.values()], pathSet, byBase);\n}\n\n// Resolved edges, for rank and for search's graph expansion. DISTINCT: a target both linked\n// and embedded is two rows sharing one edge, which must not double its weight.\nexport function linkEdges(db: DatabaseSync): [string, string][] {\n  return (db.prepare('SELECT DISTINCT src, dst FROM links WHERE dst IS NOT NULL').all() as Array<{ src: string; dst: string }>).map((r) => [r.src, r.dst]);\n}\n\n// remove() has already deleted the rows by the time afterReconcile runs, so it records here\n// whether they carried edges. Keyed on the delta object, so the state dies with the reconcile.\nconst removedWithLinks = new WeakMap<ReconcileDelta, Set<string>>();\n\nfunction recordRemoved(delta: ReconcileDelta, path: string): void {\n  let set = removedWithLinks.get(delta);\n  if (!set) {\n    set = new Set();\n    removedWithLinks.set(delta, set);\n  }\n  set.add(path);\n}\n\n// remove() runs per path; the vanished list is an array, so cache the Set per delta.\nconst vanishedSets = new WeakMap<ReconcileDelta, Set<string>>();\nfunction vanishedSet(delta: ReconcileDelta): Set<string> {\n  let set = vanishedSets.get(delta);\n  if (!set) {\n    set = new Set(delta.vanished);\n    vanishedSets.set(delta, set);\n  }\n  return set;\n}\n\nexport const links: Feature = {\n  name: 'links',\n  schema(db) {\n    db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, embed INTEGER, PRIMARY KEY (src, target, embed))');\n    db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n    db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');\n  },\n  extract,\n  // Reparsed docs are diffed in store() below instead of wiped here: deleting and\n  // re-inserting resets dst to NULL, which made every touch-only reparse read as an edge\n  // change and recompute PageRank -- measured as most of the remaining update cost.\n  remove(db, path, delta) {\n    if (!vanishedSet(delta).has(path)) return;\n    const result = db.prepare('DELETE FROM links WHERE src = ?').run(path);\n    if (Number(result.changes) > 0) recordRemoved(delta, path);\n  },\n  store(db, path, extracted, delta) {\n    const targets = extracted as Array<{ target: string; embed: boolean }>;\n    // Stale rows (target/embed pairs the new parse no longer contains) are real edge removals.\n    // \\0 can't appear in a target, so the composite key never collides.\n    let stale: ReturnType<ReturnType<DatabaseSync['prepare']>['run']>;\n    if (targets.length === 0) {\n      stale = db.prepare('DELETE FROM links WHERE src = ?').run(path);\n    } else {\n      const placeholders = targets.map(() => '?').join(', ');\n      const keys = targets.map((t) => `${t.target}\\0${t.embed ? '1' : '0'}`);\n      stale = db.prepare(`DELETE FROM links WHERE src = ? AND (target || char(0) || embed) NOT IN (${placeholders})`).run(path, ...keys);\n    }\n    if (Number(stale.changes) > 0) recordRemoved(delta, path);\n    // Upsert preserves dst on surviving rows, so an unchanged link resolves to the same\n    // value and reports no change; only genuinely new rows start at NULL.\n    const insert = db.prepare('INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base');\n    for (const { target, embed } of targets) insert.run(path, target, baseKey(cleanTarget(target)), embed ? 1 : 0);\n  },\n  afterReconcile(db, delta) {\n    // Any deleted rows that carried edges mean the edge set shrank -- whether the file\n    // vanished or was reparsed down to fewer/no links. dstChanged alone misses the\n    // reparsed-to-zero-links case (no surviving rows to re-resolve).\n    const removeHadLinks = (removedWithLinks.get(delta)?.size ?? 0) > 0;\n\n    const churn = delta.reparsed.length + delta.vanished.length;\n    // Past this share of the tree, a full pass is the only one guaranteed to match\n    // resolveTarget's ambiguity rules. A cold build clears the threshold on its own.\n    const large = delta.files.length === 0 || churn > 0.2 * delta.files.length;\n\n    const dstChanged = large ? resolveAll(db, delta.files) : resolveIncremental(db, delta);\n    delta.linksChanged = dstChanged || removeHadLinks;\n  },\n};\n"],"names":["posix","extract","_raw","body","seen","Set","results","add","target","embed","key","has","push","m","matchAll","trim","index","test","cleanTarget","replace","baseKey","path","basename","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","get","buildByBase","files","Map","map","f","relPath","sort","list","set","resolveRows","db","rows","update","prepare","changed","row","dst","run","resolveAll","all","resolveIncremental","delta","changedBasenames","p","added","vanished","candidates","collect","collectIn","column","keys","i","length","chunk","slice","placeholders","reparsed","values","linkEdges","r","removedWithLinks","WeakMap","recordRemoved","vanishedSets","vanishedSet","links","name","schema","exec","remove","result","Number","changes","store","extracted","targets","stale","t","insert","afterReconcile","removeHadLinks","size","churn","large","dstChanged","linksChanged"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAKpC,0FAA0F;AAC1F,iFAAiF;AACjF,+FAA+F;AAC/F,uFAAuF;AAEvF,qGAAqG;AACrG,SAASC,QAAQC,IAAY,EAAEC,IAAY;IACzC,MAAMC,OAAO,IAAIC;IACjB,MAAMC,UAAqD,EAAE;IAC7D,MAAMC,MAAM,CAACC,QAAgBC;QAC3B,MAAMC,MAAM,GAAGF,OAAO,EAAE,EAAEC,QAAQ,MAAM,KAAK;QAC7C,IAAIL,KAAKO,GAAG,CAACD,MAAM;QACnBN,KAAKG,GAAG,CAACG;QACTJ,QAAQM,IAAI,CAAC;YAAEJ;YAAQC;QAAM;IAC/B;IACA,KAAK,MAAMI,KAAKV,KAAKW,QAAQ,CAAC,iDAAkD;YAEhDD;QAD9B,MAAML,SAASK,CAAC,CAAC,EAAE,CAACE,IAAI;QACxB,IAAIP,QAAQD,IAAIC,QAAQL,IAAI,CAAC,EAACU,WAAAA,EAAEG,KAAK,cAAPH,sBAAAA,WAAW,KAAK,EAAE,KAAK;IACvD;IACA,KAAK,MAAMA,KAAKV,KAAKW,QAAQ,CAAC,6CAA8C;QAC1E,MAAMN,SAASK,CAAC,CAAC,EAAE,CAACE,IAAI;QACxB,IAAIP,UAAU,CAAC,gBAAgBS,IAAI,CAACT,SAASD,IAAIC,QAAQK,CAAC,CAAC,EAAE,KAAK;IACpE;IACA,OAAOP;AACT;AAEA,SAASY,YAAYV,MAAc;IACjC,OAAOA,OAAOW,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;AACrD;AAEA,SAASC,QAAQC,IAAY;IAC3B,OAAOrB,MAAMsB,QAAQ,CAACD,MAAMF,OAAO,CAAC,UAAU,IAAII,WAAW;AAC/D;AAEA,uFAAuF;AACvF,2EAA2E;AAC3E,SAASC,cAAcC,GAAW,EAAEjB,MAAc,EAAEkB,OAAoB,EAAEC,MAA6B;;QAM9FA;IALP,MAAMC,QAAQV,YAAYV;IAC1B,MAAMqB,UAAU7B,MAAM8B,SAAS,CAAC9B,MAAM+B,IAAI,CAAC/B,MAAMgC,OAAO,CAACP,MAAMG;IAC/D,KAAK,MAAMK,aAAa;QAACL;QAAO,GAAGA,MAAM,GAAG,CAAC;QAAEC;QAAS,GAAGA,QAAQ,GAAG,CAAC;KAAC,CAAE;QACxE,IAAIH,QAAQf,GAAG,CAACsB,YAAY,OAAOA;IACrC;IACA,gBAAON,cAAAA,OAAOO,GAAG,CAACd,QAAQQ,qBAAnBD,kCAAAA,WAA4B,CAAC,EAAE,uCAAI;AAC5C;AAEA,SAASQ,YAAYC,KAAiB;IACpC,MAAMT,SAAS,IAAIU;IACnB,KAAK,MAAMhB,QAAQe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,IAAI,GAAI;QACrD,MAAM/B,MAAMU,QAAQC;QACpB,MAAMqB,OAAOf,OAAOO,GAAG,CAACxB;QACxB,IAAIgC,MAAMA,KAAK9B,IAAI,CAACS;aACfM,OAAOgB,GAAG,CAACjC,KAAK;YAACW;SAAK;IAC7B;IACA,OAAOM;AACT;AASA,4FAA4F;AAC5F,0FAA0F;AAC1F,0BAA0B;AAC1B,SAASiB,YAAYC,EAAgB,EAAEC,IAAe,EAAEpB,OAAoB,EAAEC,MAA6B;IACzG,MAAMoB,SAASF,GAAGG,OAAO,CAAC;IAC1B,IAAIC,UAAU;IACd,KAAK,MAAMC,OAAOJ,KAAM;QACtB,MAAMK,MAAM3B,cAAc0B,IAAIzB,GAAG,EAAEyB,IAAI1C,MAAM,EAAEkB,SAASC;QACxD,IAAIwB,QAAQD,IAAIC,GAAG,EAAE;YACnBJ,OAAOK,GAAG,CAACD,KAAKD,IAAIzB,GAAG,EAAEyB,IAAI1C,MAAM;YACnCyC,UAAU;QACZ;IACF;IACA,OAAOA;AACT;AAEA,yFAAyF;AACzF,+EAA+E;AAC/E,SAASI,WAAWR,EAAgB,EAAET,KAAiB;IACrD,MAAMV,UAAU,IAAIrB,IAAI+B,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAClD,MAAMb,SAASQ,YAAYC;IAC3B,MAAMU,OAAOD,GAAGG,OAAO,CAAC,6CAA6CM,GAAG;IACxE,OAAOV,YAAYC,IAAIC,MAAMpB,SAASC;AACxC;AAEA,yFAAyF;AACzF,yFAAyF;AACzF,kEAAkE;AAClE,SAAS4B,mBAAmBV,EAAgB,EAAEW,KAAqB;IACjE,MAAM9B,UAAU,IAAIrB,IAAImD,MAAMpB,KAAK,CAACE,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACxD,MAAMb,SAASQ,YAAYqB,MAAMpB,KAAK;IAEtC,MAAMqB,mBAAmB,IAAIpD;IAC7B,KAAK,MAAMqD,KAAKF,MAAMG,KAAK,CAAEF,iBAAiBlD,GAAG,CAACa,QAAQsC;IAC1D,KAAK,MAAMA,KAAKF,MAAMI,QAAQ,CAAEH,iBAAiBlD,GAAG,CAACa,QAAQsC;IAE7D,+EAA+E;IAC/E,MAAMG,aAAa,IAAIxB;IACvB,MAAMyB,UAAU,CAAChB;QACf,KAAK,MAAMI,OAAOJ,KAAMe,WAAWlB,GAAG,CAAC,GAAGO,IAAIzB,GAAG,CAAC,EAAE,EAAEyB,IAAI1C,MAAM,CAAC,EAAE,EAAE0C,IAAIzC,KAAK,EAAE,EAAEyC;IACpF;IACA,qFAAqF;IACrF,6CAA6C;IAC7C,MAAMa,YAAY,CAACC,QAAgBC;QACjC,IAAK,IAAIC,IAAI,GAAGA,IAAID,KAAKE,MAAM,EAAED,KAAK,IAAK;YACzC,MAAME,QAAQH,KAAKI,KAAK,CAACH,GAAGA,IAAI;YAChC,MAAMI,eAAeF,MAAM9B,GAAG,CAAC,IAAM,KAAKP,IAAI,CAAC;YAC/C+B,QAAQjB,GAAGG,OAAO,CAAC,CAAC,gDAAgD,EAAEgB,OAAO,KAAK,EAAEM,aAAa,CAAC,CAAC,EAAEhB,GAAG,IAAIc;QAC9G;IACF;IAEAL,UAAU,OAAOP,MAAMe,QAAQ;IAC/BR,UAAU,eAAe;WAAIN;KAAiB;IAC9CM,UAAU,OAAOP,MAAMI,QAAQ;IAE/B,OAAOhB,YAAYC,IAAI;WAAIgB,WAAWW,MAAM;KAAG,EAAE9C,SAASC;AAC5D;AAEA,4FAA4F;AAC5F,+EAA+E;AAC/E,OAAO,SAAS8C,UAAU5B,EAAgB;IACxC,OAAO,AAACA,GAAGG,OAAO,CAAC,6DAA6DM,GAAG,GAA2ChB,GAAG,CAAC,CAACoC,IAAM;YAACA,EAAEjD,GAAG;YAAEiD,EAAEvB,GAAG;SAAC;AACzJ;AAEA,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMwB,mBAAmB,IAAIC;AAE7B,SAASC,cAAcrB,KAAqB,EAAEnC,IAAY;IACxD,IAAIsB,MAAMgC,iBAAiBzC,GAAG,CAACsB;IAC/B,IAAI,CAACb,KAAK;QACRA,MAAM,IAAItC;QACVsE,iBAAiBhC,GAAG,CAACa,OAAOb;IAC9B;IACAA,IAAIpC,GAAG,CAACc;AACV;AAEA,qFAAqF;AACrF,MAAMyD,eAAe,IAAIF;AACzB,SAASG,YAAYvB,KAAqB;IACxC,IAAIb,MAAMmC,aAAa5C,GAAG,CAACsB;IAC3B,IAAI,CAACb,KAAK;QACRA,MAAM,IAAItC,IAAImD,MAAMI,QAAQ;QAC5BkB,aAAanC,GAAG,CAACa,OAAOb;IAC1B;IACA,OAAOA;AACT;AAEA,OAAO,MAAMqC,QAAiB;IAC5BC,MAAM;IACNC,QAAOrC,EAAE;QACPA,GAAGsC,IAAI,CAAC;QACRtC,GAAGsC,IAAI,CAAC;QACRtC,GAAGsC,IAAI,CAAC;IACV;IACAlF;IACA,gFAAgF;IAChF,uFAAuF;IACvF,kFAAkF;IAClFmF,QAAOvC,EAAE,EAAExB,IAAI,EAAEmC,KAAK;QACpB,IAAI,CAACuB,YAAYvB,OAAO7C,GAAG,CAACU,OAAO;QACnC,MAAMgE,SAASxC,GAAGG,OAAO,CAAC,mCAAmCI,GAAG,CAAC/B;QACjE,IAAIiE,OAAOD,OAAOE,OAAO,IAAI,GAAGV,cAAcrB,OAAOnC;IACvD;IACAmE,OAAM3C,EAAE,EAAExB,IAAI,EAAEoE,SAAS,EAAEjC,KAAK;QAC9B,MAAMkC,UAAUD;QAChB,2FAA2F;QAC3F,oEAAoE;QACpE,IAAIE;QACJ,IAAID,QAAQvB,MAAM,KAAK,GAAG;YACxBwB,QAAQ9C,GAAGG,OAAO,CAAC,mCAAmCI,GAAG,CAAC/B;QAC5D,OAAO;YACL,MAAMiD,eAAeoB,QAAQpD,GAAG,CAAC,IAAM,KAAKP,IAAI,CAAC;YACjD,MAAMkC,OAAOyB,QAAQpD,GAAG,CAAC,CAACsD,IAAM,GAAGA,EAAEpF,MAAM,CAAC,EAAE,EAAEoF,EAAEnF,KAAK,GAAG,MAAM,KAAK;YACrEkF,QAAQ9C,GAAGG,OAAO,CAAC,CAAC,yEAAyE,EAAEsB,aAAa,CAAC,CAAC,EAAElB,GAAG,CAAC/B,SAAS4C;QAC/H;QACA,IAAIqB,OAAOK,MAAMJ,OAAO,IAAI,GAAGV,cAAcrB,OAAOnC;QACpD,oFAAoF;QACpF,sEAAsE;QACtE,MAAMwE,SAAShD,GAAGG,OAAO,CAAC;QAC1B,KAAK,MAAM,EAAExC,MAAM,EAAEC,KAAK,EAAE,IAAIiF,QAASG,OAAOzC,GAAG,CAAC/B,MAAMb,QAAQY,QAAQF,YAAYV,UAAUC,QAAQ,IAAI;IAC9G;IACAqF,gBAAejD,EAAE,EAAEW,KAAK;;YAIEmB;QAHxB,mFAAmF;QACnF,+EAA+E;QAC/E,iEAAiE;QACjE,MAAMoB,iBAAiB,UAACpB,wBAAAA,iBAAiBzC,GAAG,CAACsB,oBAArBmB,4CAAAA,sBAA6BqB,IAAI,uCAAI,KAAK;QAElE,MAAMC,QAAQzC,MAAMe,QAAQ,CAACJ,MAAM,GAAGX,MAAMI,QAAQ,CAACO,MAAM;QAC3D,+EAA+E;QAC/E,iFAAiF;QACjF,MAAM+B,QAAQ1C,MAAMpB,KAAK,CAAC+B,MAAM,KAAK,KAAK8B,QAAQ,MAAMzC,MAAMpB,KAAK,CAAC+B,MAAM;QAE1E,MAAMgC,aAAaD,QAAQ7C,WAAWR,IAAIW,MAAMpB,KAAK,IAAImB,mBAAmBV,IAAIW;QAChFA,MAAM4C,YAAY,GAAGD,cAAcJ;IACrC;AACF,EAAE"}

package/dist/esm/features/tags.d.ts

@@ -0,0 +1,2 @@
+import type { Feature } from './types.js';
+export declare const tags: Feature;

package/dist/esm/features/tags.js

@@ -0,0 +1,80 @@
+// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline
+// #tags from the prose, deduplicated, source not distinguished. Nested tags store full
+// (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` is how a caller matches the parent too.
+const FENCE_RE = /^(```|~~~)/;
+const INLINE_CODE_RE = /`[^`]*`/g;
+// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.
+const INLINE_TAG_RE = /(?:^|[\s([])#([\p{L}\p{N}_/-]+)/gu;
+// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an
+// all-digit result -- a tag needs at least one non-digit character.
+function normalizeTag(raw) {
+    const stripped = raw.replace(/^#/, '').replace(/\/+$/, '');
+    if (!stripped || /^\d+$/.test(stripped)) return null;
+    return stripped;
+}
+// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string
+// members are skipped rather than throwing -- `tags:\n  -` parses to [null].
+function frontmatterTags(data) {
+    const raw = data === null || data === void 0 ? void 0 : data.tags;
+    const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [
+        raw
+    ] : [];
+    const found = [];
+    for (const item of list){
+        if (typeof item !== 'string') continue;
+        const tag = normalizeTag(item);
+        if (tag) found.push(tag);
+    }
+    return found;
+}
+// #tag tokens outside fenced code blocks and inline code spans.
+function inlineTags(body) {
+    const found = [];
+    let inFence = false;
+    for (const line of body.split('\n')){
+        if (FENCE_RE.test(line)) {
+            inFence = !inFence;
+            continue;
+        }
+        if (inFence) continue;
+        const cleaned = line.replace(INLINE_CODE_RE, (m)=>' '.repeat(m.length));
+        for (const m of cleaned.matchAll(INLINE_TAG_RE)){
+            const tag = normalizeTag(m[1]);
+            if (tag) found.push(tag);
+        }
+    }
+    return found;
+}
+function extract(_raw, body, _search, data) {
+    return [
+        ...new Set([
+            ...frontmatterTags(data),
+            ...inlineTags(body)
+        ])
+    ].sort();
+}
+export const tags = {
+    name: 'tags',
+    schema (db) {
+        db.exec('CREATE TABLE IF NOT EXISTS tags ("path" TEXT, tag TEXT, PRIMARY KEY ("path", tag))');
+        db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');
+    },
+    extract,
+    // Per-file rows with nothing else to resolve, so only a vanished file needs a delete here;
+    // store() below handles a reparse's stale rows itself.
+    remove (db, path, delta) {
+        if (!delta.vanished.includes(path)) return;
+        db.prepare('DELETE FROM tags WHERE "path" = ?').run(path);
+    },
+    store (db, path, extracted) {
+        const found = extracted;
+        if (found.length === 0) {
+            db.prepare('DELETE FROM tags WHERE "path" = ?').run(path);
+        } else {
+            const placeholders = found.map(()=>'?').join(', ');
+            db.prepare(`DELETE FROM tags WHERE "path" = ? AND tag NOT IN (${placeholders})`).run(path, ...found);
+        }
+        const insert = db.prepare('INSERT OR IGNORE INTO tags ("path", tag) VALUES (?, ?)');
+        for (const tag of found)insert.run(path, tag);
+    }
+};

package/dist/esm/features/tags.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/tags.ts"],"sourcesContent":["import type { Feature } from './types.ts';\n\n// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline\n// #tags from the prose, deduplicated, source not distinguished. Nested tags store full\n// (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` is how a caller matches the parent too.\n\nconst FENCE_RE = /^(```|~~~)/;\nconst INLINE_CODE_RE = /`[^`]*`/g;\n// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.\nconst INLINE_TAG_RE = /(?:^|[\\s([])#([\\p{L}\\p{N}_/-]+)/gu;\n\n// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an\n// all-digit result -- a tag needs at least one non-digit character.\nfunction normalizeTag(raw: string): string | null {\n  const stripped = raw.replace(/^#/, '').replace(/\\/+$/, '');\n  if (!stripped || /^\\d+$/.test(stripped)) return null;\n  return stripped;\n}\n\n// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string\n// members are skipped rather than throwing -- `tags:\\n  -` parses to [null].\nfunction frontmatterTags(data?: Record<string, unknown>): string[] {\n  const raw = data?.tags;\n  const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [];\n  const found: string[] = [];\n  for (const item of list) {\n    if (typeof item !== 'string') continue;\n    const tag = normalizeTag(item);\n    if (tag) found.push(tag);\n  }\n  return found;\n}\n\n// #tag tokens outside fenced code blocks and inline code spans.\nfunction inlineTags(body: string): string[] {\n  const found: string[] = [];\n  let inFence = false;\n  for (const line of body.split('\\n')) {\n    if (FENCE_RE.test(line)) {\n      inFence = !inFence;\n      continue;\n    }\n    if (inFence) continue;\n    const cleaned = line.replace(INLINE_CODE_RE, (m) => ' '.repeat(m.length));\n    for (const m of cleaned.matchAll(INLINE_TAG_RE)) {\n      const tag = normalizeTag(m[1]);\n      if (tag) found.push(tag);\n    }\n  }\n  return found;\n}\n\nfunction extract(_raw: string, body: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): string[] {\n  return [...new Set([...frontmatterTags(data), ...inlineTags(body)])].sort();\n}\n\nexport const tags: Feature = {\n  name: 'tags',\n  schema(db) {\n    db.exec('CREATE TABLE IF NOT EXISTS tags (\"path\" TEXT, tag TEXT, PRIMARY KEY (\"path\", tag))');\n    db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');\n  },\n  extract,\n  // Per-file rows with nothing else to resolve, so only a vanished file needs a delete here;\n  // store() below handles a reparse's stale rows itself.\n  remove(db, path, delta) {\n    if (!delta.vanished.includes(path)) return;\n    db.prepare('DELETE FROM tags WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const found = extracted as string[];\n    if (found.length === 0) {\n      db.prepare('DELETE FROM tags WHERE \"path\" = ?').run(path);\n    } else {\n      const placeholders = found.map(() => '?').join(', ');\n      db.prepare(`DELETE FROM tags WHERE \"path\" = ? AND tag NOT IN (${placeholders})`).run(path, ...found);\n    }\n    const insert = db.prepare('INSERT OR IGNORE INTO tags (\"path\", tag) VALUES (?, ?)');\n    for (const tag of found) insert.run(path, tag);\n  },\n};\n"],"names":["FENCE_RE","INLINE_CODE_RE","INLINE_TAG_RE","normalizeTag","raw","stripped","replace","test","frontmatterTags","data","tags","list","Array","isArray","found","item","tag","push","inlineTags","body","inFence","line","split","cleaned","m","repeat","length","matchAll","extract","_raw","_search","Set","sort","name","schema","db","exec","remove","path","delta","vanished","includes","prepare","run","store","extracted","placeholders","map","join","insert"],"mappings":"AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAE5F,MAAMA,WAAW;AACjB,MAAMC,iBAAiB;AACvB,iGAAiG;AACjG,MAAMC,gBAAgB;AAEtB,sFAAsF;AACtF,oEAAoE;AACpE,SAASC,aAAaC,GAAW;IAC/B,MAAMC,WAAWD,IAAIE,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,QAAQ;IACvD,IAAI,CAACD,YAAY,QAAQE,IAAI,CAACF,WAAW,OAAO;IAChD,OAAOA;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASG,gBAAgBC,IAA8B;IACrD,MAAML,MAAMK,iBAAAA,2BAAAA,KAAMC,IAAI;IACtB,MAAMC,OAAOC,MAAMC,OAAO,CAACT,OAAOA,MAAM,OAAOA,QAAQ,WAAW;QAACA;KAAI,GAAG,EAAE;IAC5E,MAAMU,QAAkB,EAAE;IAC1B,KAAK,MAAMC,QAAQJ,KAAM;QACvB,IAAI,OAAOI,SAAS,UAAU;QAC9B,MAAMC,MAAMb,aAAaY;QACzB,IAAIC,KAAKF,MAAMG,IAAI,CAACD;IACtB;IACA,OAAOF;AACT;AAEA,gEAAgE;AAChE,SAASI,WAAWC,IAAY;IAC9B,MAAML,QAAkB,EAAE;IAC1B,IAAIM,UAAU;IACd,KAAK,MAAMC,QAAQF,KAAKG,KAAK,CAAC,MAAO;QACnC,IAAItB,SAASO,IAAI,CAACc,OAAO;YACvBD,UAAU,CAACA;YACX;QACF;QACA,IAAIA,SAAS;QACb,MAAMG,UAAUF,KAAKf,OAAO,CAACL,gBAAgB,CAACuB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QACvE,KAAK,MAAMF,KAAKD,QAAQI,QAAQ,CAACzB,eAAgB;YAC/C,MAAMc,MAAMb,aAAaqB,CAAC,CAAC,EAAE;YAC7B,IAAIR,KAAKF,MAAMG,IAAI,CAACD;QACtB;IACF;IACA,OAAOF;AACT;AAEA,SAASc,QAAQC,IAAY,EAAEV,IAAY,EAAEW,OAA4C,EAAErB,IAA8B;IACvH,OAAO;WAAI,IAAIsB,IAAI;eAAIvB,gBAAgBC;eAAUS,WAAWC;SAAM;KAAE,CAACa,IAAI;AAC3E;AAEA,OAAO,MAAMtB,OAAgB;IAC3BuB,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;QACRD,GAAGC,IAAI,CAAC;IACV;IACAR;IACA,2FAA2F;IAC3F,uDAAuD;IACvDS,QAAOF,EAAE,EAAEG,IAAI,EAAEC,KAAK;QACpB,IAAI,CAACA,MAAMC,QAAQ,CAACC,QAAQ,CAACH,OAAO;QACpCH,GAAGO,OAAO,CAAC,qCAAqCC,GAAG,CAACL;IACtD;IACAM,OAAMT,EAAE,EAAEG,IAAI,EAAEO,SAAS;QACvB,MAAM/B,QAAQ+B;QACd,IAAI/B,MAAMY,MAAM,KAAK,GAAG;YACtBS,GAAGO,OAAO,CAAC,qCAAqCC,GAAG,CAACL;QACtD,OAAO;YACL,MAAMQ,eAAehC,MAAMiC,GAAG,CAAC,IAAM,KAAKC,IAAI,CAAC;YAC/Cb,GAAGO,OAAO,CAAC,CAAC,kDAAkD,EAAEI,aAAa,CAAC,CAAC,EAAEH,GAAG,CAACL,SAASxB;QAChG;QACA,MAAMmC,SAASd,GAAGO,OAAO,CAAC;QAC1B,KAAK,MAAM1B,OAAOF,MAAOmC,OAAON,GAAG,CAACL,MAAMtB;IAC5C;AACF,EAAE"}

package/dist/esm/features/types.d.ts

@@ -1,5 +1,5 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { Config, FeatureName } from '../config.js';
+import type { Config, FeatureName } from '../config/index.js';
 import type { FileStat } from '../scan.js';
 export interface ReconcileDelta {
     files: FileStat[];
@@ -14,7 +14,7 @@ export interface Feature {
     extract?(raw: string, body: string, search?: {
         title: string;
         summary: string;
-    }): unknown;
+    }, data?: Record<string, unknown>): unknown;
     remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;
     store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;
     afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;

package/dist/esm/features/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/types.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config, FeatureName } from '../config.ts';\nimport type { FileStat } from '../scan.ts';\n\n// What changed this reconcile, so hooks act on the delta rather than re-reading the tree.\n// `added` is the brand-new subset of `reparsed`. Hooks communicate through this one object.\nexport interface ReconcileDelta {\n  files: FileStat[];\n  reparsed: string[];\n  added: string[];\n  vanished: string[];\n  // Set by the links feature: the resolved edge set may differ from before this reconcile.\n  linksChanged?: boolean;\n}\n\n// A feature owns its schema, per-file extraction/storage, and any whole-tree pass.\n// Adding a feature = one module here + one entry in the registry (index.ts).\nexport interface Feature {\n  name: FeatureName;\n  schema(db: DatabaseSync): void;\n  // Pure, per file: raw is the full file, body the prose after frontmatter,\n  // search the normalized title/summary strings.\n  extract?(raw: string, body: string, search?: { title: string; summary: string }): unknown;\n  remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;\n  store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;\n  // After all rows are current, inside the reconcile transaction (resolution, rank).\n  // Hooks run in registry order and communicate through fields on `delta`.\n  afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;\n  // Per-file opt-out (currently only embed, via FileStat.embed): absent means the feature\n  // applies to every file it's otherwise on for. Checked per file before extract/store so a\n  // doc in a tree with no embedding model gets no rows for it at all.\n  enabledForFile?(cfg: Config, file: FileStat): boolean;\n}\n"],"names":[],"mappings":"AAeA,mFAAmF;AACnF,6EAA6E;AAC7E,WAeC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/types.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport type { FileStat } from '../scan.ts';\n\n// What changed this reconcile, so hooks act on the delta rather than re-reading the tree.\n// `added` is the brand-new subset of `reparsed`. Hooks communicate through this one object.\nexport interface ReconcileDelta {\n  files: FileStat[];\n  reparsed: string[];\n  added: string[];\n  vanished: string[];\n  // Set by the links feature: the resolved edge set may differ from before this reconcile.\n  linksChanged?: boolean;\n}\n\n// A feature owns its schema, per-file extraction/storage, and any whole-tree pass.\n// Adding a feature = one module here + one entry in the registry (index.ts).\nexport interface Feature {\n  name: FeatureName;\n  schema(db: DatabaseSync): void;\n  // Pure, per file: raw is the full file, body the prose after frontmatter,\n  // search the normalized title/summary strings, data the parsed frontmatter.\n  extract?(raw: string, body: string, search?: { title: string; summary: string }, data?: Record<string, unknown>): unknown;\n  remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;\n  store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;\n  // After all rows are current, inside the reconcile transaction (resolution, rank).\n  // Hooks run in registry order and communicate through fields on `delta`.\n  afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;\n  // Per-file opt-out (currently only embed, via FileStat.embed): absent means the feature\n  // applies to every file it's otherwise on for. Checked per file before extract/store so a\n  // doc in a tree with no embedding model gets no rows for it at all.\n  enabledForFile?(cfg: Config, file: FileStat): boolean;\n}\n"],"names":[],"mappings":"AAeA,mFAAmF;AACnF,6EAA6E;AAC7E,WAeC"}

package/dist/esm/index.d.ts

@@ -1,9 +1,9 @@
-export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands.js';
-export { mapTree, peek, presetCoverage, search } from './commands.js';
-export type { Config, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './config.js';
-export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
-export type { OpenResult } from './db.js';
-export { clearCache, open } from './db.js';
+export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.js';
+export { mapTree, peek, presetCoverage, search } from './commands/index.js';
+export type { Config, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './config/index.js';
+export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config/index.js';
+export type { OpenResult } from './db/index.js';
+export { clearCache, open } from './db/index.js';
 export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output.js';

package/dist/esm/index.js

@@ -1,8 +1,8 @@
 // Public library API. Deliberately small: every export is a stability promise;
 // internals (feature registry, graph, scan, meta) stay module-private.
-export { mapTree, peek, presetCoverage, search } from './commands.js';
-export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
-export { clearCache, open } from './db.js';
+export { mapTree, peek, presetCoverage, search } from './commands/index.js';
+export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config/index.js';
+export { clearCache, open } from './db/index.js';
 export { SenseError } from './errors.js';
 export { printRows } from './output.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands.ts';\nexport type { Config, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './config.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\nexport type { OpenResult } from './db.ts';\nexport { clearCache, open } from './db.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","clearCache","open","SenseError","printRows","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,gBAAgB;AAEtE,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAE1H,SAASC,UAAU,EAAEC,IAAI,QAAQ,UAAU;AAE3C,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { OpenResult } from './db/index.ts';\nexport { clearCache, open } from './db/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","clearCache","open","SenseError","printRows","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,oBAAoB;AAEhI,SAASC,UAAU,EAAEC,IAAI,QAAQ,gBAAgB;AAEjD,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/scan.d.ts

@@ -1,10 +1,11 @@
-import type { Config } from './config.js';
+import type { Config } from './config/index.js';
 import type { Feature } from './features/types.js';
 export declare const RESERVED_COLUMNS: Set<string>;
 export interface FileStat {
     relPath: string;
     absPath: string;
     mtimeMs: number;
+    ctimeMs: number;
     size: number;
     presets: string[];
     embed: boolean;
@@ -14,6 +15,7 @@ export declare function listFiles(cfg: Config, baseDir: string): FileStat[];
 export interface ParsedDoc {
     relPath: string;
     mtimeMs: number;
+    ctimeMs: number;
     size: number;
     presets: string[];
     data: Record<string, string | number | bigint | null>;

package/dist/esm/scan.js

@@ -2,13 +2,14 @@ import { globSync, readFileSync, statSync } from 'node:fs';
 import { join, sep } from 'node:path';
 import removeMarkdown from 'remove-markdown';
 import { isCollection, parseDocument, visit } from 'yaml';
-import { embedEnabled, presetNames, presetSemanticEnabled } from './config.js';
+import { embedEnabled, presetNames, presetSemanticEnabled } from './config/index.js';
 // Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
 // Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell
 // a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.
 export const RESERVED_COLUMNS = new Set([
     'path',
     '_mtime',
+    '_ctime',
     '_size',
     '_rank',
     '_parse_error',
@@ -84,6 +85,7 @@ export function listFiles(cfg, baseDir) {
             relPath,
             absPath,
             mtimeMs: st.mtimeMs,
+            ctimeMs: st.birthtimeMs,
             size: st.size,
             presets,
             embed
@@ -245,6 +247,7 @@ export function parseFile(file, extractors = []) {
         doc: {
             relPath: file.relPath,
             mtimeMs: file.mtimeMs,
+            ctimeMs: file.ctimeMs,
             size: file.size,
             presets: file.presets,
             data: mapped,
@@ -254,7 +257,7 @@ export function parseFile(file, extractors = []) {
                 var _f_extract;
                 return [
                     f.name,
-                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search)
+                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search, data)
                 ];
             }))
         },

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["globSync","readFileSync","statSync","join","sep","removeMarkdown","isCollection","parseDocument","visit","embedEnabled","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","add","embedding","semanticPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","size","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","doc","warnings","found","Pair","_key","pair","key","BREAK","firstLine","message","parseFrontmatter","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","parseFile","file","extractors","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AAC3D,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AACtC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,SAASC,YAAY,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAG/E,6EAA6E;AAE7E,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAE9H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4DAA4D;AAC5D,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBlB,eAAeiB;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAWA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBvB,GAAG;IAClE,OAAOuB,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWxB,IAAI,CAAC,OAAOuB;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAchC,QAAQ;IAC5B,KAAK,MAAMiC,QAAQ3B,YAAYsB,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWxC,SAASsC,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIrB;YACzC+B,IAAIE,GAAG,CAACT;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMG,YAAYtC,aAAauB;IAC/B,MAAMgB,kBAAkBD,YAAY,IAAIlC,IAAIH,YAAYsB,KAAKiB,MAAM,CAAC,CAACZ,OAAS1B,sBAAsBqB,KAAKK,UAAU;IAEnH,MAAMa,QAAoB,EAAE;IAC5B,KAAK,MAAMxB,WAAW;WAAIQ,SAASiB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlD,KAAK8B,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM4B,KAAKpD,SAASmD,SAASxB;QAC7B,IAAI,EAACyB,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMhB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC0B,IAAI;QAChE,MAAMI,QAAQR,oBAAoB,QAAQT,QAAQkB,IAAI,CAAC,CAACpB,OAASW,gBAAgBU,GAAG,CAACrB;QACrFa,MAAMS,IAAI,CAAC;YAAEjC;YAAS2B;YAASO,SAASN,GAAGM,OAAO;YAAEC,MAAMP,GAAGO,IAAI;YAAEtB;YAASiB;QAAM;IACpF;IACA,OAAON;AACT;AAiBA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMY,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBhD,KAAa;IAC7C,OAAO+C,kBAAkBE,IAAI,CAACjD;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASkD,cAAclD,KAAa;IACzC,MAAMmD,IAAIL,aAAaM,IAAI,CAACpD;IAC5B,IAAImD,MAAM,MAAM,OAAOnD;IACvB,MAAM,GAAGqD,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAAStD,aAAasD,SAAS,MAAM,KAAKA,KAAKpD,OAAO,CAAC,KAAK;IAC3E,MAAMsD,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAe5D,QAAQ4D;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASK,SAASjE,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOkE,OAAOlE,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO6D,OAAOM,aAAa,CAACnE,SAASkE,OAAOlE,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOkD,cAAclD;IACpD,OAAOoE,KAAKC,SAAS,CAACrE;AACxB;AAEA,gEAAgE;AAChE,SAASsE,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,MAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAK7E,WAAW,OAAO;QAAEyE,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBrE,OAAe,EAAEsE,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC1F,MAAMwF,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAAC/F,aAAa+F,KAAKC,GAAG,GAAG,OAAOrF;YACpCiF,QAAQ;YACR,OAAO1F,MAAM+F,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAAS8E,UAAUC,OAAe;IAChC,OAAOA,QAAQ7E,KAAK,CAAC,KAAK,CAAC,EAAE,CAACT,OAAO,CAAC,SAAS;AACjD;AAEA,SAASuF,iBAAiBhF,OAAe,EAAEgE,EAAU,EAAEO,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMzF,cAAcmF,IAAI;QAAEiB,UAAU;IAAS;IACnD,MAAMC,UAAUZ,IAAIa,MAAM,CAAC5D,MAAM,CAAC,CAAC6D,MAAQ,CAAChG,oBAAoB4C,GAAG,CAACoD,IAAIC,IAAI;IAC5E,IAAIH,QAAQlC,MAAM,GAAG,GAAG;QACtB,MAAMsC,SAASJ,QAAQlC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEkC,QAAQlC,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMuC,aAAa,GAAGT,UAAUI,OAAO,CAAC,EAAE,CAACH,OAAO,IAAIO,QAAQ;QAC9Df,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOlB,IAAImB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMG,aAAaT,UAAU,AAACM,IAAcL,OAAO;QACnDR,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASjG,WAAW,OAAO;QAAEiG,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBhB,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,CAAC,EAAEuF,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAlB,oBAAoBrE,SAASsE,KAAKC;IAClC,OAAO;QAAEiB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMjC,MAAMtF,aAAasH,KAAKlE,OAAO,EAAE;IACvC,MAAM4C,WAAqB,EAAE;IAE7B,MAAM,EAAEP,EAAE,EAAEC,MAAM8B,OAAO,EAAE,GAAGnC,iBAAiBC;IAC/C,MAAM,EAAE2B,IAAI,EAAED,UAAU,EAAE,GAAGvB,OAAO,OAAO;QAAEwB,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIP,iBAAiBa,KAAK7F,OAAO,EAAEgE,IAAIO;IAC1I,MAAMyB,SAA0D,CAAC;IAEjE,KAAK,MAAMpB,OAAOqB,OAAOxE,IAAI,CAAC+D,MAAO;QACnC,IAAItG,iBAAiB8C,GAAG,CAAC4C,MAAM;YAC7BL,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAE4D,KAAK7F,OAAO,CAAC,8BAA8B,EAAE4E,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMtF,QAAQiE,SAASiC,IAAI,CAACZ,IAAI;QAChC,IAAI,OAAOtF,UAAU,YAAYgD,kBAAkBhD,UAAU6D,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAAChE,SAAS;YAC5FiF,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAE4D,KAAK7F,OAAO,CAAC,EAAE,EAAE4E,IAAI,sBAAsB,EAAEtF,MAAM,8CAA8C,CAAC;QAC9H;QACA0G,MAAM,CAACpB,IAAI,GAAGtF;IAChB;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAM4G,SAAS;QAAEC,OAAO9G,cAAcmG,KAAKW,KAAK;QAAGC,SAAS/G,cAAcmG,KAAKY,OAAO;QAAGC,MAAM1G,UAAUoG;IAAS;IAElH,OAAO;QACLzB,KAAK;YACHtE,SAAS6F,KAAK7F,OAAO;YACrBkC,SAAS2D,KAAK3D,OAAO;YACrBC,MAAM0D,KAAK1D,IAAI;YACftB,SAASgF,KAAKhF,OAAO;YACrB2E,MAAMQ;YACNT;YACAW;YACAI,WAAWL,OAAOM,WAAW,CAACT,WAAWvE,MAAM,CAAC,CAACiF,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAE7F,IAAI;qBAAE6F,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY3C,KAAKkC,SAASG;iBAAQ;;QAC1H;QACA3B;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config/index.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config/index.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  ctimeMs: number; // filesystem birthtime; a clone or copy resets it, like _mtime\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, ctimeMs: st.birthtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data)])),\n    },\n    warnings,\n  };\n}\n"],"names":["globSync","readFileSync","statSync","join","sep","removeMarkdown","isCollection","parseDocument","visit","embedEnabled","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","add","embedding","semanticPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","ctimeMs","birthtimeMs","size","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","doc","warnings","found","Pair","_key","pair","key","BREAK","firstLine","message","parseFrontmatter","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","parseFile","file","extractors","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AAC3D,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AACtC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,SAASC,YAAY,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,oBAAoB;AAGrF,6EAA6E;AAE7E,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4DAA4D;AAC5D,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBlB,eAAeiB;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAYA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBvB,GAAG;IAClE,OAAOuB,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWxB,IAAI,CAAC,OAAOuB;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAchC,QAAQ;IAC5B,KAAK,MAAMiC,QAAQ3B,YAAYsB,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWxC,SAASsC,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIrB;YACzC+B,IAAIE,GAAG,CAACT;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMG,YAAYtC,aAAauB;IAC/B,MAAMgB,kBAAkBD,YAAY,IAAIlC,IAAIH,YAAYsB,KAAKiB,MAAM,CAAC,CAACZ,OAAS1B,sBAAsBqB,KAAKK,UAAU;IAEnH,MAAMa,QAAoB,EAAE;IAC5B,KAAK,MAAMxB,WAAW;WAAIQ,SAASiB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlD,KAAK8B,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM4B,KAAKpD,SAASmD,SAASxB;QAC7B,IAAI,EAACyB,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMhB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC0B,IAAI;QAChE,MAAMI,QAAQR,oBAAoB,QAAQT,QAAQkB,IAAI,CAAC,CAACpB,OAASW,gBAAgBU,GAAG,CAACrB;QACrFa,MAAMS,IAAI,CAAC;YAAEjC;YAAS2B;YAASO,SAASN,GAAGM,OAAO;YAAEC,SAASP,GAAGQ,WAAW;YAAEC,MAAMT,GAAGS,IAAI;YAAExB;YAASiB;QAAM;IAC7G;IACA,OAAON;AACT;AAkBA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMc,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBlD,KAAa;IAC7C,OAAOiD,kBAAkBE,IAAI,CAACnD;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASoD,cAAcpD,KAAa;IACzC,MAAMqD,IAAIL,aAAaM,IAAI,CAACtD;IAC5B,IAAIqD,MAAM,MAAM,OAAOrD;IACvB,MAAM,GAAGuD,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASxD,aAAawD,SAAS,MAAM,KAAKA,KAAKtD,OAAO,CAAC,KAAK;IAC3E,MAAMwD,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAe9D,QAAQ8D;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASK,SAASnE,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOoE,OAAOpE,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO+D,OAAOM,aAAa,CAACrE,SAASoE,OAAOpE,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOoD,cAAcpD;IACpD,OAAOsE,KAAKC,SAAS,CAACvE;AACxB;AAEA,gEAAgE;AAChE,SAASwE,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,MAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAK/E,WAAW,OAAO;QAAE2E,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBvE,OAAe,EAAEwE,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC5F,MAAM0F,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACjG,aAAaiG,KAAKC,GAAG,GAAG,OAAOvF;YACpCmF,QAAQ;YACR,OAAO5F,MAAMiG,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASgF,UAAUC,OAAe;IAChC,OAAOA,QAAQ/E,KAAK,CAAC,KAAK,CAAC,EAAE,CAACT,OAAO,CAAC,SAAS;AACjD;AAEA,SAASyF,iBAAiBlF,OAAe,EAAEkE,EAAU,EAAEO,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAM3F,cAAcqF,IAAI;QAAEiB,UAAU;IAAS;IACnD,MAAMC,UAAUZ,IAAIa,MAAM,CAAC9D,MAAM,CAAC,CAAC+D,MAAQ,CAAClG,oBAAoB4C,GAAG,CAACsD,IAAIC,IAAI;IAC5E,IAAIH,QAAQlC,MAAM,GAAG,GAAG;QACtB,MAAMsC,SAASJ,QAAQlC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEkC,QAAQlC,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMuC,aAAa,GAAGT,UAAUI,OAAO,CAAC,EAAE,CAACH,OAAO,IAAIO,QAAQ;QAC9Df,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEyF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOlB,IAAImB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMG,aAAaT,UAAU,AAACM,IAAcL,OAAO;QACnDR,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEyF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASnG,WAAW,OAAO;QAAEmG,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBhB,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,CAAC,EAAEyF,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAlB,oBAAoBvE,SAASwE,KAAKC;IAClC,OAAO;QAAEiB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMjC,MAAMxF,aAAawH,KAAKpE,OAAO,EAAE;IACvC,MAAM8C,WAAqB,EAAE;IAE7B,MAAM,EAAEP,EAAE,EAAEC,MAAM8B,OAAO,EAAE,GAAGnC,iBAAiBC;IAC/C,MAAM,EAAE2B,IAAI,EAAED,UAAU,EAAE,GAAGvB,OAAO,OAAO;QAAEwB,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIP,iBAAiBa,KAAK/F,OAAO,EAAEkE,IAAIO;IAC1I,MAAMyB,SAA0D,CAAC;IAEjE,KAAK,MAAMpB,OAAOqB,OAAO1E,IAAI,CAACiE,MAAO;QACnC,IAAIxG,iBAAiB8C,GAAG,CAAC8C,MAAM;YAC7BL,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAE8D,KAAK/F,OAAO,CAAC,8BAA8B,EAAE8E,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMxF,QAAQmE,SAASiC,IAAI,CAACZ,IAAI;QAChC,IAAI,OAAOxF,UAAU,YAAYkD,kBAAkBlD,UAAU+D,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAAClE,SAAS;YAC5FmF,SAASxC,IAAI,CAAC,CAAC,SAAS,EAAE8D,KAAK/F,OAAO,CAAC,EAAE,EAAE8E,IAAI,sBAAsB,EAAExF,MAAM,8CAA8C,CAAC;QAC9H;QACA4G,MAAM,CAACpB,IAAI,GAAGxF;IAChB;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAM8G,SAAS;QAAEC,OAAOhH,cAAcqG,KAAKW,KAAK;QAAGC,SAASjH,cAAcqG,KAAKY,OAAO;QAAGC,MAAM5G,UAAUsG;IAAS;IAElH,OAAO;QACLzB,KAAK;YACHxE,SAAS+F,KAAK/F,OAAO;YACrBkC,SAAS6D,KAAK7D,OAAO;YACrBC,SAAS4D,KAAK5D,OAAO;YACrBE,MAAM0D,KAAK1D,IAAI;YACfxB,SAASkF,KAAKlF,OAAO;YACrB6E,MAAMQ;YACNT;YACAW;YACAI,WAAWL,OAAOM,WAAW,CAACT,WAAWzE,MAAM,CAAC,CAACmF,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAE/F,IAAI;qBAAE+F,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY3C,KAAKkC,SAASG,QAAQV;iBAAM;;QAChI;QACAjB;IACF;AACF"}

package/dist/esm/sql-functions.d.ts

@@ -0,0 +1,2 @@
+import type { DatabaseSync } from 'node:sqlite';
+export declare function registerFunctions(db: DatabaseSync, segmenting: boolean): void;

package/dist/esm/sql-functions.js

@@ -0,0 +1,47 @@
+import posix from 'node:path/posix';
+import { segmentMatch } from './segment.js';
+// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.
+export function registerFunctions(db, segmenting) {
+    db.function('has', {
+        deterministic: true,
+        varargs: false
+    }, (field, value)=>{
+        if (field === null || field === undefined) return 0;
+        const needle = String(value);
+        if (typeof field === 'string') {
+            if (field.startsWith('[')) {
+                try {
+                    const parsed = JSON.parse(field);
+                    if (Array.isArray(parsed)) {
+                        return parsed.some((item)=>String(item) === needle) ? 1 : 0;
+                    }
+                } catch  {}
+            }
+            return field.includes(needle) ? 1 : 0;
+        }
+        return String(field).includes(needle) ? 1 : 0;
+    });
+    // Unix basename(path, suffix?): the filename, minus suffix when it ends with one. SQLite
+    // itself has no filename function, and the LIKE tricks that fake one also match folder names.
+    db.function('basename', {
+        deterministic: true,
+        varargs: true
+    }, (path, suffix)=>{
+        if (path === null || path === undefined) return null;
+        const name = posix.basename(String(path));
+        const tail = suffix === null || suffix === undefined ? '' : String(suffix);
+        // POSIX: a suffix identical to the whole name is not removed (node's own basename strips it).
+        return tail !== '' && tail !== name && name.endsWith(tail) ? name.slice(0, -tail.length) : name;
+    });
+    // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back, and
+    // matches nothing, so the same transform is available to hand-written SQL by name. `segmenting`
+    // is the same config predicate commands.ts's search() gates on (contentTokenize(cfg) ===
+    // undefined), passed in by open() rather than read back from persisted state.
+    db.function('segment', {
+        deterministic: true,
+        varargs: false
+    }, (terms)=>{
+        const text = terms === null || terms === undefined ? '' : String(terms);
+        return segmenting ? segmentMatch(text) : text;
+    });
+}