sensemaking 0.9.5 → 0.11.0

package/dist/esm/scan.js

@@ -1,81 +1,98 @@
-import { readFileSync, statSync } from 'node:fs';
-import { join } from 'node:path';
-import fastGlob from 'fast-glob';
+import { globSync, readFileSync, statSync } from 'node:fs';
+import { join, sep } from 'node:path';
 import removeMarkdown from 'remove-markdown';
-import { parseDocument } from 'yaml';
-import { presetNames, presetSemanticEnabled } from './config.js';
+import { isCollection, parseDocument, visit } from 'yaml';
+import { embedEnabled, presetNames, presetSemanticEnabled } from './config.js';
 // Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
-// Reserved: colliding with these would clash with the `frontmatter` table's own columns or
-// the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert
-// can tell a feature-owned column (e.g. rank's `_rank`) apart from a parsed frontmatter
-// column and leave the former alone on reparse.
+// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell
+// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.
 export const RESERVED_COLUMNS = new Set([
     'path',
     '_mtime',
     '_size',
     '_rank',
+    '_parse_error',
     'content',
     'links',
     'sections'
 ]);
+// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than
+// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain
+// scalar for future use, so they can never be valid and the text can only be what was typed
+// (`aliases: [@handle]` -> ["@handle"]). Every other code has a second reading -- an unquoted
+// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks
+// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.
+const ACCEPTED_YAML_CODES = new Set([
+    'BAD_SCALAR_START'
+]);
 function normalizeText(value) {
     if (value === null || value === undefined) return '';
     return String(value).replace(/\s+/g, ' ').trim();
 }
-// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).
-// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.
+// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms
+// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.
 function stripText(value) {
     const withoutWikilinks = value.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2').replace(/\[\[([^\]]+)\]\]/g, '$1');
     const withoutMarkdown = removeMarkdown(withoutWikilinks);
     const withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
     return normalizeText(withoutTables);
 }
-// Presets are the file selection: each preset's include/exclude picks its files (fast-glob,
-// tsconfig-standard semantics, resolved relative to baseDir). Unlike the old layer model,
-// presets are views, not partitions -- they may overlap freely, and a file's set of covering
-// presets (not a single owner) is what indexing and embedding derive from. A preset whose
-// globs match nothing is valid. Files matched by no preset at all are simply not indexed.
+// Presets are views, not partitions: they overlap freely, and a file's covering set (not one
+// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.
+export function toPosixPath(relPath, separator = sep) {
+    return separator === '\\' ? relPath.split(separator).join('/') : relPath;
+}
+// Every command pays listFiles before it answers (the freshness check stats each file), so
+// per-file work here is the hottest path in the package. Everything derivable from the config
+// alone is computed once, above the loop.
+const NO_THROW = {
+    throwIfNoEntry: false
+};
 export function listFiles(cfg, baseDir) {
     const coverage = new Map();
+    const posixNeeded = sep === '\\';
     for (const name of presetNames(cfg)){
-        var _preset_exclude;
         const preset = cfg.presets[name];
-        const matched = fastGlob.sync(preset.include, {
+        for (const matched of globSync(preset.include, {
             cwd: baseDir,
-            ignore: (_preset_exclude = preset.exclude) !== null && _preset_exclude !== void 0 ? _preset_exclude : []
-        });
-        for (const relPath of matched){
+            exclude: preset.exclude
+        })){
             var _coverage_get;
+            const relPath = posixNeeded ? toPosixPath(matched) : matched;
             const set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
             set.add(name);
             coverage.set(relPath, set);
         }
     }
-    const relPaths = [
+    // Which presets want vectors is a property of the config, not of any file.
+    const embedding = embedEnabled(cfg);
+    const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name)=>presetSemanticEnabled(cfg, name))) : null;
+    const files = [];
+    for (const relPath of [
         ...coverage.keys()
-    ].sort();
-    return relPaths.map((relPath)=>{
-        const absPath = join(baseDir, relPath);
-        const st = statSync(absPath);
+    ].sort()){
+        const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls
+        // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so
+        // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).
+        const st = statSync(absPath, NO_THROW);
+        if (!(st === null || st === void 0 ? void 0 : st.isFile())) continue;
         const presets = [
             ...coverage.get(relPath)
         ].sort();
-        const embed = presets.some((name)=>presetSemanticEnabled(cfg, name));
-        return {
+        const embed = semanticPresets !== null && presets.some((name)=>semanticPresets.has(name));
+        files.push({
             relPath,
             absPath,
             mtimeMs: st.mtimeMs,
             size: st.size,
             presets,
             embed
-        };
-    });
+        });
+    }
+    return files;
 }
-// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole
-// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.
-// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`
-// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so
-// that mismatch is visible rather than a silent zero-row answer.
+// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches
+// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.
 function mapValue(value) {
     if (value === null || value === undefined) return null;
     if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
@@ -101,32 +118,90 @@ function splitFrontmatter(raw) {
         body: rest.slice(close.index + close[0].length)
     };
 }
-// Lenient by design: parseDocument collects syntax errors as data and still yields values,
-// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.
+// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid
+// YAML for a flow map used as a mapping key, so it raises no error and stores
+// {"{ date }": null}. No error code can catch that, but yaml notices the stringified key, so
+// this reports it with the path instead (yaml's own warning has none, fires once per document,
+// and is what trains readers to discard stderr).
+function warnStringifiedKeys(relPath, doc, warnings) {
+    let found = false;
+    // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside
+    // the flow map that `{{...}}` parses as.
+    visit(doc, {
+        Pair (_key, pair) {
+            if (!isCollection(pair.key)) return undefined;
+            found = true;
+            return visit.BREAK;
+        }
+    });
+    // One per file: a template repeats the same mistake on every field it stamps.
+    if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);
+}
+// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).
+// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the
+// reason. Recovering it would write values nobody wrote, which is worse than absence because
+// no query can see it. The file is still indexed -- content, links and sections never touch
+// frontmatter -- so a broken note stays searchable while it is being hunted for.
+// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus
+// the colon that introduced the part being dropped.
+function firstLine(message) {
+    return message.split('\n')[0].replace(/:\s*$/, '');
+}
 function parseFrontmatter(relPath, fm, warnings) {
-    const doc = parseDocument(fm);
-    if (doc.errors.length > 0) {
-        warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\n')[0]}); parsed leniently`);
+    // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one
+    // that carries information, with the file it came from.
+    const doc = parseDocument(fm, {
+        logLevel: 'silent'
+    });
+    const refused = doc.errors.filter((err)=>!ACCEPTED_YAML_CODES.has(err.code));
+    if (refused.length > 0) {
+        const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';
+        const parseError = `${firstLine(refused[0].message)}${detail}`;
+        warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);
+        return {
+            data: {},
+            parseError
+        };
     }
     let data;
     try {
         data = doc.toJS();
     } catch (err) {
-        warnings.push(`warning: ${relPath} has unparseable frontmatter (${err.message.split('\n')[0]}); indexing without it`);
-        return {};
+        // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on
+        // materialisation. An empty error list is not a successful parse.
+        const parseError = firstLine(err.message);
+        warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);
+        return {
+            data: {},
+            parseError
+        };
     }
-    if (data === null || data === undefined) return {};
+    if (data === null || data === undefined) return {
+        data: {},
+        parseError: null
+    };
     if (typeof data !== 'object' || Array.isArray(data)) {
-        warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);
-        return {};
+        const parseError = 'frontmatter is not a key-value mapping';
+        warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);
+        return {
+            data: {},
+            parseError
+        };
     }
-    return data;
+    warnStringifiedKeys(relPath, doc, warnings);
+    return {
+        data: data,
+        parseError: null
+    };
 }
 export function parseFile(file, extractors = []) {
     const raw = readFileSync(file.absPath, 'utf8');
     const warnings = [];
     const { fm, body: content } = splitFrontmatter(raw);
-    const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);
+    const { data, parseError } = fm === null ? {
+        data: {},
+        parseError: null
+    } : parseFrontmatter(file.relPath, fm, warnings);
     const mapped = {};
     for (const key of Object.keys(data)){
         if (RESERVED_COLUMNS.has(key)) {
@@ -149,6 +224,7 @@ export function parseFile(file, extractors = []) {
             size: file.size,
             presets: file.presets,
             data: mapped,
+            parseError,
             search,
             extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
                 var _f_extract;

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport { presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or\n// the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert\n// can tell a feature-owned column (e.g. rank's `_rank`) apart from a parsed frontmatter\n// column and leave the former alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff any covering preset has semantic !== false\n}\n\n// Presets are the file selection: each preset's include/exclude picks its files (fast-glob,\n// tsconfig-standard semantics, resolved relative to baseDir). Unlike the old layer model,\n// presets are views, not partitions -- they may overlap freely, and a file's set of covering\n// presets (not a single owner) is what indexing and embedding derive from. A preset whose\n// globs match nothing is valid. Files matched by no preset at all are simply not indexed.\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    const matched = fastGlob.sync(preset.include, { cwd: baseDir, ignore: preset.exclude ?? [] });\n    for (const relPath of matched) {\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  const relPaths = [...coverage.keys()].sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = presets.some((name) => presetSemanticEnabled(cfg, name));\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","coverage","Map","name","preset","presets","matched","sync","include","cwd","ignore","exclude","relPath","set","get","add","relPaths","keys","sort","map","absPath","st","embed","some","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","has","search","title","summary","text","extracted","fromEntries","filter","f","extract"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAErC,SAASC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAGjE,6EAA6E;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,wFAAwF;AACxF,gDAAgD;AAChD,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW,EAAE;AAE9G,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBd,eAAea;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAWA,4FAA4F;AAC5F,0FAA0F;AAC1F,6FAA6F;AAC7F,0FAA0F;AAC1F,0FAA0F;AAC1F,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,KAAK,MAAMC,QAAQnB,YAAYe,KAAM;YAEmCK;QADtE,MAAMA,SAASL,IAAIM,OAAO,CAACF,KAAK;QAChC,MAAMG,UAAUzB,SAAS0B,IAAI,CAACH,OAAOI,OAAO,EAAE;YAAEC,KAAKT;YAASU,MAAM,GAAEN,kBAAAA,OAAOO,OAAO,cAAdP,6BAAAA,kBAAkB,EAAE;QAAC;QAC3F,KAAK,MAAMQ,WAAWN,QAAS;gBACjBL;YAAZ,MAAMY,OAAMZ,gBAAAA,SAASa,GAAG,CAACF,sBAAbX,2BAAAA,gBAAyB,IAAId;YACzC0B,IAAIE,GAAG,CAACZ;YACRF,SAASY,GAAG,CAACD,SAASC;QACxB;IACF;IAEA,MAAMG,WAAW;WAAIf,SAASgB,IAAI;KAAG,CAACC,IAAI;IAC1C,OAAOF,SAASG,GAAG,CAAC,CAACP;QACnB,MAAMQ,UAAUxC,KAAKoB,SAASY;QAC9B,MAAMS,KAAK1C,SAASyC;QACpB,MAAMf,UAAU;eAAKJ,SAASa,GAAG,CAACF;SAAyB,CAACM,IAAI;QAChE,MAAMI,QAAQjB,QAAQkB,IAAI,CAAC,CAACpB,OAASlB,sBAAsBc,KAAKI;QAChE,OAAO;YAAES;YAASQ;YAASI,SAASH,GAAGG,OAAO;YAAEC,MAAMJ,GAAGI,IAAI;YAAEpB;YAASiB;QAAM;IAChF;AACF;AAcA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASI,SAASrC,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOsC,OAAOtC,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOuC,OAAOC,aAAa,CAACxC,SAASsC,OAAOtC,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOyC,KAAKC,SAAS,CAAC1C;AACxB;AAEA,gEAAgE;AAChE,SAAS2C,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKpD,WAAW,OAAO;QAAE8C,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiB/B,OAAe,EAAEwB,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM9D,cAAcqD;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,iBAAiB,EAAEiC,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,8BAA8B,EAAE,AAACwC,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS5D,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO4D,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAOsC;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMvD,aAAa8E,KAAKpC,OAAO,EAAE;IACvC,MAAMwB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAK5C,OAAO,EAAEwB,IAAIQ;IACnE,MAAMe,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAO5C,IAAI,CAACiC,MAAO;QACnC,IAAIhE,iBAAiB4E,GAAG,CAACF,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAK5C,OAAO,CAAC,8BAA8B,EAAEgD,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGlC,SAASwB,IAAI,CAACU,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMG,SAAS;QAAEC,OAAO5E,cAAc8D,KAAKc,KAAK;QAAGC,SAAS7E,cAAc8D,KAAKe,OAAO;QAAGC,MAAMxE,UAAUgE;IAAS;IAElH,OAAO;QACLb,KAAK;YACHjC,SAAS4C,KAAK5C,OAAO;YACrBY,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACfpB,SAASmD,KAAKnD,OAAO;YACrB6C,MAAMS;YACNI;YACAI,WAAWN,OAAOO,WAAW,CAACX,WAAWY,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEpD,GAAG,CAAC,CAACmD;oBAAeA;uBAAT;oBAACA,EAAEnE,IAAI;qBAAEmE,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYrC,KAAKyB,SAASK;iBAAQ;;QAC1H;QACAnB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["globSync","readFileSync","statSync","join","sep","removeMarkdown","isCollection","parseDocument","visit","embedEnabled","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","add","embedding","semanticPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","warnStringifiedKeys","doc","warnings","found","Pair","_key","pair","key","BREAK","firstLine","message","parseFrontmatter","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","parseFile","file","extractors","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AAC3D,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AACtC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,SAASC,YAAY,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAG/E,6EAA6E;AAE7E,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAE9H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,mGAAmG;AACnG,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBlB,eAAeiB;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAWA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBvB,GAAG;IAClE,OAAOuB,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWxB,IAAI,CAAC,OAAOuB;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAchC,QAAQ;IAC5B,KAAK,MAAMiC,QAAQ3B,YAAYsB,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWxC,SAASsC,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIrB;YACzC+B,IAAIE,GAAG,CAACT;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMG,YAAYtC,aAAauB;IAC/B,MAAMgB,kBAAkBD,YAAY,IAAIlC,IAAIH,YAAYsB,KAAKiB,MAAM,CAAC,CAACZ,OAAS1B,sBAAsBqB,KAAKK,UAAU;IAEnH,MAAMa,QAAoB,EAAE;IAC5B,KAAK,MAAMxB,WAAW;WAAIQ,SAASiB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlD,KAAK8B,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM4B,KAAKpD,SAASmD,SAASxB;QAC7B,IAAI,EAACyB,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMhB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC0B,IAAI;QAChE,MAAMI,QAAQR,oBAAoB,QAAQT,QAAQkB,IAAI,CAAC,CAACpB,OAASW,gBAAgBU,GAAG,CAACrB;QACrFa,MAAMS,IAAI,CAAC;YAAEjC;YAAS2B;YAASO,SAASN,GAAGM,OAAO;YAAEC,MAAMP,GAAGO,IAAI;YAAEtB;YAASiB;QAAM;IACpF;IACA,OAAON;AACT;AAiBA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASY,SAAS9C,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAO+C,OAAO/C,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOgD,OAAOC,aAAa,CAACjD,SAAS+C,OAAO/C,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOkD,KAAKC,SAAS,CAACnD;AACxB;AAEA,gEAAgE;AAChE,SAASoD,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAK7D,WAAW,OAAO;QAAEuD,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASG,oBAAoBrD,OAAe,EAAEsD,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC1E,MAAMwE,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAAC/E,aAAa+E,KAAKC,GAAG,GAAG,OAAOrE;YACpCiE,QAAQ;YACR,OAAO1E,MAAM+E,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAAS8D,UAAUC,OAAe;IAChC,OAAOA,QAAQ7D,KAAK,CAAC,KAAK,CAAC,EAAE,CAACT,OAAO,CAAC,SAAS;AACjD;AAEA,SAASuE,iBAAiBhE,OAAe,EAAE8C,EAAU,EAAES,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMzE,cAAciE,IAAI;QAAEmB,UAAU;IAAS;IACnD,MAAMC,UAAUZ,IAAIa,MAAM,CAAC5C,MAAM,CAAC,CAAC6C,MAAQ,CAAChF,oBAAoB4C,GAAG,CAACoC,IAAIC,IAAI;IAC5E,IAAIH,QAAQhB,MAAM,GAAG,GAAG;QACtB,MAAMoB,SAASJ,QAAQhB,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEgB,QAAQhB,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMqB,aAAa,GAAGT,UAAUI,OAAO,CAAC,EAAE,CAACH,OAAO,IAAIO,QAAQ;QAC9Df,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuE,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOlB,IAAImB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMG,aAAaT,UAAU,AAACM,IAAcL,OAAO;QACnDR,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuE,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASjF,WAAW,OAAO;QAAEiF,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBhB,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,CAAC,EAAEuE,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAlB,oBAAoBrD,SAASsD,KAAKC;IAClC,OAAO;QAAEiB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMnC,MAAMpE,aAAasG,KAAKlD,OAAO,EAAE;IACvC,MAAM4B,WAAqB,EAAE;IAE7B,MAAM,EAAET,EAAE,EAAEC,MAAMgC,OAAO,EAAE,GAAGrC,iBAAiBC;IAC/C,MAAM,EAAE6B,IAAI,EAAED,UAAU,EAAE,GAAGzB,OAAO,OAAO;QAAE0B,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIP,iBAAiBa,KAAK7E,OAAO,EAAE8C,IAAIS;IAC1I,MAAMyB,SAA0D,CAAC;IAEjE,KAAK,MAAMpB,OAAOqB,OAAOxD,IAAI,CAAC+C,MAAO;QACnC,IAAItF,iBAAiB8C,GAAG,CAAC4B,MAAM;YAC7BL,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAE4C,KAAK7E,OAAO,CAAC,8BAA8B,EAAE4D,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAoB,MAAM,CAACpB,IAAI,GAAGxB,SAASoC,IAAI,CAACZ,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMsB,SAAS;QAAEC,OAAO9F,cAAcmF,KAAKW,KAAK;QAAGC,SAAS/F,cAAcmF,KAAKY,OAAO;QAAGC,MAAM1F,UAAUoF;IAAS;IAElH,OAAO;QACLzB,KAAK;YACHtD,SAAS6E,KAAK7E,OAAO;YACrBkC,SAAS2C,KAAK3C,OAAO;YACrBC,MAAM0C,KAAK1C,IAAI;YACftB,SAASgE,KAAKhE,OAAO;YACrB2D,MAAMQ;YACNT;YACAW;YACAI,WAAWL,OAAOM,WAAW,CAACT,WAAWvD,MAAM,CAAC,CAACiE,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAE7E,IAAI;qBAAE6E,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY7C,KAAKoC,SAASG;iBAAQ;;QAC1H;QACA3B;IACF;AACF"}

package/dist/esm/search-error.js

@@ -1,8 +1,6 @@
 import { SenseError } from './errors.js';
-// FTS5 reads these as operators, so a bare term containing one is a syntax error or --
-// worse -- a column filter: `end-to-end` parses as a filter on column `to`, and SQLite
-// reports `no such column: to`, which is a true statement about the parse and a false
-// one about the input. Both field reports on 0.6.0 misdiagnosed that message.
+// FTS5 reads these as operators, so `end-to-end` parses as a filter on column `to` and errors
+// `no such column: to` -- true about the parse, misleading about the input.
 const FTS5_OPERATORS = /[-'"/.:^*()]/;
 export function searchError(err, terms, scope) {
     var _terms_match;
@@ -19,7 +17,7 @@ export function searchError(err, terms, scope) {
         return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t)=>`\`${t}\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '"${suspects[0]}"'. Searchable columns are title, summary, text.`);
     }
     if (col !== undefined && scope !== undefined) {
-        return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense query "SELECT name FROM pragma_table_info('frontmatter')".`);
+        return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql "SELECT name FROM pragma_table_info('frontmatter')".`);
     }
-    return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense query (list them with pragma_table_info('frontmatter')).`);
+    return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);
 }

package/dist/esm/search-error.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/search-error.ts"],"sourcesContent":["import { SenseError } from './errors.ts';\n\n// FTS5 reads these as operators, so a bare term containing one is a syntax error or --\n// worse -- a column filter: `end-to-end` parses as a filter on column `to`, and SQLite\n// reports `no such column: to`, which is a true statement about the parse and a false\n// one about the input. Both field reports on 0.6.0 misdiagnosed that message.\nconst FTS5_OPERATORS = /[-'\"/.:^*()]/;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_OPERATORS.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where (or the tree's default scope) raises \"no such column\" through this same\n  // statement, and naming a term for it would state a false fact about the input.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense query \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense query (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","FTS5_OPERATORS","searchError","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","map","join"],"mappings":"AAAA,SAASA,UAAU,QAAQ,cAAc;AAEzC,uFAAuF;AACvF,uFAAuF;AACvF,sFAAsF;AACtF,8EAA8E;AAC9E,MAAMC,iBAAiB;AAEvB,OAAO,SAASC,YAAYC,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAIN;IANZ,MAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,MAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQX,eAAeM,IAAI,CAACI;IACrG,wFAAwF;IACxF,qFAAqF;IACrF,gFAAgF;IAChF,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,CAACP,IAAMA,EAAEQ,KAAK,CAAC,mBAAmBC,QAAQ,CAACP;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIjB,WAAW,iBAAiB,GAAGM,QAAQ,uBAAuB,EAAEE,SAASa,GAAG,CAAC,CAACV,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEW,IAAI,CAAC,MAAM,gFAAgF,EAAEd,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIhB,WAAW,iBAAiB,GAAGM,QAAQ,yBAAyB,EAAED,MAAM,mHAAmH,CAAC;IACzM;IACA,OAAO,IAAIL,WAAW,iBAAiB,GAAGM,QAAQ,8JAA8J,CAAC;AACnN"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/search-error.ts"],"sourcesContent":["import { SenseError } from './errors.ts';\n\n// FTS5 reads these as operators, so `end-to-end` parses as a filter on column `to` and errors\n// `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_OPERATORS = /[-'\"/.:^*()]/;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_OPERATORS.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where (or the tree's default scope) raises \"no such column\" through this same\n  // statement, and naming a term for it would state a false fact about the input.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["SenseError","FTS5_OPERATORS","searchError","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","map","join"],"mappings":"AAAA,SAASA,UAAU,QAAQ,cAAc;AAEzC,8FAA8F;AAC9F,4EAA4E;AAC5E,MAAMC,iBAAiB;AAEvB,OAAO,SAASC,YAAYC,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAIN;IANZ,MAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,MAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEC,UAAU,CAAC,QAAQX,eAAeM,IAAI,CAACI;IACrG,wFAAwF;IACxF,qFAAqF;IACrF,gFAAgF;IAChF,MAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,MAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,CAACP,IAAMA,EAAEQ,KAAK,CAAC,mBAAmBC,QAAQ,CAACP;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAIjB,WAAW,iBAAiB,GAAGM,QAAQ,uBAAuB,EAAEE,SAASa,GAAG,CAAC,CAACV,IAAM,CAAC,EAAE,EAAEA,EAAE,EAAE,CAAC,EAAEW,IAAI,CAAC,MAAM,gFAAgF,EAAEd,QAAQ,CAAC,EAAE,CAAC,gDAAgD,CAAC;IACvQ;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIhB,WAAW,iBAAiB,GAAGM,QAAQ,yBAAyB,EAAED,MAAM,iHAAiH,CAAC;IACvM;IACA,OAAO,IAAIL,WAAW,iBAAiB,GAAGM,QAAQ,4JAA4J,CAAC;AACjN"}

package/dist/esm/traverse.d.ts

@@ -0,0 +1,8 @@
+import type { DatabaseSync } from 'node:sqlite';
+interface FindPathOptions {
+    directed?: boolean;
+    maxDepth?: number;
+    allowed?: Set<string>;
+}
+export declare function findPath(db: DatabaseSync, from: string, to: string, opts?: FindPathOptions): string[] | null;
+export {};

package/dist/esm/traverse.js

@@ -0,0 +1,71 @@
+// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a
+// local question touches a neighborhood, never the whole edge list.
+function resetVisited(db) {
+    db.exec('DROP TABLE IF EXISTS temp.visited');
+    db.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');
+}
+// Materializes the caller-resolved scope Set into a temp table so it joins like any other
+// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.
+function setupAllowed(db, allowed, extra) {
+    db.exec('DROP TABLE IF EXISTS temp.allowed_nodes');
+    db.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');
+    db.prepare('INSERT INTO allowed_nodes SELECT DISTINCT value FROM json_each(?1)').run(JSON.stringify([
+        ...allowed
+    ]));
+    if (extra !== undefined) db.prepare('INSERT OR IGNORE INTO allowed_nodes VALUES (?)').run(extra);
+}
+// Only resolved links (dst IS NOT NULL) are followed, in either direction.
+function candidateSelect(direction) {
+    const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';
+    const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';
+    if (direction === 'forward') return forward;
+    if (direction === 'reverse') return reverse;
+    return `${forward} UNION ALL ${reverse}`;
+}
+// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against
+// everything already visited, optionally filtered to the allowed set, capped (?3) and
+// inserted at the new depth (?2). RETURNING hands back exactly what this ring newly reached.
+function ringSql(direction, hasAllowed) {
+    return `
+INSERT INTO visited (path, depth, pred)
+SELECT cand.path, ?2, MIN(cand.pred)
+FROM (${candidateSelect(direction)}) cand
+WHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)
+${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}
+GROUP BY cand.path
+ORDER BY cand.path
+LIMIT ?3
+RETURNING path`;
+}
+function reconstructPath(db, to) {
+    const lookup = db.prepare('SELECT pred FROM visited WHERE path = ?');
+    const path = [];
+    let cur = to;
+    while(cur !== null){
+        path.push(cur);
+        const row = lookup.get(cur);
+        cur = row ? row.pred : null;
+    }
+    return path.reverse();
+}
+// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching
+// personalizedRank; null when `to` is unreached within `maxDepth`.
+export function findPath(db, from, to, opts = {}) {
+    var _opts_maxDepth;
+    if (from === to) return [
+        from
+    ];
+    const direction = opts.directed ? 'forward' : 'both';
+    const maxDepth = (_opts_maxDepth = opts.maxDepth) !== null && _opts_maxDepth !== void 0 ? _opts_maxDepth : Infinity;
+    if (maxDepth <= 0) return null;
+    resetVisited(db);
+    db.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)').run(from);
+    if (opts.allowed) setupAllowed(db, opts.allowed, to);
+    const stmt = db.prepare(ringSql(direction, !!opts.allowed));
+    for(let ring = 1; ring <= maxDepth; ring++){
+        const rows = stmt.all(ring - 1, ring, -1);
+        if (rows.length === 0) break;
+        if (rows.some((row)=>row.path === to)) return reconstructPath(db, to);
+    }
+    return null;
+}

package/dist/esm/traverse.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\nimport type { DatabaseSync } from 'node:sqlite';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\ninterface FindPathOptions {\n  directed?: boolean;\n  maxDepth?: number;\n  allowed?: Set<string>;\n}\n\nfunction resetVisited(db: DatabaseSync): void {\n  db.exec('DROP TABLE IF EXISTS temp.visited');\n  db.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');\n}\n\n// Materializes the caller-resolved scope Set into a temp table so it joins like any other\n// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.\nfunction setupAllowed(db: DatabaseSync, allowed: Set<string>, extra?: string): void {\n  db.exec('DROP TABLE IF EXISTS temp.allowed_nodes');\n  db.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');\n  db.prepare('INSERT INTO allowed_nodes SELECT DISTINCT value FROM json_each(?1)').run(JSON.stringify([...allowed]));\n  if (extra !== undefined) db.prepare('INSERT OR IGNORE INTO allowed_nodes VALUES (?)').run(extra);\n}\n\n// Only resolved links (dst IS NOT NULL) are followed, in either direction.\nfunction candidateSelect(direction: Direction): string {\n  const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  if (direction === 'forward') return forward;\n  if (direction === 'reverse') return reverse;\n  return `${forward} UNION ALL ${reverse}`;\n}\n\n// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against\n// everything already visited, optionally filtered to the allowed set, capped (?3) and\n// inserted at the new depth (?2). RETURNING hands back exactly what this ring newly reached.\nfunction ringSql(direction: Direction, hasAllowed: boolean): string {\n  return `\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (${candidateSelect(direction)}) cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}\nGROUP BY cand.path\nORDER BY cand.path\nLIMIT ?3\nRETURNING path`;\n}\n\nfunction reconstructPath(db: DatabaseSync, to: string): string[] {\n  const lookup = db.prepare('SELECT pred FROM visited WHERE path = ?');\n  const path: string[] = [];\n  let cur: string | null = to;\n  while (cur !== null) {\n    path.push(cur);\n    const row = lookup.get(cur) as { pred: string | null } | undefined;\n    cur = row ? row.pred : null;\n  }\n  return path.reverse();\n}\n\n// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching\n// personalizedRank; null when `to` is unreached within `maxDepth`.\nexport function findPath(db: DatabaseSync, from: string, to: string, opts: FindPathOptions = {}): string[] | null {\n  if (from === to) return [from];\n\n  const direction: Direction = opts.directed ? 'forward' : 'both';\n  const maxDepth = opts.maxDepth ?? Infinity;\n  if (maxDepth <= 0) return null;\n\n  resetVisited(db);\n  db.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)').run(from);\n  if (opts.allowed) setupAllowed(db, opts.allowed, to);\n\n  const stmt = db.prepare(ringSql(direction, !!opts.allowed));\n  for (let ring = 1; ring <= maxDepth; ring++) {\n    const rows = stmt.all(ring - 1, ring, -1) as Array<{ path: string }>;\n    if (rows.length === 0) break;\n    if (rows.some((row) => row.path === to)) return reconstructPath(db, to);\n  }\n  return null;\n}\n"],"names":["resetVisited","db","exec","setupAllowed","allowed","extra","prepare","run","JSON","stringify","undefined","candidateSelect","direction","forward","reverse","ringSql","hasAllowed","reconstructPath","to","lookup","path","cur","push","row","get","pred","findPath","from","opts","directed","maxDepth","Infinity","stmt","ring","rows","all","length","some"],"mappings":"AAAA,2FAA2F;AAC3F,oEAAoE;AAWpE,SAASA,aAAaC,EAAgB;IACpCA,GAAGC,IAAI,CAAC;IACRD,GAAGC,IAAI,CAAC;AACV;AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,SAASC,aAAaF,EAAgB,EAAEG,OAAoB,EAAEC,KAAc;IAC1EJ,GAAGC,IAAI,CAAC;IACRD,GAAGC,IAAI,CAAC;IACRD,GAAGK,OAAO,CAAC,sEAAsEC,GAAG,CAACC,KAAKC,SAAS,CAAC;WAAIL;KAAQ;IAChH,IAAIC,UAAUK,WAAWT,GAAGK,OAAO,CAAC,kDAAkDC,GAAG,CAACF;AAC5F;AAEA,2EAA2E;AAC3E,SAASM,gBAAgBC,SAAoB;IAC3C,MAAMC,UAAU;IAChB,MAAMC,UAAU;IAChB,IAAIF,cAAc,WAAW,OAAOC;IACpC,IAAID,cAAc,WAAW,OAAOE;IACpC,OAAO,GAAGD,QAAQ,WAAW,EAAEC,SAAS;AAC1C;AAEA,0FAA0F;AAC1F,sFAAsF;AACtF,6FAA6F;AAC7F,SAASC,QAAQH,SAAoB,EAAEI,UAAmB;IACxD,OAAO,CAAC;;;MAGJ,EAAEL,gBAAgBC,WAAW;;AAEnC,EAAEI,aAAa,wEAAwE,GAAG;;;;cAI5E,CAAC;AACf;AAEA,SAASC,gBAAgBhB,EAAgB,EAAEiB,EAAU;IACnD,MAAMC,SAASlB,GAAGK,OAAO,CAAC;IAC1B,MAAMc,OAAiB,EAAE;IACzB,IAAIC,MAAqBH;IACzB,MAAOG,QAAQ,KAAM;QACnBD,KAAKE,IAAI,CAACD;QACV,MAAME,MAAMJ,OAAOK,GAAG,CAACH;QACvBA,MAAME,MAAMA,IAAIE,IAAI,GAAG;IACzB;IACA,OAAOL,KAAKN,OAAO;AACrB;AAEA,wFAAwF;AACxF,mEAAmE;AACnE,OAAO,SAASY,SAASzB,EAAgB,EAAE0B,IAAY,EAAET,EAAU,EAAEU,OAAwB,CAAC,CAAC;QAI5EA;IAHjB,IAAID,SAAST,IAAI,OAAO;QAACS;KAAK;IAE9B,MAAMf,YAAuBgB,KAAKC,QAAQ,GAAG,YAAY;IACzD,MAAMC,YAAWF,iBAAAA,KAAKE,QAAQ,cAAbF,4BAAAA,iBAAiBG;IAClC,IAAID,YAAY,GAAG,OAAO;IAE1B9B,aAAaC;IACbA,GAAGK,OAAO,CAAC,+DAA+DC,GAAG,CAACoB;IAC9E,IAAIC,KAAKxB,OAAO,EAAED,aAAaF,IAAI2B,KAAKxB,OAAO,EAAEc;IAEjD,MAAMc,OAAO/B,GAAGK,OAAO,CAACS,QAAQH,WAAW,CAAC,CAACgB,KAAKxB,OAAO;IACzD,IAAK,IAAI6B,OAAO,GAAGA,QAAQH,UAAUG,OAAQ;QAC3C,MAAMC,OAAOF,KAAKG,GAAG,CAACF,OAAO,GAAGA,MAAM,CAAC;QACvC,IAAIC,KAAKE,MAAM,KAAK,GAAG;QACvB,IAAIF,KAAKG,IAAI,CAAC,CAACd,MAAQA,IAAIH,IAAI,KAAKF,KAAK,OAAOD,gBAAgBhB,IAAIiB;IACtE;IACA,OAAO;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.9.5",
+  "version": "0.11.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -14,6 +14,11 @@
     "hybrid-search",
     "knowledge-base",
     "rag",
+    "progressive-disclosure",
+    "graph-traversal",
+    "link-graph",
+    "wikilinks",
+    "backlinks",
     "markdown-database",
     "document-os",
     "notes",
@@ -67,7 +72,6 @@
   },
   "dependencies": {
     "@huggingface/tokenizers": "^0.1.3",
-    "fast-glob": "^3.3.3",
     "remove-markdown": "^0.6.4",
     "yaml": "^2.9.0"
   },
@@ -79,6 +83,6 @@
     "tsds-config": "*"
   },
   "engines": {
-    "node": ">=22.16"
+    "node": ">=22.20"
   }
 }

package/schema.json

@@ -12,12 +12,12 @@
     },
     "version": {
       "type": "integer",
-      "enum": [1, 2, 3],
+      "enum": [1, 2, 3, 4],
       "description": "Config format version. Older versions are auto-migrated (and the file rewritten) on load; a version newer than this sense build supports makes it exit with an error rather than misinterpret the file. Omit to default to 1."
     },
     "presets": {
       "type": "object",
-      "description": "File selection, index-time, and named search defaults. A preset is self-contained: no inheritance between presets, and no ordering -- presets may overlap freely (they are views, not partitions), and a preset whose globs match nothing is valid. A file is indexed iff any preset's include/exclude covers it (union across presets); it is embedded iff any covering preset has semantic on. `default` is used when a command names no preset, and must be declared. `sense init` generates `default` plus one `large` example.",
+      "description": "File selection, index-time, and named search defaults. A preset is self-contained: no inheritance between presets, and no ordering -- presets may overlap freely (they are views, not partitions), and a preset whose globs match nothing is valid. A file is indexed iff any preset's include/exclude covers it (union across presets); `default` is used when a command names no preset, and must be declared. A file is embedded when the top-level `embed` block names a model and some covering preset has `semantic` on. `sense init` generates `default` plus one `large` example.",
       "minProperties": 1,
       "additionalProperties": {
         "type": "object",
@@ -36,15 +36,15 @@
             "minItems": 1,
             "description": "Glob patterns excluded from this preset's `include`, tsconfig-standard semantics."
           },
+          "semantic": {
+            "type": "boolean",
+            "description": "Vectors for the files this preset covers, and for searches scoped to it. Absent or true means on; only ever meaningfully written as false. A layer searched for exact wording (ingested sources, archives, generated output) sets it false: its files cost no embedding, and searches under it run on words and links. Requires the top-level `embed` block to name a model before any preset can use vectors."
+          },
           "k": {
             "type": "integer",
             "minimum": 1,
             "description": "Default result count for `search` when this preset is in effect (named, or `default` when none is). A --k on the invocation, or a saved search's own `k`, overrides it. Defaults to 10 when the preset omits it too."
           },
-          "semantic": {
-            "type": "boolean",
-            "description": "Vector participation for files this preset covers. Absent or true means on (the default); only ever meaningfully written as false. A file covered only by semantic-false presets is indexed but never embedded, so it searches lexically automatically -- no error, just fewer signals."
-          },
           "where": {
             "type": "string",
             "description": "Standing SQL condition against frontmatter alias `f`, applied when `search` runs under this preset without an explicit --where. An explicit --where (or a saved search's own `where`) replaces it rather than ANDing with it, so a caller can always widen back to the preset's whole scope."
@@ -55,7 +55,7 @@
     "features": {
       "type": "object",
       "additionalProperties": false,
-      "description": "Global feature defaults. Absent block or key means enabled. `embed` is not a member of this block -- vector participation is controlled per preset by `semantic` (see \"presets\" and the top-level \"embed\" block for provider settings). Toggling a feature rebuilds the cache. Commands degrade rather than fail: `search` without links is BM25-only, `map` without rank omits hubs, `peek` without sections omits the outline.",
+      "description": "Global feature defaults. Absent block or key means enabled. `embed` is not a member of this block -- vectors are on exactly when the top-level \"embed\" block names a model. Toggling a feature rebuilds the cache. Commands degrade rather than fail: `search` without links is BM25-only, `map` without rank omits hubs, `peek` without sections omits the outline.",
       "properties": {
         "links": {
           "type": "boolean",
@@ -74,7 +74,8 @@
     "embed": {
       "type": "object",
       "additionalProperties": false,
-      "description": "Embed provider settings only -- presence alone does not enable embedding. Vector participation is decided per preset by that preset's `semantic` field; this block only says which model/provider to use when at least one preset wants vectors. `model` (Hugging Face id or local path, default minishlab/potion-retrieval-32M), `type` (`static` = built-in pure-JS Model2Vec loader, default; `api` = OpenAI-compatible POST /embeddings), `url` (api base, e.g. http://localhost:11434/v1), `key` (name of the env var holding the bearer token). Changing model or type rebuilds the cache like a feature toggle.",
+      "description": "The vector switch and the model it uses. Present with a `model` means every indexed file gets vectors; omit the block entirely to index without vectors. `model` (required: Hugging Face id or local path), `type` (`static` = built-in pure-JS Model2Vec loader, default; `api` = OpenAI-compatible POST /embeddings), `url` (api base, e.g. http://localhost:11434/v1), `key` (name of the env var holding the bearer token). Nothing downloads the model implicitly -- run `sense download` once per machine. Changing model or type rebuilds the cache like a feature toggle.",
+      "required": ["model"],
       "properties": {
         "model": { "type": "string" },
         "type": { "type": "string", "enum": ["static", "api"] },
@@ -84,16 +85,15 @@
     },
     "queries": {
       "type": "object",
-      "description": "Named queries runnable as `sense <name> [params...]`, one of three shapes: a raw SQL string, `{ sql }`, or a saved search. SQL (string or `{ sql }`): `?` placeholders bind to CLI positional args in order; deterministic and enumerating, including raw FTS5 `MATCH` for word search under your own SQL -- \"0 rows = not in the tree\" lives here. Tables: `frontmatter` (one row per file, one column per discovered frontmatter key, plus `path`/`_mtime`/`_size`/`_rank`), `content` (FTS5: `title`, `summary`, `text`, `path`), `links` (`src`, `target`, `dst`), `sections` (`path`, `idx`, `level`, `heading`, `start_line`, `end_line`, `tokens`), and `preset_files` (`preset`, `path`) -- which presets cover which files. `has(field, value)`: array membership on a JSON-array field, substring match on a string (so has(f.status, 'active') also matches 'inactive'), false on NULL. Exact matches: `=` for scalars, `EXISTS (SELECT 1 FROM json_each(f.tags) WHERE value = ?)` for array members. Canonical query: `SELECT f.path, content.title, content.summary, CASE WHEN length(content.text) <= 16384 THEN snippet(content, -1, '«', '»', '…', 10) END AS hit FROM frontmatter f JOIN content ON content.path = f.path WHERE content MATCH ? ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10`. The CASE bounds snippet(), which re-tokenizes each matched document and costs seconds per query once a tree holds a megabyte-scale note; `search` applies the same bound internally. Saved search object: one text driving every engine the scoped preset has -- word match, links, vectors -- fused into one ranked list, `via` labeling which engine produced each row; `search` text must be non-empty (a saved query saves a question -- a scope without one is just flags). `preset` names one declared preset (defaults to `default`); `include` is an ad hoc glob scope that replaces the preset's include/exclude entirely, same as `search --include`; `where` and `k` behave like the `search` command's flags. `semantic: false` opts a query down to lexical only -- the rare precision opt-out, not a gate (semantic participation otherwise follows the scoped preset's `semantic` automatically). `sense <name>` behaves like `sense search <search> [--preset] [--include] [--where] [--k] [--lexical]` with zero flags; it takes no positional parameters. `sense check` probes every saved query/search (lexically, k=1 for searches) so a broken one fails loudly at check time -- it makes no assertion about the result itself; a returned row set is the reader's judgment. Reserved frontmatter keys: `path`, `_mtime`, `_size`, `_rank`, `content`, `links`, `sections`. Reserved query names (unreachable as subcommands, any shape): `init`, `query`, `search`, `find`, `map`, `peek`, `watch`, `status`, `rebuild`, `check`.",
+      "description": "Saved queries runnable as `sense <name> [params...]`, each naming the verb it runs, one to one with the two commands: `{ sql }` runs like `sense sql`, `{ search }` like `sense search`. A bare string is rejected -- it silently meant SQL. `{ sql }`: `?` placeholders bind to CLI positional args in order; deterministic and enumerating, including raw FTS5 `MATCH` for word search under your own SQL -- \"0 rows = not in the tree\" lives here. Tables: `frontmatter` (one row per file, one column per discovered frontmatter key, plus `path`/`_mtime`/`_size`/`_rank`/`_parse_error`, the last being NULL when the frontmatter parsed and the YAML message when it did not, in which case no other column is populated), `content` (FTS5: `title`, `summary`, `text`, `path`), `links` (`src`, `target`, `dst`), `sections` (`path`, `idx`, `level`, `heading`, `start_line`, `end_line`, `tokens`), and `preset_files` (`preset`, `path`) -- which presets cover which files. `has(field, value)`: array membership on a JSON-array field, substring match on a string (so has(f.status, 'active') also matches 'inactive'), false on NULL. Exact matches: `=` for scalars, `EXISTS (SELECT 1 FROM json_each(f.tags) WHERE value = ?)` for array members. Canonical query: `SELECT f.path, content.title, content.summary, CASE WHEN length(content.text) <= 16384 THEN snippet(content, -1, '«', '»', '…', 10) END AS hit FROM frontmatter f JOIN content ON content.path = f.path WHERE content MATCH ? ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10`. The CASE bounds snippet(), which re-tokenizes each matched document and costs seconds per query once a tree holds a megabyte-scale note; `search` applies the same bound internally. Saved search object: one text driving every engine the scoped preset has -- word match, links, vectors -- fused into one ranked list, `via` labeling which engine produced each row; `search` text must be non-empty (a saved query saves a question -- a scope without one is just flags). `preset` names one declared preset (defaults to `default`); `include` is an ad hoc glob scope that replaces the preset's include/exclude entirely, same as `search --include`; `where` and `k` behave like the `search` command's flags. `sense <name>` behaves like `sense search <search> [--preset] [--include] [--where] [--k]` with zero flags; it takes no positional parameters. Running an entry validates it: a typo'd column or unknown preset errors and exits nonzero, and a parameterised entry validates with any argument, since SQL is prepared before parameters bind. Nothing asserts on the result itself; a returned row set is the reader's judgment. Reserved frontmatter keys: `path`, `_mtime`, `_size`, `_rank`, `_parse_error`, `content`, `links`, `sections`. Reserved names (unreachable as saved queries, any shape): `init`, `sql`, `search`, `map`, `peek`, `path`, `related`, `download`, `watch`, `status`.",
       "additionalProperties": {
         "oneOf": [
-          { "type": "string" },
           {
             "type": "object",
             "additionalProperties": false,
             "required": ["sql"],
             "properties": {
-              "sql": { "type": "string", "description": "Raw SQL, same as the string shape -- `?` placeholders bind to CLI positional args in order." }
+              "sql": { "type": "string", "description": "Raw SQL, run like `sense sql`. `?` placeholders bind to CLI positional args in order." }
             }
           },
           {
@@ -117,11 +117,7 @@
                 "description": "Ad hoc glob scope, same as `search --include`. Replaces the preset's include/exclude entirely rather than layering on top of it."
               },
               "k": { "type": "integer", "minimum": 1, "description": "Result count, same as `search --k`. Defaults to the preset's own `k`, then 10. A --k on the invocation overrides this." },
-              "where": { "type": "string", "description": "SQL condition against frontmatter alias `f`, same as `search --where`. A --where on the invocation replaces this rather than ANDing with it." },
-              "semantic": {
-                "type": "boolean",
-                "description": "false opts this query down to lexical-only, same as `search --lexical` -- the rare precision opt-out. Semantic participation otherwise follows the scoped preset's `semantic` automatically, so true is redundant with the default."
-              }
+              "where": { "type": "string", "description": "SQL condition against frontmatter alias `f`, same as `search --where`. A --where on the invocation replaces this rather than ANDing with it." }
             }
           }
         ]