sensemaking 0.9.5 → 0.11.0

package/dist/cjs/scan.js

@@ -17,11 +17,13 @@ _export(exports, {
     },
     get parseFile () {
         return parseFile;
+    },
+    get toPosixPath () {
+        return toPosixPath;
     }
 });
 var _nodefs = require("node:fs");
 var _nodepath = require("node:path");
-var _fastglob = /*#__PURE__*/ _interop_require_default(require("fast-glob"));
 var _removemarkdown = /*#__PURE__*/ _interop_require_default(require("remove-markdown"));
 var _yaml = require("yaml");
 var _configts = require("./config.js");
@@ -64,39 +66,59 @@ var RESERVED_COLUMNS = new Set([
     '_mtime',
     '_size',
     '_rank',
+    '_parse_error',
     'content',
     'links',
     'sections'
 ]);
+// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than
+// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain
+// scalar for future use, so they can never be valid and the text can only be what was typed
+// (`aliases: [@handle]` -> ["@handle"]). Every other code has a second reading -- an unquoted
+// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks
+// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.
+var ACCEPTED_YAML_CODES = new Set([
+    'BAD_SCALAR_START'
+]);
 function normalizeText(value) {
     if (value === null || value === undefined) return '';
     return String(value).replace(/\s+/g, ' ').trim();
 }
-// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).
-// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.
+// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms
+// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.
 function stripText(value) {
     var withoutWikilinks = value.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2').replace(/\[\[([^\]]+)\]\]/g, '$1');
     var withoutMarkdown = (0, _removemarkdown.default)(withoutWikilinks);
     var withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
     return normalizeText(withoutTables);
 }
+function toPosixPath(relPath) {
+    var separator = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : _nodepath.sep;
+    return separator === '\\' ? relPath.split(separator).join('/') : relPath;
+}
+// Every command pays listFiles before it answers (the freshness check stats each file), so
+// per-file work here is the hottest path in the package. Everything derivable from the config
+// alone is computed once, above the loop.
+var NO_THROW = {
+    throwIfNoEntry: false
+};
 function listFiles(cfg, baseDir) {
     var coverage = new Map();
+    var posixNeeded = _nodepath.sep === '\\';
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
     try {
         for(var _iterator = (0, _configts.presetNames)(cfg)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var name = _step.value;
-            var _preset_exclude;
             var preset = cfg.presets[name];
-            var matched = _fastglob.default.sync(preset.include, {
-                cwd: baseDir,
-                ignore: (_preset_exclude = preset.exclude) !== null && _preset_exclude !== void 0 ? _preset_exclude : []
-            });
             var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
             try {
-                for(var _iterator1 = matched[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
-                    var relPath = _step1.value;
+                for(var _iterator1 = (0, _nodefs.globSync)(preset.include, {
+                    cwd: baseDir,
+                    exclude: preset.exclude
+                })[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                    var matched = _step1.value;
                     var _coverage_get;
+                    var relPath = posixNeeded ? toPosixPath(matched) : matched;
                     var set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
                     set.add(name);
                     coverage.set(relPath, set);
@@ -130,29 +152,52 @@ function listFiles(cfg, baseDir) {
             }
         }
     }
-    var relPaths = _to_consumable_array(coverage.keys()).sort();
-    return relPaths.map(function(relPath) {
-        var absPath = (0, _nodepath.join)(baseDir, relPath);
-        var st = (0, _nodefs.statSync)(absPath);
-        var presets = _to_consumable_array(coverage.get(relPath)).sort();
-        var embed = presets.some(function(name) {
-            return (0, _configts.presetSemanticEnabled)(cfg, name);
-        });
-        return {
-            relPath: relPath,
-            absPath: absPath,
-            mtimeMs: st.mtimeMs,
-            size: st.size,
-            presets: presets,
-            embed: embed
-        };
-    });
+    // Which presets want vectors is a property of the config, not of any file.
+    var embedding = (0, _configts.embedEnabled)(cfg);
+    var semanticPresets = embedding ? new Set((0, _configts.presetNames)(cfg).filter(function(name) {
+        return (0, _configts.presetSemanticEnabled)(cfg, name);
+    })) : null;
+    var files = [];
+    var _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+    try {
+        for(var _iterator2 = _to_consumable_array(coverage.keys()).sort()[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true){
+            var relPath1 = _step2.value;
+            var absPath = (0, _nodepath.join)(baseDir, relPath1); // join re-applies the platform separator for fs calls
+            // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so
+            // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).
+            var st = (0, _nodefs.statSync)(absPath, NO_THROW);
+            if (!(st === null || st === void 0 ? void 0 : st.isFile())) continue;
+            var presets = _to_consumable_array(coverage.get(relPath1)).sort();
+            var embed = semanticPresets !== null && presets.some(function(name) {
+                return semanticPresets.has(name);
+            });
+            files.push({
+                relPath: relPath1,
+                absPath: absPath,
+                mtimeMs: st.mtimeMs,
+                size: st.size,
+                presets: presets,
+                embed: embed
+            });
+        }
+    } catch (err) {
+        _didIteratorError2 = true;
+        _iteratorError2 = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
+                _iterator2.return();
+            }
+        } finally{
+            if (_didIteratorError2) {
+                throw _iteratorError2;
+            }
+        }
+    }
+    return files;
 }
-// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole
-// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.
-// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`
-// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so
-// that mismatch is visible rather than a silent zero-row answer.
+// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches
+// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.
 function mapValue(value) {
     if (value === null || value === undefined) return null;
     if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
@@ -178,33 +223,93 @@ function splitFrontmatter(raw) {
         body: rest.slice(close.index + close[0].length)
     };
 }
-// Lenient by design: parseDocument collects syntax errors as data and still yields values,
-// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.
+// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid
+// YAML for a flow map used as a mapping key, so it raises no error and stores
+// {"{ date }": null}. No error code can catch that, but yaml notices the stringified key, so
+// this reports it with the path instead (yaml's own warning has none, fires once per document,
+// and is what trains readers to discard stderr).
+function warnStringifiedKeys(relPath, doc, warnings) {
+    var found = false;
+    // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside
+    // the flow map that `{{...}}` parses as.
+    (0, _yaml.visit)(doc, {
+        Pair: function Pair(_key, pair) {
+            if (!(0, _yaml.isCollection)(pair.key)) return undefined;
+            found = true;
+            return _yaml.visit.BREAK;
+        }
+    });
+    // One per file: a template repeats the same mistake on every field it stamps.
+    if (found) warnings.push("warning: ".concat(relPath, " frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}"));
+}
+// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).
+// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the
+// reason. Recovering it would write values nobody wrote, which is worse than absence because
+// no query can see it. The file is still indexed -- content, links and sections never touch
+// frontmatter -- so a broken note stays searchable while it is being hunted for.
+// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus
+// the colon that introduced the part being dropped.
+function firstLine(message) {
+    return message.split('\n')[0].replace(/:\s*$/, '');
+}
 function parseFrontmatter(relPath, fm, warnings) {
-    var doc = (0, _yaml.parseDocument)(fm);
-    if (doc.errors.length > 0) {
-        warnings.push("warning: ".concat(relPath, " frontmatter has ").concat(doc.errors.length, " syntax error(s) (").concat(doc.errors[0].message.split('\n')[0], "); parsed leniently"));
+    // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one
+    // that carries information, with the file it came from.
+    var doc = (0, _yaml.parseDocument)(fm, {
+        logLevel: 'silent'
+    });
+    var refused = doc.errors.filter(function(err) {
+        return !ACCEPTED_YAML_CODES.has(err.code);
+    });
+    if (refused.length > 0) {
+        var detail = refused.length > 1 ? " (and ".concat(refused.length - 1, " more)") : '';
+        var parseError = "".concat(firstLine(refused[0].message)).concat(detail);
+        warnings.push("warning: ".concat(relPath, " frontmatter did not parse, so none of it is indexed: ").concat(parseError));
+        return {
+            data: {},
+            parseError: parseError
+        };
     }
     var data;
     try {
         data = doc.toJS();
     } catch (err) {
-        warnings.push("warning: ".concat(relPath, " has unparseable frontmatter (").concat(err.message.split('\n')[0], "); indexing without it"));
-        return {};
+        // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on
+        // materialisation. An empty error list is not a successful parse.
+        var parseError1 = firstLine(err.message);
+        warnings.push("warning: ".concat(relPath, " frontmatter did not parse, so none of it is indexed: ").concat(parseError1));
+        return {
+            data: {},
+            parseError: parseError1
+        };
     }
-    if (data === null || data === undefined) return {};
+    if (data === null || data === undefined) return {
+        data: {},
+        parseError: null
+    };
     if ((typeof data === "undefined" ? "undefined" : _type_of(data)) !== 'object' || Array.isArray(data)) {
-        warnings.push("warning: ".concat(relPath, " frontmatter is not a key-value mapping; ignoring it"));
-        return {};
+        var parseError2 = 'frontmatter is not a key-value mapping';
+        warnings.push("warning: ".concat(relPath, " ").concat(parseError2, "; none of it is indexed"));
+        return {
+            data: {},
+            parseError: parseError2
+        };
     }
-    return data;
+    warnStringifiedKeys(relPath, doc, warnings);
+    return {
+        data: data,
+        parseError: null
+    };
 }
 function parseFile(file) {
     var extractors = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : [];
     var raw = (0, _nodefs.readFileSync)(file.absPath, 'utf8');
     var warnings = [];
     var _splitFrontmatter = splitFrontmatter(raw), fm = _splitFrontmatter.fm, content = _splitFrontmatter.body;
-    var data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);
+    var _ref = fm === null ? {
+        data: {},
+        parseError: null
+    } : parseFrontmatter(file.relPath, fm, warnings), data = _ref.data, parseError = _ref.parseError;
     var mapped = {};
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
     try {
@@ -244,6 +349,7 @@ function parseFile(file) {
             size: file.size,
             presets: file.presets,
             data: mapped,
+            parseError: parseError,
             search: search,
             extracted: Object.fromEntries(extractors.filter(function(f) {
                 return f.extract;

package/dist/cjs/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport { presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or\n// the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert\n// can tell a feature-owned column (e.g. rank's `_rank`) apart from a parsed frontmatter\n// column and leave the former alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff any covering preset has semantic !== false\n}\n\n// Presets are the file selection: each preset's include/exclude picks its files (fast-glob,\n// tsconfig-standard semantics, resolved relative to baseDir). Unlike the old layer model,\n// presets are views, not partitions -- they may overlap freely, and a file's set of covering\n// presets (not a single owner) is what indexing and embedding derive from. A preset whose\n// globs match nothing is valid. Files matched by no preset at all are simply not indexed.\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    const matched = fastGlob.sync(preset.include, { cwd: baseDir, ignore: preset.exclude ?? [] });\n    for (const relPath of matched) {\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  const relPaths = [...coverage.keys()].sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = presets.some((name) => presetSemanticEnabled(cfg, name));\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","parseFile","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","cfg","baseDir","coverage","Map","presetNames","name","preset","presets","matched","fastGlob","sync","include","cwd","ignore","exclude","relPath","set","get","add","relPaths","keys","sort","map","absPath","join","st","statSync","embed","some","presetSemanticEnabled","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","parseDocument","errors","push","message","split","data","toJS","err","Array","isArray","file","extractors","readFileSync","content","mapped","Object","key","has","search","title","summary","text","extracted","fromEntries","filter","f","extract"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QA8BGC;eAAAA;;QA+EAC;eAAAA;;;sBA5HuB;wBAClB;+DACA;qEACM;oBACG;wBAEqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAS5C,IAAMF,mBAAmB,IAAIG,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAE5G,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AAgBO,SAASb,UAAUc,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAW,IAAIC;QAChB,kCAAA,2BAAA;;QAAL,QAAK,YAAcC,IAAAA,qBAAW,EAACJ,yBAA1B,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMK,OAAN;gBAEmEC;YADtE,IAAMA,SAASN,IAAIO,OAAO,CAACF,KAAK;YAChC,IAAMG,UAAUC,iBAAQ,CAACC,IAAI,CAACJ,OAAOK,OAAO,EAAE;gBAAEC,KAAKX;gBAASY,MAAM,GAAEP,kBAAAA,OAAOQ,OAAO,cAAdR,6BAAAA,kBAAkB,EAAE;YAAC;gBACtF,mCAAA,4BAAA;;gBAAL,QAAK,aAAiBE,4BAAjB,UAAA,8BAAA,SAAA,0BAAA,kCAA0B;oBAA1B,IAAMO,UAAN;wBACSb;oBAAZ,IAAMc,OAAMd,gBAAAA,SAASe,GAAG,CAACF,sBAAbb,2BAAAA,gBAAyB,IAAId;oBACzC4B,IAAIE,GAAG,CAACb;oBACRH,SAASc,GAAG,CAACD,SAASC;gBACxB;;gBAJK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAKP;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAMG,WAAW,AAAC,qBAAGjB,SAASkB,IAAI,IAAIC,IAAI;IAC1C,OAAOF,SAASG,GAAG,CAAC,SAACP;QACnB,IAAMQ,UAAUC,IAAAA,cAAI,EAACvB,SAASc;QAC9B,IAAMU,KAAKC,IAAAA,gBAAQ,EAACH;QACpB,IAAMhB,UAAU,AAAC,qBAAIL,SAASe,GAAG,CAACF,UAA0BM,IAAI;QAChE,IAAMM,QAAQpB,QAAQqB,IAAI,CAAC,SAACvB;mBAASwB,IAAAA,+BAAqB,EAAC7B,KAAKK;;QAChE,OAAO;YAAEU,SAAAA;YAASQ,SAAAA;YAASO,SAASL,GAAGK,OAAO;YAAEC,MAAMN,GAAGM,IAAI;YAAExB,SAAAA;YAASoB,OAAAA;QAAM;IAChF;AACF;AAcA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASK,SAAS1C,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAO2C,OAAO3C,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO4C,OAAOC,aAAa,CAAC7C,SAAS2C,OAAO3C,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAO8C,KAAKC,SAAS,CAAC/C;AACxB;AAEA,gEAAgE;AAChE,SAASgD,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,IAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKzD,WAAW,OAAO;QAAEmD,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBlC,OAAe,EAAE2B,EAAU,EAAEQ,QAAkB;IACvE,IAAMC,MAAMC,IAAAA,mBAAa,EAACV;IAC1B,IAAIS,IAAIE,MAAM,CAACP,MAAM,GAAG,GAAG;QACzBI,SAASI,IAAI,CAAC,AAAC,YAAsCH,OAA3BpC,SAAQ,qBAAyDoC,OAAtCA,IAAIE,MAAM,CAACP,MAAM,EAAC,sBAAyD,OAArCK,IAAIE,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;IAClI;IACA,IAAIC;IACJ,IAAI;QACFA,OAAON,IAAIO,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZT,SAASI,IAAI,CAAC,AAAC,YAAmD,OAAxCvC,SAAQ,kCAAsE,OAAtC,AAAC4C,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;QACxG,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAASlE,WAAW,OAAO,CAAC;IACjD,IAAI,CAAA,OAAOkE,qCAAP,SAAOA,KAAG,MAAM,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDP,SAASI,IAAI,CAAC,AAAC,YAAmB,OAARvC,SAAQ;QAClC,OAAO,CAAC;IACV;IACA,OAAO0C;AACT;AAEO,SAAStE,UAAU2E,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMxB,MAAMyB,IAAAA,oBAAY,EAACF,KAAKvC,OAAO,EAAE;IACvC,IAAM2B,WAAqB,EAAE;IAE7B,IAA8BZ,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMsB,UAAY3B,kBAAlBK;IACZ,IAAMc,OAAOf,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAK/C,OAAO,EAAE2B,IAAIQ;IACnE,IAAMgB,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAO/C,IAAI,CAACqC,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMW,MAAN;YACH,IAAInF,iBAAiBoF,GAAG,CAACD,MAAM;gBAC7BlB,SAASI,IAAI,CAAC,AAAC,YAAwDc,OAA7CN,KAAK/C,OAAO,EAAC,kCAAoC,OAAJqD,KAAI;gBAC3E;YACF;YACAF,MAAM,CAACE,IAAI,GAAGpC,SAASyB,IAAI,CAACW,IAAI;QAClC;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAME,SAAS;QAAEC,OAAOlF,cAAcoE,KAAKc,KAAK;QAAGC,SAASnF,cAAcoE,KAAKe,OAAO;QAAGC,MAAM9E,UAAUsE;IAAS;IAElH,OAAO;QACLd,KAAK;YACHpC,SAAS+C,KAAK/C,OAAO;YACrBe,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACfxB,SAASuD,KAAKvD,OAAO;YACrBkD,MAAMS;YACNI,QAAAA;YACAI,WAAWP,OAAOQ,WAAW,CAACZ,WAAWa,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAExD,GAAG,CAAC,SAACuD;oBAAeA;uBAAT;oBAACA,EAAExE,IAAI;qBAAEwE,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYtC,KAAK0B,SAASK;iBAAQ;;QAC1H;QACApB,UAAAA;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","parseFile","toPosixPath","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","relPath","separator","sep","split","join","NO_THROW","throwIfNoEntry","cfg","baseDir","coverage","Map","posixNeeded","presetNames","name","preset","presets","globSync","include","cwd","exclude","matched","set","get","add","embedding","embedEnabled","semanticPresets","filter","presetSemanticEnabled","files","keys","sort","absPath","st","statSync","isFile","embed","some","has","push","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","warnStringifiedKeys","doc","warnings","found","visit","Pair","_key","pair","isCollection","key","BREAK","firstLine","message","parseFrontmatter","parseDocument","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","file","extractors","readFileSync","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QA4CGC;eAAAA;;QAkIAC;eAAAA;;QA3IAC;eAAAA;;;sBA/CiC;wBACvB;qEACC;oBACwB;wBAEc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO1D,IAAMH,mBAAmB,IAAII,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW;AAE5H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,mGAAmG;AACnG,IAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AAaO,SAASb,YAAYc,OAAe;QAAEC,YAAAA,iEAAoBC,aAAG;IAClE,OAAOD,cAAc,OAAOD,QAAQG,KAAK,CAACF,WAAWG,IAAI,CAAC,OAAOJ;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,IAAMK,WAAW;IAAEC,gBAAgB;AAAM;AAElC,SAAStB,UAAUuB,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAW,IAAIC;IACrB,IAAMC,cAAcT,aAAG,KAAK;QACvB,kCAAA,2BAAA;;QAAL,QAAK,YAAcU,IAAAA,qBAAW,EAACL,yBAA1B,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMM,OAAN;YACH,IAAMC,SAASP,IAAIQ,OAAO,CAACF,KAAK;gBAC3B,mCAAA,4BAAA;;gBAAL,QAAK,aAAiBG,IAAAA,gBAAQ,EAACF,OAAOG,OAAO,EAAE;oBAAEC,KAAKV;oBAASW,SAASL,OAAOK,OAAO;gBAAC,uBAAlF,UAAA,8BAAA,SAAA,0BAAA,kCAAsF;oBAAtF,IAAMC,UAAN;wBAESX;oBADZ,IAAMT,UAAUW,cAAczB,YAAYkC,WAAWA;oBACrD,IAAMC,OAAMZ,gBAAAA,SAASa,GAAG,CAACtB,sBAAbS,2BAAAA,gBAAyB,IAAItB;oBACzCkC,IAAIE,GAAG,CAACV;oBACRJ,SAASY,GAAG,CAACrB,SAASqB;gBACxB;;gBALK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAMP;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,2EAA2E;IAC3E,IAAMG,YAAYC,IAAAA,sBAAY,EAAClB;IAC/B,IAAMmB,kBAAkBF,YAAY,IAAIrC,IAAIyB,IAAAA,qBAAW,EAACL,KAAKoB,MAAM,CAAC,SAACd;eAASe,IAAAA,+BAAqB,EAACrB,KAAKM;UAAU;IAEnH,IAAMgB,QAAoB,EAAE;QACvB,mCAAA,4BAAA;;QAAL,QAAK,aAAiB,AAAC,qBAAGpB,SAASqB,IAAI,IAAIC,IAAI,uBAA1C,UAAA,8BAAA,SAAA,0BAAA,kCAA8C;YAA9C,IAAM/B,WAAN;YACH,IAAMgC,UAAU5B,IAAAA,cAAI,EAACI,SAASR,WAAU,sDAAsD;YAC9F,yFAAyF;YACzF,uFAAuF;YACvF,IAAMiC,KAAKC,IAAAA,gBAAQ,EAACF,SAAS3B;YAC7B,IAAI,EAAC4B,eAAAA,yBAAAA,GAAIE,MAAM,KAAI;YACnB,IAAMpB,UAAU,AAAC,qBAAIN,SAASa,GAAG,CAACtB,WAA0B+B,IAAI;YAChE,IAAMK,QAAQV,oBAAoB,QAAQX,QAAQsB,IAAI,CAAC,SAACxB;uBAASa,gBAAgBY,GAAG,CAACzB;;YACrFgB,MAAMU,IAAI,CAAC;gBAAEvC,SAAAA;gBAASgC,SAAAA;gBAASQ,SAASP,GAAGO,OAAO;gBAAEC,MAAMR,GAAGQ,IAAI;gBAAE1B,SAAAA;gBAASqB,OAAAA;YAAM;QACpF;;QATK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAOP;AACT;AAiBA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASa,SAASpD,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOqD,OAAOrD,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOsD,OAAOC,aAAa,CAACvD,SAASqD,OAAOrD,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOwD,KAAKC,SAAS,CAACzD;AACxB;AAEA,gEAAgE;AAChE,SAAS0D,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,IAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKnE,WAAW,OAAO;QAAE6D,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASG,oBAAoB3D,OAAe,EAAE4D,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCC,IAAAA,WAAK,EAACH,KAAK;QACTI,MAAAA,SAAAA,KAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACC,IAAAA,kBAAY,EAACD,KAAKE,GAAG,GAAG,OAAO7E;YACpCuE,QAAQ;YACR,OAAOC,WAAK,CAACM,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIP,OAAOD,SAAStB,IAAI,CAAC,AAAC,YAAmB,OAARvC,SAAQ;AAC/C;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASsE,UAAUC,OAAe;IAChC,OAAOA,QAAQpE,KAAK,CAAC,KAAK,CAAC,EAAE,CAACV,OAAO,CAAC,SAAS;AACjD;AAEA,SAAS+E,iBAAiBxE,OAAe,EAAEoD,EAAU,EAAES,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,IAAMD,MAAMa,IAAAA,mBAAa,EAACrB,IAAI;QAAEsB,UAAU;IAAS;IACnD,IAAMC,UAAUf,IAAIgB,MAAM,CAACjD,MAAM,CAAC,SAACkD;eAAQ,CAACzF,oBAAoBkD,GAAG,CAACuC,IAAIC,IAAI;;IAC5E,IAAIH,QAAQnB,MAAM,GAAG,GAAG;QACtB,IAAMuB,SAASJ,QAAQnB,MAAM,GAAG,IAAI,AAAC,SAA2B,OAAnBmB,QAAQnB,MAAM,GAAG,GAAE,YAAU;QAC1E,IAAMwB,aAAa,AAAC,GAAkCD,OAAhCT,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,GAAW,OAAPQ;QACtDlB,SAAStB,IAAI,CAAC,AAAC,YAA2EyC,OAAhEhF,SAAQ,0DAAmE,OAAXgF;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOrB,IAAIsB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,IAAMG,cAAaV,UAAU,AAACO,IAAcN,OAAO;QACnDV,SAAStB,IAAI,CAAC,AAAC,YAA2EyC,OAAhEhF,SAAQ,0DAAmE,OAAXgF;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS1F,WAAW,OAAO;QAAE0F,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,CAAA,OAAOC,qCAAP,SAAOA,KAAG,MAAM,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,IAAMD,cAAa;QACnBnB,SAAStB,IAAI,CAAC,AAAC,YAAsByC,OAAXhF,SAAQ,KAAc,OAAXgF,aAAW;QAChD,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IACArB,oBAAoB3D,SAAS4D,KAAKC;IAClC,OAAO;QAAEoB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEO,SAAS/F,UAAUoG,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMrC,MAAMsC,IAAAA,oBAAY,EAACF,KAAKrD,OAAO,EAAE;IACvC,IAAM6B,WAAqB,EAAE;IAE7B,IAA8Bb,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMmC,UAAYxC,kBAAlBK;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAE6B,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIR,iBAAiBa,KAAKrF,OAAO,EAAEoD,IAAIS,WAAlIoB,OAAqB7B,KAArB6B,MAAMD,aAAe5B,KAAf4B;IACd,IAAMS,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAO5D,IAAI,CAACmD,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMb,MAAN;YACH,IAAIrF,iBAAiBuD,GAAG,CAAC8B,MAAM;gBAC7BP,SAAStB,IAAI,CAAC,AAAC,YAAwD6B,OAA7CiB,KAAKrF,OAAO,EAAC,kCAAoC,OAAJoE,KAAI;gBAC3E;YACF;YACAqB,MAAM,CAACrB,IAAI,GAAG1B,SAASuC,IAAI,CAACb,IAAI;QAClC;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMuB,SAAS;QAAEC,OAAOvG,cAAc4F,KAAKW,KAAK;QAAGC,SAASxG,cAAc4F,KAAKY,OAAO;QAAGC,MAAMnG,UAAU6F;IAAS;IAElH,OAAO;QACL5B,KAAK;YACH5D,SAASqF,KAAKrF,OAAO;YACrBwC,SAAS6C,KAAK7C,OAAO;YACrBC,MAAM4C,KAAK5C,IAAI;YACf1B,SAASsE,KAAKtE,OAAO;YACrBkE,MAAMQ;YACNT,YAAAA;YACAW,QAAAA;YACAI,WAAWL,OAAOM,WAAW,CAACV,WAAW3D,MAAM,CAAC,SAACsE;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEpF,IAAI;qBAAEoF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhD,KAAKuC,SAASG;iBAAQ;;QAC1H;QACA9B,UAAAA;IACF;AACF"}

package/dist/cjs/search-error.js

@@ -9,10 +9,8 @@ Object.defineProperty(exports, "searchError", {
     }
 });
 var _errorsts = require("./errors.js");
-// FTS5 reads these as operators, so a bare term containing one is a syntax error or --
-// worse -- a column filter: `end-to-end` parses as a filter on column `to`, and SQLite
-// reports `no such column: to`, which is a true statement about the parse and a false
-// one about the input. Both field reports on 0.6.0 misdiagnosed that message.
+// FTS5 reads these as operators, so `end-to-end` parses as a filter on column `to` and errors
+// `no such column: to` -- true about the parse, misleading about the input.
 var FTS5_OPERATORS = /[-'"/.:^*()]/;
 function searchError(err, terms, scope) {
     var _terms_match;
@@ -35,8 +33,8 @@ function searchError(err, terms, scope) {
         }).join(', '), " is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"").concat(suspects[0], "\"'. Searchable columns are title, summary, text."));
     }
     if (col !== undefined && scope !== undefined) {
-        return new _errorsts.SenseError('SEARCH_SYNTAX', "".concat(message, " -- the where condition (").concat(scope, ") references it; frontmatter columns are listed by sense query \"SELECT name FROM pragma_table_info('frontmatter')\"."));
+        return new _errorsts.SenseError('SEARCH_SYNTAX', "".concat(message, " -- the where condition (").concat(scope, ") references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\"."));
     }
-    return new _errorsts.SenseError('SEARCH_SYNTAX', "".concat(message, " -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense query (list them with pragma_table_info('frontmatter'))."));
+    return new _errorsts.SenseError('SEARCH_SYNTAX', "".concat(message, " -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter'))."));
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/search-error.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/search-error.ts"],"sourcesContent":["import { SenseError } from './errors.ts';\n\n// FTS5 reads these as operators, so a bare term containing one is a syntax error or --\n// worse -- a column filter: `end-to-end` parses as a filter on column `to`, and SQLite\n// reports `no such column: to`, which is a true statement about the parse and a false\n// one about the input. Both field reports on 0.6.0 misdiagnosed that message.\nconst FTS5_OPERATORS = /[-'\"/.:^*()]/;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_OPERATORS.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where (or the tree's default scope) raises \"no such column\" through this same\n  // statement, and naming a term for it would state a false fact about the input.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense query \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense query (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["searchError","FTS5_OPERATORS","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","SenseError","map","join"],"mappings":";;;;+BAQgBA;;;eAAAA;;;wBARW;AAE3B,uFAAuF;AACvF,uFAAuF;AACvF,sFAAsF;AACtF,8EAA8E;AAC9E,IAAMC,iBAAiB;AAEhB,SAASD,YAAYE,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAIN;IANZ,IAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,IAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,SAACC;eAAM,CAACA,EAAEC,UAAU,CAAC,QAAQV,eAAeK,IAAI,CAACI;;IACrG,wFAAwF;IACxF,qFAAqF;IACrF,gFAAgF;IAChF,IAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,IAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,SAACP;eAAMA,EAAEQ,KAAK,CAAC,8maAAmBC,QAAQ,CAACP;;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAII,oBAAU,CAAC,iBAAiB,AAAC,GAAmCb,OAAjCF,SAAQ,2BAAsJE,OAA7HA,SAASc,GAAG,CAAC,SAACX;mBAAM,AAAC,IAAM,OAAFA,GAAE;WAAKY,IAAI,CAAC,OAAM,qFAA8F,OAAZf,QAAQ,CAAC,EAAE,EAAC;IACtN;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIK,oBAAU,CAAC,iBAAiB,AAAC,GAAqChB,OAAnCC,SAAQ,6BAAiC,OAAND,OAAM;IACrF;IACA,OAAO,IAAIgB,oBAAU,CAAC,iBAAiB,AAAC,GAAU,OAARf,SAAQ;AACpD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/search-error.ts"],"sourcesContent":["import { SenseError } from './errors.ts';\n\n// FTS5 reads these as operators, so `end-to-end` parses as a filter on column `to` and errors\n// `no such column: to` -- true about the parse, misleading about the input.\nconst FTS5_OPERATORS = /[-'\"/.:^*()]/;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_OPERATORS.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where (or the tree's default scope) raises \"no such column\" through this same\n  // statement, and naming a term for it would state a false fact about the input.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["searchError","FTS5_OPERATORS","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","SenseError","map","join"],"mappings":";;;;+BAMgBA;;;eAAAA;;;wBANW;AAE3B,8FAA8F;AAC9F,4EAA4E;AAC5E,IAAMC,iBAAiB;AAEhB,SAASD,YAAYE,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAIN;IANZ,IAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,IAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,SAACC;eAAM,CAACA,EAAEC,UAAU,CAAC,QAAQV,eAAeK,IAAI,CAACI;;IACrG,wFAAwF;IACxF,qFAAqF;IACrF,gFAAgF;IAChF,IAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,IAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,SAACP;eAAMA,EAAEQ,KAAK,CAAC,8maAAmBC,QAAQ,CAACP;;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAII,oBAAU,CAAC,iBAAiB,AAAC,GAAmCb,OAAjCF,SAAQ,2BAAsJE,OAA7HA,SAASc,GAAG,CAAC,SAACX;mBAAM,AAAC,IAAM,OAAFA,GAAE;WAAKY,IAAI,CAAC,OAAM,qFAA8F,OAAZf,QAAQ,CAAC,EAAE,EAAC;IACtN;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIK,oBAAU,CAAC,iBAAiB,AAAC,GAAqChB,OAAnCC,SAAQ,6BAAiC,OAAND,OAAM;IACrF;IACA,OAAO,IAAIgB,oBAAU,CAAC,iBAAiB,AAAC,GAAU,OAARf,SAAQ;AACpD"}

package/dist/cjs/traverse.d.cts

@@ -0,0 +1,8 @@
+import type { DatabaseSync } from 'node:sqlite';
+interface FindPathOptions {
+    directed?: boolean;
+    maxDepth?: number;
+    allowed?: Set<string>;
+}
+export declare function findPath(db: DatabaseSync, from: string, to: string, opts?: FindPathOptions): string[] | null;
+export {};

package/dist/cjs/traverse.d.ts

@@ -0,0 +1,8 @@
+import type { DatabaseSync } from 'node:sqlite';
+interface FindPathOptions {
+    directed?: boolean;
+    maxDepth?: number;
+    allowed?: Set<string>;
+}
+export declare function findPath(db: DatabaseSync, from: string, to: string, opts?: FindPathOptions): string[] | null;
+export {};

package/dist/cjs/traverse.js

@@ -0,0 +1,97 @@
+// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a
+// local question touches a neighborhood, never the whole edge list.
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "findPath", {
+    enumerable: true,
+    get: function() {
+        return findPath;
+    }
+});
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter);
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+function resetVisited(db) {
+    db.exec('DROP TABLE IF EXISTS temp.visited');
+    db.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');
+}
+// Materializes the caller-resolved scope Set into a temp table so it joins like any other
+// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.
+function setupAllowed(db, allowed, extra) {
+    db.exec('DROP TABLE IF EXISTS temp.allowed_nodes');
+    db.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');
+    db.prepare('INSERT INTO allowed_nodes SELECT DISTINCT value FROM json_each(?1)').run(JSON.stringify(_to_consumable_array(allowed)));
+    if (extra !== undefined) db.prepare('INSERT OR IGNORE INTO allowed_nodes VALUES (?)').run(extra);
+}
+// Only resolved links (dst IS NOT NULL) are followed, in either direction.
+function candidateSelect(direction) {
+    var forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';
+    var reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';
+    if (direction === 'forward') return forward;
+    if (direction === 'reverse') return reverse;
+    return "".concat(forward, " UNION ALL ").concat(reverse);
+}
+// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against
+// everything already visited, optionally filtered to the allowed set, capped (?3) and
+// inserted at the new depth (?2). RETURNING hands back exactly what this ring newly reached.
+function ringSql(direction, hasAllowed) {
+    return "\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (".concat(candidateSelect(direction), ") cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n").concat(hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : '', "\nGROUP BY cand.path\nORDER BY cand.path\nLIMIT ?3\nRETURNING path");
+}
+function reconstructPath(db, to) {
+    var lookup = db.prepare('SELECT pred FROM visited WHERE path = ?');
+    var path = [];
+    var cur = to;
+    while(cur !== null){
+        path.push(cur);
+        var row = lookup.get(cur);
+        cur = row ? row.pred : null;
+    }
+    return path.reverse();
+}
+function findPath(db, from, to) {
+    var opts = arguments.length > 3 && arguments[3] !== void 0 ? arguments[3] : {};
+    var _opts_maxDepth;
+    if (from === to) return [
+        from
+    ];
+    var direction = opts.directed ? 'forward' : 'both';
+    var maxDepth = (_opts_maxDepth = opts.maxDepth) !== null && _opts_maxDepth !== void 0 ? _opts_maxDepth : Infinity;
+    if (maxDepth <= 0) return null;
+    resetVisited(db);
+    db.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)').run(from);
+    if (opts.allowed) setupAllowed(db, opts.allowed, to);
+    var stmt = db.prepare(ringSql(direction, !!opts.allowed));
+    for(var ring = 1; ring <= maxDepth; ring++){
+        var rows = stmt.all(ring - 1, ring, -1);
+        if (rows.length === 0) break;
+        if (rows.some(function(row) {
+            return row.path === to;
+        })) return reconstructPath(db, to);
+    }
+    return null;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/traverse.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\nimport type { DatabaseSync } from 'node:sqlite';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\ninterface FindPathOptions {\n  directed?: boolean;\n  maxDepth?: number;\n  allowed?: Set<string>;\n}\n\nfunction resetVisited(db: DatabaseSync): void {\n  db.exec('DROP TABLE IF EXISTS temp.visited');\n  db.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');\n}\n\n// Materializes the caller-resolved scope Set into a temp table so it joins like any other\n// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.\nfunction setupAllowed(db: DatabaseSync, allowed: Set<string>, extra?: string): void {\n  db.exec('DROP TABLE IF EXISTS temp.allowed_nodes');\n  db.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');\n  db.prepare('INSERT INTO allowed_nodes SELECT DISTINCT value FROM json_each(?1)').run(JSON.stringify([...allowed]));\n  if (extra !== undefined) db.prepare('INSERT OR IGNORE INTO allowed_nodes VALUES (?)').run(extra);\n}\n\n// Only resolved links (dst IS NOT NULL) are followed, in either direction.\nfunction candidateSelect(direction: Direction): string {\n  const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  if (direction === 'forward') return forward;\n  if (direction === 'reverse') return reverse;\n  return `${forward} UNION ALL ${reverse}`;\n}\n\n// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against\n// everything already visited, optionally filtered to the allowed set, capped (?3) and\n// inserted at the new depth (?2). RETURNING hands back exactly what this ring newly reached.\nfunction ringSql(direction: Direction, hasAllowed: boolean): string {\n  return `\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (${candidateSelect(direction)}) cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}\nGROUP BY cand.path\nORDER BY cand.path\nLIMIT ?3\nRETURNING path`;\n}\n\nfunction reconstructPath(db: DatabaseSync, to: string): string[] {\n  const lookup = db.prepare('SELECT pred FROM visited WHERE path = ?');\n  const path: string[] = [];\n  let cur: string | null = to;\n  while (cur !== null) {\n    path.push(cur);\n    const row = lookup.get(cur) as { pred: string | null } | undefined;\n    cur = row ? row.pred : null;\n  }\n  return path.reverse();\n}\n\n// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching\n// personalizedRank; null when `to` is unreached within `maxDepth`.\nexport function findPath(db: DatabaseSync, from: string, to: string, opts: FindPathOptions = {}): string[] | null {\n  if (from === to) return [from];\n\n  const direction: Direction = opts.directed ? 'forward' : 'both';\n  const maxDepth = opts.maxDepth ?? Infinity;\n  if (maxDepth <= 0) return null;\n\n  resetVisited(db);\n  db.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)').run(from);\n  if (opts.allowed) setupAllowed(db, opts.allowed, to);\n\n  const stmt = db.prepare(ringSql(direction, !!opts.allowed));\n  for (let ring = 1; ring <= maxDepth; ring++) {\n    const rows = stmt.all(ring - 1, ring, -1) as Array<{ path: string }>;\n    if (rows.length === 0) break;\n    if (rows.some((row) => row.path === to)) return reconstructPath(db, to);\n  }\n  return null;\n}\n"],"names":["findPath","resetVisited","db","exec","setupAllowed","allowed","extra","prepare","run","JSON","stringify","undefined","candidateSelect","direction","forward","reverse","ringSql","hasAllowed","reconstructPath","to","lookup","path","cur","push","row","get","pred","from","opts","directed","maxDepth","Infinity","stmt","ring","rows","all","length","some"],"mappings":"AAAA,2FAA2F;AAC3F,oEAAoE;;;;;+BAgEpDA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;AArDhB,SAASC,aAAaC,EAAgB;IACpCA,GAAGC,IAAI,CAAC;IACRD,GAAGC,IAAI,CAAC;AACV;AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,SAASC,aAAaF,EAAgB,EAAEG,OAAoB,EAAEC,KAAc;IAC1EJ,GAAGC,IAAI,CAAC;IACRD,GAAGC,IAAI,CAAC;IACRD,GAAGK,OAAO,CAAC,sEAAsEC,GAAG,CAACC,KAAKC,SAAS,CAAE,qBAAGL;IACxG,IAAIC,UAAUK,WAAWT,GAAGK,OAAO,CAAC,kDAAkDC,GAAG,CAACF;AAC5F;AAEA,2EAA2E;AAC3E,SAASM,gBAAgBC,SAAoB;IAC3C,IAAMC,UAAU;IAChB,IAAMC,UAAU;IAChB,IAAIF,cAAc,WAAW,OAAOC;IACpC,IAAID,cAAc,WAAW,OAAOE;IACpC,OAAO,AAAC,GAAuBA,OAArBD,SAAQ,eAAqB,OAARC;AACjC;AAEA,0FAA0F;AAC1F,sFAAsF;AACtF,6FAA6F;AAC7F,SAASC,QAAQH,SAAoB,EAAEI,UAAmB;IACxD,OAAO,AAAC,0FAKRA,OAFML,gBAAgBC,YAAW,mFAEuD,OAAxFI,aAAa,wEAAwE,IAAG;AAK1F;AAEA,SAASC,gBAAgBhB,EAAgB,EAAEiB,EAAU;IACnD,IAAMC,SAASlB,GAAGK,OAAO,CAAC;IAC1B,IAAMc,OAAiB,EAAE;IACzB,IAAIC,MAAqBH;IACzB,MAAOG,QAAQ,KAAM;QACnBD,KAAKE,IAAI,CAACD;QACV,IAAME,MAAMJ,OAAOK,GAAG,CAACH;QACvBA,MAAME,MAAMA,IAAIE,IAAI,GAAG;IACzB;IACA,OAAOL,KAAKN,OAAO;AACrB;AAIO,SAASf,SAASE,EAAgB,EAAEyB,IAAY,EAAER,EAAU;QAAES,OAAAA,iEAAwB,CAAC;QAI3EA;IAHjB,IAAID,SAASR,IAAI,OAAO;QAACQ;KAAK;IAE9B,IAAMd,YAAuBe,KAAKC,QAAQ,GAAG,YAAY;IACzD,IAAMC,YAAWF,iBAAAA,KAAKE,QAAQ,cAAbF,4BAAAA,iBAAiBG;IAClC,IAAID,YAAY,GAAG,OAAO;IAE1B7B,aAAaC;IACbA,GAAGK,OAAO,CAAC,+DAA+DC,GAAG,CAACmB;IAC9E,IAAIC,KAAKvB,OAAO,EAAED,aAAaF,IAAI0B,KAAKvB,OAAO,EAAEc;IAEjD,IAAMa,OAAO9B,GAAGK,OAAO,CAACS,QAAQH,WAAW,CAAC,CAACe,KAAKvB,OAAO;IACzD,IAAK,IAAI4B,OAAO,GAAGA,QAAQH,UAAUG,OAAQ;QAC3C,IAAMC,OAAOF,KAAKG,GAAG,CAACF,OAAO,GAAGA,MAAM,CAAC;QACvC,IAAIC,KAAKE,MAAM,KAAK,GAAG;QACvB,IAAIF,KAAKG,IAAI,CAAC,SAACb;mBAAQA,IAAIH,IAAI,KAAKF;YAAK,OAAOD,gBAAgBhB,IAAIiB;IACtE;IACA,OAAO;AACT"}

package/dist/esm/cli/download.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from './types.js';
+declare const download: Command;
+export default download;

package/dist/esm/cli/download.js

@@ -0,0 +1,27 @@
+import { embedConfig } from '../config.js';
+import { SenseError } from '../errors.js';
+import { downloadModel, modelDir, modelPresent } from '../features/embed.js';
+import { USAGE } from './index.js';
+import { CONFIG, parse } from './shared.js';
+// The one command that fetches weights. Nothing else touches the network for them, so a
+// query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there
+// is no prompt for CI to bypass; running it twice is a no-op.
+const download = async (ctx)=>{
+    const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.download}`, {
+        ...CONFIG
+    });
+    const cfg = ctx.resolveConfig(values.config);
+    const e = embedConfig(cfg);
+    if (!e) throw new SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');
+    if (e.type === 'api') {
+        console.log(`embed type is "api" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);
+        return;
+    }
+    if (modelPresent(cfg)) {
+        console.log(`${e.model} is already available in ${modelDir(e.model)}`);
+        return;
+    }
+    const dir = await downloadModel(e.model, (file, into)=>console.error(`fetching ${e.model}/${file} into ${into}`));
+    console.log(`downloaded ${e.model} into ${dir}`);
+};
+export default download;

package/dist/esm/cli/download.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/download.ts"],"sourcesContent":["import { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, modelDir, modelPresent } from '../features/embed.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, parse } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// The one command that fetches weights. Nothing else touches the network for them, so a\n// query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there\n// is no prompt for CI to bypass; running it twice is a no-op.\nconst download: Command = async (ctx) => {\n  const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.download}`, { ...CONFIG });\n  const cfg = ctx.resolveConfig(values.config as string | undefined);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');\n\n  if (e.type === 'api') {\n    console.log(`embed type is \"api\" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);\n    return;\n  }\n  if (modelPresent(cfg)) {\n    console.log(`${e.model} is already available in ${modelDir(e.model)}`);\n    return;\n  }\n  const dir = await downloadModel(e.model, (file, into) => console.error(`fetching ${e.model}/${file} into ${into}`));\n  console.log(`downloaded ${e.model} into ${dir}`);\n};\nexport default download;\n"],"names":["embedConfig","SenseError","downloadModel","modelDir","modelPresent","USAGE","CONFIG","parse","download","ctx","values","argv","name","cfg","resolveConfig","config","e","type","console","log","url","model","dir","file","into","error"],"mappings":"AAAA,SAASA,WAAW,QAAQ,eAAe;AAC3C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,QAAQ,EAAEC,YAAY,QAAQ,uBAAuB;AAC7E,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,MAAM,EAAEC,KAAK,QAAQ,cAAc;AAG5C,wFAAwF;AACxF,4FAA4F;AAC5F,8DAA8D;AAC9D,MAAMC,WAAoB,OAAOC;IAC/B,MAAM,EAAEC,MAAM,EAAE,GAAGH,MAAME,IAAIE,IAAI,EAAE,CAAC,OAAO,EAAEF,IAAIG,IAAI,CAAC,CAAC,EAAEP,MAAMG,QAAQ,EAAE,EAAE;QAAE,GAAGF,MAAM;IAAC;IACvF,MAAMO,MAAMJ,IAAIK,aAAa,CAACJ,OAAOK,MAAM;IAC3C,MAAMC,IAAIhB,YAAYa;IACtB,IAAI,CAACG,GAAG,MAAM,IAAIf,WAAW,kBAAkB;IAE/C,IAAIe,EAAEC,IAAI,KAAK,OAAO;QACpBC,QAAQC,GAAG,CAAC,CAAC,qBAAqB,EAAEH,EAAEI,GAAG,CAAC,oCAAoC,EAAEX,IAAIG,IAAI,CAAC,mCAAmC,CAAC;QAC7H;IACF;IACA,IAAIR,aAAaS,MAAM;QACrBK,QAAQC,GAAG,CAAC,GAAGH,EAAEK,KAAK,CAAC,yBAAyB,EAAElB,SAASa,EAAEK,KAAK,GAAG;QACrE;IACF;IACA,MAAMC,MAAM,MAAMpB,cAAcc,EAAEK,KAAK,EAAE,CAACE,MAAMC,OAASN,QAAQO,KAAK,CAAC,CAAC,SAAS,EAAET,EAAEK,KAAK,CAAC,CAAC,EAAEE,KAAK,MAAM,EAAEC,MAAM;IACjHN,QAAQC,GAAG,CAAC,CAAC,WAAW,EAAEH,EAAEK,KAAK,CAAC,MAAM,EAAEC,KAAK;AACjD;AACA,eAAed,SAAS"}

package/dist/esm/cli/index.d.ts

@@ -3,12 +3,13 @@ export declare const USAGE: {
     readonly init: "init";
     readonly watch: "watch [--force] [--config path]";
     readonly status: "status [--format table|json] [--config path]";
-    readonly check: "check [--format table|json] [--config path]";
-    readonly rebuild: "rebuild [--config path]";
-    readonly query: "query \"<sql>\" [params...] [--format table|json] [--config path]";
-    readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--where \"<sql>\"] [--k n] [--lexical] [--format table|json] [--config path]";
+    readonly download: "download [--config path]";
+    readonly sql: "sql \"<statement>\" [params...] [--preset name] [--format table|json] [--config path]";
+    readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
     readonly map: "map [--format table|json] [--config path]";
-    readonly peek: "peek <path> [--format table|json] [--config path]";
+    readonly peek: "peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]";
+    readonly path: "path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]";
+    readonly related: "related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
 };
 export declare const COMMANDS: Record<string, () => Promise<{
     default: Command;