sensemaking 0.10.0 → 0.11.1

package/dist/cjs/scan.js

@@ -66,10 +66,20 @@ var RESERVED_COLUMNS = new Set([
     '_mtime',
     '_size',
     '_rank',
+    '_parse_error',
     'content',
     'links',
     'sections'
 ]);
+// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than
+// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain
+// scalar for future use, so they can never be valid and the text can only be what was typed
+// (`aliases: [@handle]` -> ["@handle"]). Every other code has a second reading -- an unquoted
+// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks
+// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.
+var ACCEPTED_YAML_CODES = new Set([
+    'BAD_SCALAR_START'
+]);
 function normalizeText(value) {
     if (value === null || value === undefined) return '';
     return String(value).replace(/\s+/g, ' ').trim();
@@ -213,33 +223,93 @@ function splitFrontmatter(raw) {
         body: rest.slice(close.index + close[0].length)
     };
 }
-// Lenient by design: parseDocument collects syntax errors as data and still yields values,
-// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.
+// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid
+// YAML for a flow map used as a mapping key, so it raises no error and stores
+// {"{ date }": null}. No error code can catch that, but yaml notices the stringified key, so
+// this reports it with the path instead (yaml's own warning has none, fires once per document,
+// and is what trains readers to discard stderr).
+function warnStringifiedKeys(relPath, doc, warnings) {
+    var found = false;
+    // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside
+    // the flow map that `{{...}}` parses as.
+    (0, _yaml.visit)(doc, {
+        Pair: function Pair(_key, pair) {
+            if (!(0, _yaml.isCollection)(pair.key)) return undefined;
+            found = true;
+            return _yaml.visit.BREAK;
+        }
+    });
+    // One per file: a template repeats the same mistake on every field it stamps.
+    if (found) warnings.push("warning: ".concat(relPath, " frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}"));
+}
+// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).
+// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the
+// reason. Recovering it would write values nobody wrote, which is worse than absence because
+// no query can see it. The file is still indexed -- content, links and sections never touch
+// frontmatter -- so a broken note stays searchable while it is being hunted for.
+// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus
+// the colon that introduced the part being dropped.
+function firstLine(message) {
+    return message.split('\n')[0].replace(/:\s*$/, '');
+}
 function parseFrontmatter(relPath, fm, warnings) {
-    var doc = (0, _yaml.parseDocument)(fm);
-    if (doc.errors.length > 0) {
-        warnings.push("warning: ".concat(relPath, " frontmatter has ").concat(doc.errors.length, " syntax error(s) (").concat(doc.errors[0].message.split('\n')[0], "); parsed leniently"));
+    // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one
+    // that carries information, with the file it came from.
+    var doc = (0, _yaml.parseDocument)(fm, {
+        logLevel: 'silent'
+    });
+    var refused = doc.errors.filter(function(err) {
+        return !ACCEPTED_YAML_CODES.has(err.code);
+    });
+    if (refused.length > 0) {
+        var detail = refused.length > 1 ? " (and ".concat(refused.length - 1, " more)") : '';
+        var parseError = "".concat(firstLine(refused[0].message)).concat(detail);
+        warnings.push("warning: ".concat(relPath, " frontmatter did not parse, so none of it is indexed: ").concat(parseError));
+        return {
+            data: {},
+            parseError: parseError
+        };
     }
     var data;
     try {
         data = doc.toJS();
     } catch (err) {
-        warnings.push("warning: ".concat(relPath, " has unparseable frontmatter (").concat(err.message.split('\n')[0], "); indexing without it"));
-        return {};
+        // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on
+        // materialisation. An empty error list is not a successful parse.
+        var parseError1 = firstLine(err.message);
+        warnings.push("warning: ".concat(relPath, " frontmatter did not parse, so none of it is indexed: ").concat(parseError1));
+        return {
+            data: {},
+            parseError: parseError1
+        };
     }
-    if (data === null || data === undefined) return {};
+    if (data === null || data === undefined) return {
+        data: {},
+        parseError: null
+    };
     if ((typeof data === "undefined" ? "undefined" : _type_of(data)) !== 'object' || Array.isArray(data)) {
-        warnings.push("warning: ".concat(relPath, " frontmatter is not a key-value mapping; ignoring it"));
-        return {};
+        var parseError2 = 'frontmatter is not a key-value mapping';
+        warnings.push("warning: ".concat(relPath, " ").concat(parseError2, "; none of it is indexed"));
+        return {
+            data: {},
+            parseError: parseError2
+        };
     }
-    return data;
+    warnStringifiedKeys(relPath, doc, warnings);
+    return {
+        data: data,
+        parseError: null
+    };
 }
 function parseFile(file) {
     var extractors = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : [];
     var raw = (0, _nodefs.readFileSync)(file.absPath, 'utf8');
     var warnings = [];
     var _splitFrontmatter = splitFrontmatter(raw), fm = _splitFrontmatter.fm, content = _splitFrontmatter.body;
-    var data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);
+    var _ref = fm === null ? {
+        data: {},
+        parseError: null
+    } : parseFrontmatter(file.relPath, fm, warnings), data = _ref.data, parseError = _ref.parseError;
     var mapped = {};
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
     try {
@@ -279,6 +349,7 @@ function parseFile(file) {
             size: file.size,
             presets: file.presets,
             data: mapped,
+            parseError: parseError,
             search: search,
             extracted: Object.fromEntries(extractors.filter(function(f) {
                 return f.extract;

package/dist/cjs/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","parseFile","toPosixPath","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","relPath","separator","sep","split","join","NO_THROW","throwIfNoEntry","cfg","baseDir","coverage","Map","posixNeeded","presetNames","name","preset","presets","globSync","include","cwd","exclude","matched","set","get","add","embedding","embedEnabled","semanticPresets","filter","presetSemanticEnabled","files","keys","sort","absPath","st","statSync","isFile","embed","some","has","push","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","parseDocument","errors","message","data","toJS","err","Array","isArray","file","extractors","readFileSync","content","mapped","Object","key","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAoCGC;eAAAA;;QAqFAC;eAAAA;;QA9FAC;eAAAA;;;sBAvCiC;wBACvB;qEACC;oBACG;wBAEmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO1D,IAAMH,mBAAmB,IAAII,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAE5G,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AAaO,SAASZ,YAAYa,OAAe;QAAEC,YAAAA,iEAAoBC,aAAG;IAClE,OAAOD,cAAc,OAAOD,QAAQG,KAAK,CAACF,WAAWG,IAAI,CAAC,OAAOJ;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,IAAMK,WAAW;IAAEC,gBAAgB;AAAM;AAElC,SAASrB,UAAUsB,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAW,IAAIC;IACrB,IAAMC,cAAcT,aAAG,KAAK;QACvB,kCAAA,2BAAA;;QAAL,QAAK,YAAcU,IAAAA,qBAAW,EAACL,yBAA1B,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMM,OAAN;YACH,IAAMC,SAASP,IAAIQ,OAAO,CAACF,KAAK;gBAC3B,mCAAA,4BAAA;;gBAAL,QAAK,aAAiBG,IAAAA,gBAAQ,EAACF,OAAOG,OAAO,EAAE;oBAAEC,KAAKV;oBAASW,SAASL,OAAOK,OAAO;gBAAC,uBAAlF,UAAA,8BAAA,SAAA,0BAAA,kCAAsF;oBAAtF,IAAMC,UAAN;wBAESX;oBADZ,IAAMT,UAAUW,cAAcxB,YAAYiC,WAAWA;oBACrD,IAAMC,OAAMZ,gBAAAA,SAASa,GAAG,CAACtB,sBAAbS,2BAAAA,gBAAyB,IAAIrB;oBACzCiC,IAAIE,GAAG,CAACV;oBACRJ,SAASY,GAAG,CAACrB,SAASqB;gBACxB;;gBALK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAMP;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,2EAA2E;IAC3E,IAAMG,YAAYC,IAAAA,sBAAY,EAAClB;IAC/B,IAAMmB,kBAAkBF,YAAY,IAAIpC,IAAIwB,IAAAA,qBAAW,EAACL,KAAKoB,MAAM,CAAC,SAACd;eAASe,IAAAA,+BAAqB,EAACrB,KAAKM;UAAU;IAEnH,IAAMgB,QAAoB,EAAE;QACvB,mCAAA,4BAAA;;QAAL,QAAK,aAAiB,AAAC,qBAAGpB,SAASqB,IAAI,IAAIC,IAAI,uBAA1C,UAAA,8BAAA,SAAA,0BAAA,kCAA8C;YAA9C,IAAM/B,WAAN;YACH,IAAMgC,UAAU5B,IAAAA,cAAI,EAACI,SAASR,WAAU,sDAAsD;YAC9F,yFAAyF;YACzF,uFAAuF;YACvF,IAAMiC,KAAKC,IAAAA,gBAAQ,EAACF,SAAS3B;YAC7B,IAAI,EAAC4B,eAAAA,yBAAAA,GAAIE,MAAM,KAAI;YACnB,IAAMpB,UAAU,AAAC,qBAAIN,SAASa,GAAG,CAACtB,WAA0B+B,IAAI;YAChE,IAAMK,QAAQV,oBAAoB,QAAQX,QAAQsB,IAAI,CAAC,SAACxB;uBAASa,gBAAgBY,GAAG,CAACzB;;YACrFgB,MAAMU,IAAI,CAAC;gBAAEvC,SAAAA;gBAASgC,SAAAA;gBAASQ,SAASP,GAAGO,OAAO;gBAAEC,MAAMR,GAAGQ,IAAI;gBAAE1B,SAAAA;gBAASqB,OAAAA;YAAM;QACpF;;QATK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAOP;AACT;AAcA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASa,SAASpD,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOqD,OAAOrD,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOsD,OAAOC,aAAa,CAACvD,SAASqD,OAAOrD,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOwD,KAAKC,SAAS,CAACzD;AACxB;AAEA,gEAAgE;AAChE,SAAS0D,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,IAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKnE,WAAW,OAAO;QAAE6D,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiB3D,OAAe,EAAEoD,EAAU,EAAEQ,QAAkB;IACvE,IAAMC,MAAMC,IAAAA,mBAAa,EAACV;IAC1B,IAAIS,IAAIE,MAAM,CAACP,MAAM,GAAG,GAAG;QACzBI,SAASrB,IAAI,CAAC,AAAC,YAAsCsB,OAA3B7D,SAAQ,qBAAyD6D,OAAtCA,IAAIE,MAAM,CAACP,MAAM,EAAC,sBAAyD,OAArCK,IAAIE,MAAM,CAAC,EAAE,CAACC,OAAO,CAAC7D,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;IAClI;IACA,IAAI8D;IACJ,IAAI;QACFA,OAAOJ,IAAIK,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZP,SAASrB,IAAI,CAAC,AAAC,YAAmD,OAAxCvC,SAAQ,kCAAsE,OAAtC,AAACmE,IAAcH,OAAO,CAAC7D,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;QACxG,OAAO,CAAC;IACV;IACA,IAAI8D,SAAS,QAAQA,SAAS1E,WAAW,OAAO,CAAC;IACjD,IAAI,CAAA,OAAO0E,qCAAP,SAAOA,KAAG,MAAM,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDL,SAASrB,IAAI,CAAC,AAAC,YAAmB,OAARvC,SAAQ;QAClC,OAAO,CAAC;IACV;IACA,OAAOiE;AACT;AAEO,SAAS/E,UAAUoF,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMtB,MAAMuB,IAAAA,oBAAY,EAACF,KAAKtC,OAAO,EAAE;IACvC,IAAM4B,WAAqB,EAAE;IAE7B,IAA8BZ,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMoB,UAAYzB,kBAAlBK;IACZ,IAAMY,OAAOb,OAAO,OAAO,CAAC,IAAIO,iBAAiBW,KAAKtE,OAAO,EAAEoD,IAAIQ;IACnE,IAAMc,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAO7C,IAAI,CAACmC,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMW,MAAN;YACH,IAAI5F,iBAAiBsD,GAAG,CAACsC,MAAM;gBAC7BhB,SAASrB,IAAI,CAAC,AAAC,YAAwDqC,OAA7CN,KAAKtE,OAAO,EAAC,kCAAoC,OAAJ4E,KAAI;gBAC3E;YACF;YACAF,MAAM,CAACE,IAAI,GAAGlC,SAASuB,IAAI,CAACW,IAAI;QAClC;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMC,SAAS;QAAEC,OAAOzF,cAAc4E,KAAKa,KAAK;QAAGC,SAAS1F,cAAc4E,KAAKc,OAAO;QAAGC,MAAMrF,UAAU8E;IAAS;IAElH,OAAO;QACLZ,KAAK;YACH7D,SAASsE,KAAKtE,OAAO;YACrBwC,SAAS8B,KAAK9B,OAAO;YACrBC,MAAM6B,KAAK7B,IAAI;YACf1B,SAASuD,KAAKvD,OAAO;YACrBkD,MAAMS;YACNG,QAAAA;YACAI,WAAWN,OAAOO,WAAW,CAACX,WAAW5C,MAAM,CAAC,SAACwD;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEtE,IAAI;qBAAEsE,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYlC,KAAKwB,SAASI;iBAAQ;;QAC1H;QACAjB,UAAAA;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","parseFile","toPosixPath","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","relPath","separator","sep","split","join","NO_THROW","throwIfNoEntry","cfg","baseDir","coverage","Map","posixNeeded","presetNames","name","preset","presets","globSync","include","cwd","exclude","matched","set","get","add","embedding","embedEnabled","semanticPresets","filter","presetSemanticEnabled","files","keys","sort","absPath","st","statSync","isFile","embed","some","has","push","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","warnStringifiedKeys","doc","warnings","found","visit","Pair","_key","pair","isCollection","key","BREAK","firstLine","message","parseFrontmatter","parseDocument","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","file","extractors","readFileSync","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QA4CGC;eAAAA;;QAkIAC;eAAAA;;QA3IAC;eAAAA;;;sBA/CiC;wBACvB;qEACC;oBACwB;wBAEc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO1D,IAAMH,mBAAmB,IAAII,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW;AAE5H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,mGAAmG;AACnG,IAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AAaO,SAASb,YAAYc,OAAe;QAAEC,YAAAA,iEAAoBC,aAAG;IAClE,OAAOD,cAAc,OAAOD,QAAQG,KAAK,CAACF,WAAWG,IAAI,CAAC,OAAOJ;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,IAAMK,WAAW;IAAEC,gBAAgB;AAAM;AAElC,SAAStB,UAAUuB,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAW,IAAIC;IACrB,IAAMC,cAAcT,aAAG,KAAK;QACvB,kCAAA,2BAAA;;QAAL,QAAK,YAAcU,IAAAA,qBAAW,EAACL,yBAA1B,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMM,OAAN;YACH,IAAMC,SAASP,IAAIQ,OAAO,CAACF,KAAK;gBAC3B,mCAAA,4BAAA;;gBAAL,QAAK,aAAiBG,IAAAA,gBAAQ,EAACF,OAAOG,OAAO,EAAE;oBAAEC,KAAKV;oBAASW,SAASL,OAAOK,OAAO;gBAAC,uBAAlF,UAAA,8BAAA,SAAA,0BAAA,kCAAsF;oBAAtF,IAAMC,UAAN;wBAESX;oBADZ,IAAMT,UAAUW,cAAczB,YAAYkC,WAAWA;oBACrD,IAAMC,OAAMZ,gBAAAA,SAASa,GAAG,CAACtB,sBAAbS,2BAAAA,gBAAyB,IAAItB;oBACzCkC,IAAIE,GAAG,CAACV;oBACRJ,SAASY,GAAG,CAACrB,SAASqB;gBACxB;;gBALK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAMP;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,2EAA2E;IAC3E,IAAMG,YAAYC,IAAAA,sBAAY,EAAClB;IAC/B,IAAMmB,kBAAkBF,YAAY,IAAIrC,IAAIyB,IAAAA,qBAAW,EAACL,KAAKoB,MAAM,CAAC,SAACd;eAASe,IAAAA,+BAAqB,EAACrB,KAAKM;UAAU;IAEnH,IAAMgB,QAAoB,EAAE;QACvB,mCAAA,4BAAA;;QAAL,QAAK,aAAiB,AAAC,qBAAGpB,SAASqB,IAAI,IAAIC,IAAI,uBAA1C,UAAA,8BAAA,SAAA,0BAAA,kCAA8C;YAA9C,IAAM/B,WAAN;YACH,IAAMgC,UAAU5B,IAAAA,cAAI,EAACI,SAASR,WAAU,sDAAsD;YAC9F,yFAAyF;YACzF,uFAAuF;YACvF,IAAMiC,KAAKC,IAAAA,gBAAQ,EAACF,SAAS3B;YAC7B,IAAI,EAAC4B,eAAAA,yBAAAA,GAAIE,MAAM,KAAI;YACnB,IAAMpB,UAAU,AAAC,qBAAIN,SAASa,GAAG,CAACtB,WAA0B+B,IAAI;YAChE,IAAMK,QAAQV,oBAAoB,QAAQX,QAAQsB,IAAI,CAAC,SAACxB;uBAASa,gBAAgBY,GAAG,CAACzB;;YACrFgB,MAAMU,IAAI,CAAC;gBAAEvC,SAAAA;gBAASgC,SAAAA;gBAASQ,SAASP,GAAGO,OAAO;gBAAEC,MAAMR,GAAGQ,IAAI;gBAAE1B,SAAAA;gBAASqB,OAAAA;YAAM;QACpF;;QATK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAOP;AACT;AAiBA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASa,SAASpD,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOqD,OAAOrD,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOsD,OAAOC,aAAa,CAACvD,SAASqD,OAAOrD,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOwD,KAAKC,SAAS,CAACzD;AACxB;AAEA,gEAAgE;AAChE,SAAS0D,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,IAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKnE,WAAW,OAAO;QAAE6D,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASG,oBAAoB3D,OAAe,EAAE4D,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCC,IAAAA,WAAK,EAACH,KAAK;QACTI,MAAAA,SAAAA,KAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACC,IAAAA,kBAAY,EAACD,KAAKE,GAAG,GAAG,OAAO7E;YACpCuE,QAAQ;YACR,OAAOC,WAAK,CAACM,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIP,OAAOD,SAAStB,IAAI,CAAC,AAAC,YAAmB,OAARvC,SAAQ;AAC/C;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASsE,UAAUC,OAAe;IAChC,OAAOA,QAAQpE,KAAK,CAAC,KAAK,CAAC,EAAE,CAACV,OAAO,CAAC,SAAS;AACjD;AAEA,SAAS+E,iBAAiBxE,OAAe,EAAEoD,EAAU,EAAES,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,IAAMD,MAAMa,IAAAA,mBAAa,EAACrB,IAAI;QAAEsB,UAAU;IAAS;IACnD,IAAMC,UAAUf,IAAIgB,MAAM,CAACjD,MAAM,CAAC,SAACkD;eAAQ,CAACzF,oBAAoBkD,GAAG,CAACuC,IAAIC,IAAI;;IAC5E,IAAIH,QAAQnB,MAAM,GAAG,GAAG;QACtB,IAAMuB,SAASJ,QAAQnB,MAAM,GAAG,IAAI,AAAC,SAA2B,OAAnBmB,QAAQnB,MAAM,GAAG,GAAE,YAAU;QAC1E,IAAMwB,aAAa,AAAC,GAAkCD,OAAhCT,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,GAAW,OAAPQ;QACtDlB,SAAStB,IAAI,CAAC,AAAC,YAA2EyC,OAAhEhF,SAAQ,0DAAmE,OAAXgF;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOrB,IAAIsB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,IAAMG,cAAaV,UAAU,AAACO,IAAcN,OAAO;QACnDV,SAAStB,IAAI,CAAC,AAAC,YAA2EyC,OAAhEhF,SAAQ,0DAAmE,OAAXgF;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS1F,WAAW,OAAO;QAAE0F,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,CAAA,OAAOC,qCAAP,SAAOA,KAAG,MAAM,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,IAAMD,cAAa;QACnBnB,SAAStB,IAAI,CAAC,AAAC,YAAsByC,OAAXhF,SAAQ,KAAc,OAAXgF,aAAW;QAChD,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IACArB,oBAAoB3D,SAAS4D,KAAKC;IAClC,OAAO;QAAEoB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEO,SAAS/F,UAAUoG,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMrC,MAAMsC,IAAAA,oBAAY,EAACF,KAAKrD,OAAO,EAAE;IACvC,IAAM6B,WAAqB,EAAE;IAE7B,IAA8Bb,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMmC,UAAYxC,kBAAlBK;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAE6B,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIR,iBAAiBa,KAAKrF,OAAO,EAAEoD,IAAIS,WAAlIoB,OAAqB7B,KAArB6B,MAAMD,aAAe5B,KAAf4B;IACd,IAAMS,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAO5D,IAAI,CAACmD,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMb,MAAN;YACH,IAAIrF,iBAAiBuD,GAAG,CAAC8B,MAAM;gBAC7BP,SAAStB,IAAI,CAAC,AAAC,YAAwD6B,OAA7CiB,KAAKrF,OAAO,EAAC,kCAAoC,OAAJoE,KAAI;gBAC3E;YACF;YACAqB,MAAM,CAACrB,IAAI,GAAG1B,SAASuC,IAAI,CAACb,IAAI;QAClC;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMuB,SAAS;QAAEC,OAAOvG,cAAc4F,KAAKW,KAAK;QAAGC,SAASxG,cAAc4F,KAAKY,OAAO;QAAGC,MAAMnG,UAAU6F;IAAS;IAElH,OAAO;QACL5B,KAAK;YACH5D,SAASqF,KAAKrF,OAAO;YACrBwC,SAAS6C,KAAK7C,OAAO;YACrBC,MAAM4C,KAAK5C,IAAI;YACf1B,SAASsE,KAAKtE,OAAO;YACrBkE,MAAMQ;YACNT,YAAAA;YACAW,QAAAA;YACAI,WAAWL,OAAOM,WAAW,CAACV,WAAW3D,MAAM,CAAC,SAACsE;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEpF,IAAI;qBAAEoF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhD,KAAKuC,SAASG;iBAAQ;;QAC1H;QACA9B,UAAAA;IACF;AACF"}

package/dist/esm/cli/index.d.ts

@@ -3,10 +3,8 @@ export declare const USAGE: {
     readonly init: "init";
     readonly watch: "watch [--force] [--config path]";
     readonly status: "status [--format table|json] [--config path]";
-    readonly check: "check [--format table|json] [--config path]";
-    readonly rebuild: "rebuild [--config path]";
     readonly download: "download [--config path]";
-    readonly sql: "sql \"<statement>\" [params...] [--format table|json] [--config path]";
+    readonly sql: "sql \"<statement>\" [params...] [--preset name] [--format table|json] [--config path]";
     readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
     readonly map: "map [--format table|json] [--config path]";
     readonly peek: "peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]";

package/dist/esm/cli/index.js

@@ -5,10 +5,8 @@ export const USAGE = {
     init: 'init',
     watch: 'watch [--force] [--config path]',
     status: 'status [--format table|json] [--config path]',
-    check: 'check [--format table|json] [--config path]',
-    rebuild: 'rebuild [--config path]',
     download: 'download [--config path]',
-    sql: 'sql "<statement>" [params...] [--format table|json] [--config path]',
+    sql: 'sql "<statement>" [params...] [--preset name] [--format table|json] [--config path]',
     search: 'search "<terms>" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where "<sql>"] [--k n] [--format table|json] [--config path]',
     map: 'map [--format table|json] [--config path]',
     peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where "<sql>"] [--format table|json] [--config path]',
@@ -22,8 +20,6 @@ export const COMMANDS = {
     init: ()=>import('./init.js'),
     watch: ()=>import('./watch.js'),
     status: ()=>import('./status.js'),
-    check: ()=>import('./check.js'),
-    rebuild: ()=>import('./rebuild.js'),
     download: ()=>import('./download.js'),
     sql: ()=>import('./sql.js'),
     search: ()=>import('./search.js'),

package/dist/esm/cli/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \" -- shared by cli.ts's\n// top-level usage() and the command's own parse() error/--help output, so there is one\n// copy of each line, not two that can drift.\nexport const USAGE = {\n  init: 'init',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  check: 'check [--format table|json] [--config path]',\n  rebuild: 'rebuild [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--format table|json] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved: a saved\n// entry in config can never shadow them.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  check: () => import('./check.ts'),\n  rebuild: () => import('./rebuild.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","check","rebuild","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,sFAAsF;AACtF,uFAAuF;AACvF,6CAA6C;AAC7C,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,OAAO;IACPC,SAAS;IACTC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,0FAA0F;AAC1F,yCAAyC;AACzC,OAAO,MAAMC,WAAgE;IAC3EZ,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,OAAO,IAAM,MAAM,CAAC;IACpBC,SAAS,IAAM,MAAM,CAAC;IACtBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \" -- shared by cli.ts's\n// top-level usage() and the command's own parse() error/--help output, so there is one\n// copy of each line, not two that can drift.\nexport const USAGE = {\n  init: 'init',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--preset name] [--format table|json] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved: a saved\n// entry in config can never shadow them.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,sFAAsF;AACtF,uFAAuF;AACvF,6CAA6C;AAC7C,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,0FAA0F;AAC1F,yCAAyC;AACzC,OAAO,MAAMC,WAAgE;IAC3EV,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}

package/dist/esm/cli/named.js

@@ -21,7 +21,9 @@ export default async function named(ctx, queryName) {
         process.exit(2);
     }
     if ('sql' in entry) {
-        runSql(cfg, entry.sql, params, format, `query "${queryName}"`);
+        // A saved statement written against `scope` is preset-agnostic, so the same entry can be
+        // re-pointed at another layer from the command line instead of being copied per preset.
+        runSql(cfg, entry.sql, params, format, `query "${queryName}"`, values.preset);
         return;
     }
     // A saved search's text is fixed in config; there is nowhere for a positional to bind.

package/dist/esm/cli/named.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/named.ts"],"sourcesContent":["import { search } from '../commands.ts';\nimport { printRows } from '../output.ts';\nimport { CONFIG, FORMAT, formatOf, parse, parseK, runSql, SEARCH_FLAGS, withDb } from './shared.ts';\nimport type { Ctx } from './types.ts';\n\n// Fallback when the first positional is not a command: a saved query, { sql } or { search }.\n// SEARCH_FLAGS override a saved search's fields the same way they override a preset's.\nexport default async function named(ctx: Ctx, queryName: string): Promise<void> {\n  const usage = `usage: ${ctx.name} ${queryName} [params...] [--format table|json] [--config path] [--where \"<sql>\"] [--k n] [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude]`;\n  const { values, positionals: params } = parse(ctx.argv, usage, { ...SEARCH_FLAGS, ...FORMAT, ...CONFIG });\n  const format = formatOf(values);\n  const configPath = values.config as string | undefined;\n\n  const cfg = ctx.resolveConfig(configPath);\n  const entry = cfg.queries[queryName];\n  if (entry === undefined) {\n    console.error(`unknown command or saved entry: \"${queryName}\"`);\n    console.error(`saved queries: ${Object.keys(cfg.queries).sort().join(', ')}`);\n    process.exit(2);\n  }\n\n  if ('sql' in entry) {\n    runSql(cfg, entry.sql, params, format, `query \"${queryName}\"`);\n    return;\n  }\n\n  // A saved search's text is fixed in config; there is nowhere for a positional to bind.\n  if (params.length > 0) {\n    ctx.usageError(`\"${queryName}\" is a saved search and takes no positional parameters; edit its \"search\" in sense.config.json, or use \"${ctx.name} search\" directly`);\n  }\n  const k = parseK(values.k as string | undefined, ctx.usageError) ?? entry.k;\n  const where = (values.where as string | undefined) ?? entry.where;\n  const preset = (values.preset as string | undefined) ?? entry.preset;\n  const include = (values.include as string[] | undefined) ?? entry.include;\n  const exclude = (values.exclude as string[] | undefined) ?? entry.exclude;\n  const noExclude = values['no-exclude'] === true;\n  await withDb(ctx, configPath, async (db, resolvedCfg) => printRows(await search(db, resolvedCfg, entry.search, { k, where, preset, include, exclude, noExclude }), format));\n}\n"],"names":["search","printRows","CONFIG","FORMAT","formatOf","parse","parseK","runSql","SEARCH_FLAGS","withDb","named","ctx","queryName","values","usage","name","positionals","params","argv","format","configPath","config","cfg","resolveConfig","entry","queries","undefined","console","error","Object","keys","sort","join","process","exit","sql","length","usageError","k","where","preset","include","exclude","noExclude","db","resolvedCfg"],"mappings":"AAAA,SAASA,MAAM,QAAQ,iBAAiB;AACxC,SAASC,SAAS,QAAQ,eAAe;AACzC,SAASC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAEC,KAAK,EAAEC,MAAM,EAAEC,MAAM,EAAEC,YAAY,EAAEC,MAAM,QAAQ,cAAc;AAGpG,6FAA6F;AAC7F,uFAAuF;AACvF,eAAe,eAAeC,MAAMC,GAAQ,EAAEC,SAAiB;QAuBnDN,SACKO,eACCA,gBACCA,iBACAA;IA1BjB,MAAMC,QAAQ,CAAC,OAAO,EAAEH,IAAII,IAAI,CAAC,CAAC,EAAEH,UAAU,qJAAqJ,CAAC;IACpM,MAAM,EAAEC,MAAM,EAAEG,aAAaC,MAAM,EAAE,GAAGZ,MAAMM,IAAIO,IAAI,EAAEJ,OAAO;QAAE,GAAGN,YAAY;QAAE,GAAGL,MAAM;QAAE,GAAGD,MAAM;IAAC;IACvG,MAAMiB,SAASf,SAASS;IACxB,MAAMO,aAAaP,OAAOQ,MAAM;IAEhC,MAAMC,MAAMX,IAAIY,aAAa,CAACH;IAC9B,MAAMI,QAAQF,IAAIG,OAAO,CAACb,UAAU;IACpC,IAAIY,UAAUE,WAAW;QACvBC,QAAQC,KAAK,CAAC,CAAC,iCAAiC,EAAEhB,UAAU,CAAC,CAAC;QAC9De,QAAQC,KAAK,CAAC,CAAC,eAAe,EAAEC,OAAOC,IAAI,CAACR,IAAIG,OAAO,EAAEM,IAAI,GAAGC,IAAI,CAAC,OAAO;QAC5EC,QAAQC,IAAI,CAAC;IACf;IAEA,IAAI,SAASV,OAAO;QAClBjB,OAAOe,KAAKE,MAAMW,GAAG,EAAElB,QAAQE,QAAQ,CAAC,OAAO,EAAEP,UAAU,CAAC,CAAC;QAC7D;IACF;IAEA,uFAAuF;IACvF,IAAIK,OAAOmB,MAAM,GAAG,GAAG;QACrBzB,IAAI0B,UAAU,CAAC,CAAC,CAAC,EAAEzB,UAAU,wGAAwG,EAAED,IAAII,IAAI,CAAC,iBAAiB,CAAC;IACpK;IACA,MAAMuB,KAAIhC,UAAAA,OAAOO,OAAOyB,CAAC,EAAwB3B,IAAI0B,UAAU,eAArD/B,qBAAAA,UAA0DkB,MAAMc,CAAC;IAC3E,MAAMC,SAAS1B,gBAAAA,OAAO0B,KAAK,cAAZ1B,2BAAAA,gBAAuCW,MAAMe,KAAK;IACjE,MAAMC,UAAU3B,iBAAAA,OAAO2B,MAAM,cAAb3B,4BAAAA,iBAAwCW,MAAMgB,MAAM;IACpE,MAAMC,WAAW5B,kBAAAA,OAAO4B,OAAO,cAAd5B,6BAAAA,kBAA2CW,MAAMiB,OAAO;IACzE,MAAMC,WAAW7B,kBAAAA,OAAO6B,OAAO,cAAd7B,6BAAAA,kBAA2CW,MAAMkB,OAAO;IACzE,MAAMC,YAAY9B,MAAM,CAAC,aAAa,KAAK;IAC3C,MAAMJ,OAAOE,KAAKS,YAAY,OAAOwB,IAAIC,cAAgB5C,UAAU,MAAMD,OAAO4C,IAAIC,aAAarB,MAAMxB,MAAM,EAAE;YAAEsC;YAAGC;YAAOC;YAAQC;YAASC;YAASC;QAAU,IAAIxB;AACrK"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/named.ts"],"sourcesContent":["import { search } from '../commands.ts';\nimport { printRows } from '../output.ts';\nimport { CONFIG, FORMAT, formatOf, parse, parseK, runSql, SEARCH_FLAGS, withDb } from './shared.ts';\nimport type { Ctx } from './types.ts';\n\n// Fallback when the first positional is not a command: a saved query, { sql } or { search }.\n// SEARCH_FLAGS override a saved search's fields the same way they override a preset's.\nexport default async function named(ctx: Ctx, queryName: string): Promise<void> {\n  const usage = `usage: ${ctx.name} ${queryName} [params...] [--format table|json] [--config path] [--where \"<sql>\"] [--k n] [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude]`;\n  const { values, positionals: params } = parse(ctx.argv, usage, { ...SEARCH_FLAGS, ...FORMAT, ...CONFIG });\n  const format = formatOf(values);\n  const configPath = values.config as string | undefined;\n\n  const cfg = ctx.resolveConfig(configPath);\n  const entry = cfg.queries[queryName];\n  if (entry === undefined) {\n    console.error(`unknown command or saved entry: \"${queryName}\"`);\n    console.error(`saved queries: ${Object.keys(cfg.queries).sort().join(', ')}`);\n    process.exit(2);\n  }\n\n  if ('sql' in entry) {\n    // A saved statement written against `scope` is preset-agnostic, so the same entry can be\n    // re-pointed at another layer from the command line instead of being copied per preset.\n    runSql(cfg, entry.sql, params, format, `query \"${queryName}\"`, values.preset as string | undefined);\n    return;\n  }\n\n  // A saved search's text is fixed in config; there is nowhere for a positional to bind.\n  if (params.length > 0) {\n    ctx.usageError(`\"${queryName}\" is a saved search and takes no positional parameters; edit its \"search\" in sense.config.json, or use \"${ctx.name} search\" directly`);\n  }\n  const k = parseK(values.k as string | undefined, ctx.usageError) ?? entry.k;\n  const where = (values.where as string | undefined) ?? entry.where;\n  const preset = (values.preset as string | undefined) ?? entry.preset;\n  const include = (values.include as string[] | undefined) ?? entry.include;\n  const exclude = (values.exclude as string[] | undefined) ?? entry.exclude;\n  const noExclude = values['no-exclude'] === true;\n  await withDb(ctx, configPath, async (db, resolvedCfg) => printRows(await search(db, resolvedCfg, entry.search, { k, where, preset, include, exclude, noExclude }), format));\n}\n"],"names":["search","printRows","CONFIG","FORMAT","formatOf","parse","parseK","runSql","SEARCH_FLAGS","withDb","named","ctx","queryName","values","usage","name","positionals","params","argv","format","configPath","config","cfg","resolveConfig","entry","queries","undefined","console","error","Object","keys","sort","join","process","exit","sql","preset","length","usageError","k","where","include","exclude","noExclude","db","resolvedCfg"],"mappings":"AAAA,SAASA,MAAM,QAAQ,iBAAiB;AACxC,SAASC,SAAS,QAAQ,eAAe;AACzC,SAASC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAEC,KAAK,EAAEC,MAAM,EAAEC,MAAM,EAAEC,YAAY,EAAEC,MAAM,QAAQ,cAAc;AAGpG,6FAA6F;AAC7F,uFAAuF;AACvF,eAAe,eAAeC,MAAMC,GAAQ,EAAEC,SAAiB;QAyBnDN,SACKO,eACCA,gBACCA,iBACAA;IA5BjB,MAAMC,QAAQ,CAAC,OAAO,EAAEH,IAAII,IAAI,CAAC,CAAC,EAAEH,UAAU,qJAAqJ,CAAC;IACpM,MAAM,EAAEC,MAAM,EAAEG,aAAaC,MAAM,EAAE,GAAGZ,MAAMM,IAAIO,IAAI,EAAEJ,OAAO;QAAE,GAAGN,YAAY;QAAE,GAAGL,MAAM;QAAE,GAAGD,MAAM;IAAC;IACvG,MAAMiB,SAASf,SAASS;IACxB,MAAMO,aAAaP,OAAOQ,MAAM;IAEhC,MAAMC,MAAMX,IAAIY,aAAa,CAACH;IAC9B,MAAMI,QAAQF,IAAIG,OAAO,CAACb,UAAU;IACpC,IAAIY,UAAUE,WAAW;QACvBC,QAAQC,KAAK,CAAC,CAAC,iCAAiC,EAAEhB,UAAU,CAAC,CAAC;QAC9De,QAAQC,KAAK,CAAC,CAAC,eAAe,EAAEC,OAAOC,IAAI,CAACR,IAAIG,OAAO,EAAEM,IAAI,GAAGC,IAAI,CAAC,OAAO;QAC5EC,QAAQC,IAAI,CAAC;IACf;IAEA,IAAI,SAASV,OAAO;QAClB,yFAAyF;QACzF,wFAAwF;QACxFjB,OAAOe,KAAKE,MAAMW,GAAG,EAAElB,QAAQE,QAAQ,CAAC,OAAO,EAAEP,UAAU,CAAC,CAAC,EAAEC,OAAOuB,MAAM;QAC5E;IACF;IAEA,uFAAuF;IACvF,IAAInB,OAAOoB,MAAM,GAAG,GAAG;QACrB1B,IAAI2B,UAAU,CAAC,CAAC,CAAC,EAAE1B,UAAU,wGAAwG,EAAED,IAAII,IAAI,CAAC,iBAAiB,CAAC;IACpK;IACA,MAAMwB,KAAIjC,UAAAA,OAAOO,OAAO0B,CAAC,EAAwB5B,IAAI2B,UAAU,eAArDhC,qBAAAA,UAA0DkB,MAAMe,CAAC;IAC3E,MAAMC,SAAS3B,gBAAAA,OAAO2B,KAAK,cAAZ3B,2BAAAA,gBAAuCW,MAAMgB,KAAK;IACjE,MAAMJ,UAAUvB,iBAAAA,OAAOuB,MAAM,cAAbvB,4BAAAA,iBAAwCW,MAAMY,MAAM;IACpE,MAAMK,WAAW5B,kBAAAA,OAAO4B,OAAO,cAAd5B,6BAAAA,kBAA2CW,MAAMiB,OAAO;IACzE,MAAMC,WAAW7B,kBAAAA,OAAO6B,OAAO,cAAd7B,6BAAAA,kBAA2CW,MAAMkB,OAAO;IACzE,MAAMC,YAAY9B,MAAM,CAAC,aAAa,KAAK;IAC3C,MAAMJ,OAAOE,KAAKS,YAAY,OAAOwB,IAAIC,cAAgB5C,UAAU,MAAMD,OAAO4C,IAAIC,aAAarB,MAAMxB,MAAM,EAAE;YAAEuC;YAAGC;YAAOJ;YAAQK;YAASC;YAASC;QAAU,IAAIxB;AACrK"}

package/dist/esm/cli/shared.d.ts

@@ -16,5 +16,5 @@ export declare function parse(argv: string[], usage: string, options: ParseArgsO
 export declare function parseK(k: string | undefined, usageError: (message: string) => never): number | undefined;
 export declare function printWarnings(warnings: string[]): void;
 export declare function withDb(ctx: Ctx, configPath: string | undefined, fn: (db: OpenResult['db'], cfg: ResolvedConfig) => void | Promise<void>): Promise<void>;
-export declare function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: 'table' | 'json', label: string): void;
+export declare function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: 'table' | 'json', label: string, preset?: string): void;
 export {};

package/dist/esm/cli/shared.js

@@ -1,4 +1,5 @@
 import { parseArgs } from 'node:util';
+import { resolvePreset } from '../config.js';
 import { open } from '../db.js';
 import { printRows } from '../output.js';
 import { searchError } from '../search-error.js';
@@ -114,16 +115,35 @@ export async function withDb(ctx, configPath, fn) {
         db.close();
     }
 }
+// `--preset` on SQL binds the scope rather than applying it: the statement joins `scope`
+// itself. Filtering behind the query's back is not available -- the obvious version, temp views
+// shadowing the base tables, cannot cover `content`, because FTS5 `MATCH` uses the table name
+// as a hidden column and a view has none. A flag that silently scoped three tables of four
+// would look scoped and not be. See plans/vault-field-report-fixes.md item F.
+function bindScope(db, cfg, preset) {
+    const { name } = resolvePreset(cfg, preset); // unknown names throw, listing what is declared
+    db.exec('DROP TABLE IF EXISTS temp.scope');
+    db.exec('CREATE TEMP TABLE scope ("path" TEXT PRIMARY KEY)');
+    db.prepare('INSERT INTO temp.scope ("path") SELECT "path" FROM preset_files WHERE preset = ?').run(name);
+}
 // An unbound `?` silently binds NULL, so mismatched param counts fail loudly instead.
-export function runSql(cfg, sql, params, format, label) {
+export function runSql(cfg, sql, params, format, label, preset) {
     var _sql_match;
     const placeholderCount = ((_sql_match = sql.match(/\?/g)) !== null && _sql_match !== void 0 ? _sql_match : []).length;
     if (params.length !== placeholderCount) {
         console.error(`${label} expects ${placeholderCount} parameter(s), got ${params.length}`);
         process.exit(2);
     }
+    // Naming a preset and never joining it would return the whole index while reading as scoped,
+    // which is the one hazard of binding rather than applying. Refuse instead.
+    if (preset !== undefined && !/\bscope\b/i.test(sql)) {
+        console.error(`--preset binds a temporary "scope" table of the preset's paths, and ${label} never joins it, so the preset would have no effect`);
+        console.error(`add: JOIN scope ON scope."path" = <table>."path"`);
+        process.exit(2);
+    }
     const { db, warnings } = open(cfg);
     printWarnings(warnings);
+    if (preset !== undefined) bindScope(db, cfg, preset);
     let rows;
     try {
         rows = db.prepare(sql).all(...params);

package/dist/esm/cli/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/shared.ts"],"sourcesContent":["import type { ParseArgsOptionsConfig } from 'node:util';\nimport { parseArgs } from 'node:util';\nimport type { ResolvedConfig, SearchOverrides } from '../config.ts';\nimport type { OpenResult } from '../db.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport { searchError } from '../search-error.ts';\nimport type { Ctx } from './types.ts';\n\n// Spreadable option fragments -- one flag name keeps one meaning across every command's table.\nexport const FORMAT: ParseArgsOptionsConfig = { format: { type: 'string', default: 'table' } };\nexport const CONFIG: ParseArgsOptionsConfig = { config: { type: 'string' } };\n// The scope vocabulary every scoped command shares: named preset, ad hoc include/exclude\n// globs, a where SQL condition. search adds k on top.\nexport const SCOPE: ParseArgsOptionsConfig = {\n  where: { type: 'string' },\n  preset: { type: 'string' },\n  include: { type: 'string', multiple: true },\n  exclude: { type: 'string', multiple: true },\n  // Widening, which no other flag can express: --include and --exclude each override their\n  // own side of the preset, so neither can drop an exclusion the preset declares.\n  'no-exclude': { type: 'boolean', default: false },\n};\nexport const SEARCH_FLAGS: ParseArgsOptionsConfig = { ...SCOPE, k: { type: 'string' } };\n\ntype Values = Record<string, string | boolean | string[] | undefined>;\n\n// The SCOPE fragment's parsed values as the overrides every scoped command passes down. One\n// place to read them, so a new scope field is added to the fragment and here, not in each\n// command's call.\nexport function scopeOf(values: Values): SearchOverrides {\n  return {\n    preset: values.preset as string | undefined,\n    include: values.include as string[] | undefined,\n    exclude: values.exclude as string[] | undefined,\n    where: values.where as string | undefined,\n    noExclude: values['no-exclude'] === true,\n  };\n}\n\nexport function formatOf(values: Values): 'table' | 'json' {\n  return values.format === 'json' ? 'json' : 'table';\n}\n\n// Per-command parseArgs: strict (a foreign flag exits 2), and every table gets --help for free.\nexport function parse(argv: string[], usage: string, options: ParseArgsOptionsConfig): { values: Values; positionals: string[] } {\n  let values: Values;\n  let positionals: string[];\n  try {\n    ({ values, positionals } = parseArgs({\n      args: argv,\n      options: { ...options, help: { type: 'boolean', default: false, short: 'h' } },\n      strict: true,\n      allowPositionals: true,\n    }));\n  } catch (err) {\n    console.error((err as Error).message);\n    console.error(usage);\n    process.exit(2);\n  }\n  if (values.help) {\n    console.log(usage);\n    process.exit(0);\n  }\n  return { values, positionals };\n}\n\n// --k must be a positive integer: SQLite reads a bound LIMIT of -1 as \"unlimited\" and 0 as\n// \"nothing\", and parseInt would silently truncate \"5.9\" -- all three are caller mistakes\n// worth a usage error, matching the config-level SavedSearch validation.\nexport function parseK(k: string | undefined, usageError: (message: string) => never): number | undefined {\n  if (k === undefined) return undefined;\n  const parsed = Number(k);\n  if (!Number.isInteger(parsed) || parsed <= 0) usageError(`--k expects a positive integer, got \"${k}\"`);\n  return parsed;\n}\n\n// Shared open-query-close envelope for commands that touch the tree.\n\nexport function printWarnings(warnings: string[]): void {\n  for (const w of warnings) console.warn(w);\n}\n\nexport async function withDb(ctx: Ctx, configPath: string | undefined, fn: (db: OpenResult['db'], cfg: ResolvedConfig) => void | Promise<void>): Promise<void> {\n  const cfg = ctx.resolveConfig(configPath);\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  try {\n    await fn(db, cfg);\n  } finally {\n    db.close();\n  }\n}\n\n// An unbound `?` silently binds NULL, so mismatched param counts fail loudly instead.\nexport function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: 'table' | 'json', label: string): void {\n  const placeholderCount = (sql.match(/\\?/g) ?? []).length;\n  if (params.length !== placeholderCount) {\n    console.error(`${label} expects ${placeholderCount} parameter(s), got ${params.length}`);\n    process.exit(2);\n  }\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  let rows: Row[];\n  try {\n    rows = db.prepare(sql).all(...params) as Row[];\n  } catch (err) {\n    db.close();\n    // A saved query or ad-hoc SQL can carry `content MATCH ?` too, so the same FTS5\n    // punctuation trap applies -- the bound parameters are the search terms there. SQL\n    // without MATCH gets its error verbatim; search advice on a plain typo would mislead.\n    if (/\\bMATCH\\b/i.test(sql)) throw searchError(err as Error, params.join(' '));\n    throw err;\n  }\n  printRows(rows, format);\n  db.close();\n}\n"],"names":["parseArgs","open","printRows","searchError","FORMAT","format","type","default","CONFIG","config","SCOPE","where","preset","include","multiple","exclude","SEARCH_FLAGS","k","scopeOf","values","noExclude","formatOf","parse","argv","usage","options","positionals","args","help","short","strict","allowPositionals","err","console","error","message","process","exit","log","parseK","usageError","undefined","parsed","Number","isInteger","printWarnings","warnings","w","warn","withDb","ctx","configPath","fn","cfg","resolveConfig","db","close","runSql","sql","params","label","placeholderCount","match","length","rows","prepare","all","test","join"],"mappings":"AACA,SAASA,SAAS,QAAQ,YAAY;AAGtC,SAASC,IAAI,QAAQ,WAAW;AAEhC,SAASC,SAAS,QAAQ,eAAe;AACzC,SAASC,WAAW,QAAQ,qBAAqB;AAGjD,+FAA+F;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEC,MAAM;QAAUC,SAAS;IAAQ;AAAE,EAAE;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEH,MAAM;IAAS;AAAE,EAAE;AAC7E,yFAAyF;AACzF,sDAAsD;AACtD,OAAO,MAAMI,QAAgC;IAC3CC,OAAO;QAAEL,MAAM;IAAS;IACxBM,QAAQ;QAAEN,MAAM;IAAS;IACzBO,SAAS;QAAEP,MAAM;QAAUQ,UAAU;IAAK;IAC1CC,SAAS;QAAET,MAAM;QAAUQ,UAAU;IAAK;IAC1C,yFAAyF;IACzF,gFAAgF;IAChF,cAAc;QAAER,MAAM;QAAWC,SAAS;IAAM;AAClD,EAAE;AACF,OAAO,MAAMS,eAAuC;IAAE,GAAGN,KAAK;IAAEO,GAAG;QAAEX,MAAM;IAAS;AAAE,EAAE;AAIxF,4FAA4F;AAC5F,0FAA0F;AAC1F,kBAAkB;AAClB,OAAO,SAASY,QAAQC,MAAc;IACpC,OAAO;QACLP,QAAQO,OAAOP,MAAM;QACrBC,SAASM,OAAON,OAAO;QACvBE,SAASI,OAAOJ,OAAO;QACvBJ,OAAOQ,OAAOR,KAAK;QACnBS,WAAWD,MAAM,CAAC,aAAa,KAAK;IACtC;AACF;AAEA,OAAO,SAASE,SAASF,MAAc;IACrC,OAAOA,OAAOd,MAAM,KAAK,SAAS,SAAS;AAC7C;AAEA,gGAAgG;AAChG,OAAO,SAASiB,MAAMC,IAAc,EAAEC,KAAa,EAAEC,OAA+B;IAClF,IAAIN;IACJ,IAAIO;IACJ,IAAI;QACD,CAAA,EAAEP,MAAM,EAAEO,WAAW,EAAE,GAAG1B,UAAU;YACnC2B,MAAMJ;YACNE,SAAS;gBAAE,GAAGA,OAAO;gBAAEG,MAAM;oBAAEtB,MAAM;oBAAWC,SAAS;oBAAOsB,OAAO;gBAAI;YAAE;YAC7EC,QAAQ;YACRC,kBAAkB;QACpB,EAAC;IACH,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,AAACF,IAAcG,OAAO;QACpCF,QAAQC,KAAK,CAACV;QACdY,QAAQC,IAAI,CAAC;IACf;IACA,IAAIlB,OAAOS,IAAI,EAAE;QACfK,QAAQK,GAAG,CAACd;QACZY,QAAQC,IAAI,CAAC;IACf;IACA,OAAO;QAAElB;QAAQO;IAAY;AAC/B;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,yEAAyE;AACzE,OAAO,SAASa,OAAOtB,CAAqB,EAAEuB,UAAsC;IAClF,IAAIvB,MAAMwB,WAAW,OAAOA;IAC5B,MAAMC,SAASC,OAAO1B;IACtB,IAAI,CAAC0B,OAAOC,SAAS,CAACF,WAAWA,UAAU,GAAGF,WAAW,CAAC,qCAAqC,EAAEvB,EAAE,CAAC,CAAC;IACrG,OAAOyB;AACT;AAEA,qEAAqE;AAErE,OAAO,SAASG,cAAcC,QAAkB;IAC9C,KAAK,MAAMC,KAAKD,SAAUb,QAAQe,IAAI,CAACD;AACzC;AAEA,OAAO,eAAeE,OAAOC,GAAQ,EAAEC,UAA8B,EAAEC,EAAuE;IAC5I,MAAMC,MAAMH,IAAII,aAAa,CAACH;IAC9B,MAAM,EAAEI,EAAE,EAAET,QAAQ,EAAE,GAAG7C,KAAKoD;IAC9BR,cAAcC;IACd,IAAI;QACF,MAAMM,GAAGG,IAAIF;IACf,SAAU;QACRE,GAAGC,KAAK;IACV;AACF;AAEA,sFAAsF;AACtF,OAAO,SAASC,OAAOJ,GAAmB,EAAEK,GAAW,EAAEC,MAAgB,EAAEtD,MAAwB,EAAEuD,KAAa;QACtFF;IAA1B,MAAMG,mBAAmB,EAACH,aAAAA,IAAII,KAAK,CAAC,oBAAVJ,wBAAAA,aAAoB,EAAE,EAAEK,MAAM;IACxD,IAAIJ,OAAOI,MAAM,KAAKF,kBAAkB;QACtC5B,QAAQC,KAAK,CAAC,GAAG0B,MAAM,SAAS,EAAEC,iBAAiB,mBAAmB,EAAEF,OAAOI,MAAM,EAAE;QACvF3B,QAAQC,IAAI,CAAC;IACf;IACA,MAAM,EAAEkB,EAAE,EAAET,QAAQ,EAAE,GAAG7C,KAAKoD;IAC9BR,cAAcC;IACd,IAAIkB;IACJ,IAAI;QACFA,OAAOT,GAAGU,OAAO,CAACP,KAAKQ,GAAG,IAAIP;IAChC,EAAE,OAAO3B,KAAK;QACZuB,GAAGC,KAAK;QACR,gFAAgF;QAChF,mFAAmF;QACnF,sFAAsF;QACtF,IAAI,aAAaW,IAAI,CAACT,MAAM,MAAMvD,YAAY6B,KAAc2B,OAAOS,IAAI,CAAC;QACxE,MAAMpC;IACR;IACA9B,UAAU8D,MAAM3D;IAChBkD,GAAGC,KAAK;AACV"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/shared.ts"],"sourcesContent":["import type { ParseArgsOptionsConfig } from 'node:util';\nimport { parseArgs } from 'node:util';\nimport type { ResolvedConfig, SearchOverrides } from '../config.ts';\nimport { resolvePreset } from '../config.ts';\nimport type { OpenResult } from '../db.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport { searchError } from '../search-error.ts';\nimport type { Ctx } from './types.ts';\n\n// Spreadable option fragments -- one flag name keeps one meaning across every command's table.\nexport const FORMAT: ParseArgsOptionsConfig = { format: { type: 'string', default: 'table' } };\nexport const CONFIG: ParseArgsOptionsConfig = { config: { type: 'string' } };\n// The scope vocabulary every scoped command shares: named preset, ad hoc include/exclude\n// globs, a where SQL condition. search adds k on top.\nexport const SCOPE: ParseArgsOptionsConfig = {\n  where: { type: 'string' },\n  preset: { type: 'string' },\n  include: { type: 'string', multiple: true },\n  exclude: { type: 'string', multiple: true },\n  // Widening, which no other flag can express: --include and --exclude each override their\n  // own side of the preset, so neither can drop an exclusion the preset declares.\n  'no-exclude': { type: 'boolean', default: false },\n};\nexport const SEARCH_FLAGS: ParseArgsOptionsConfig = { ...SCOPE, k: { type: 'string' } };\n\ntype Values = Record<string, string | boolean | string[] | undefined>;\n\n// The SCOPE fragment's parsed values as the overrides every scoped command passes down. One\n// place to read them, so a new scope field is added to the fragment and here, not in each\n// command's call.\nexport function scopeOf(values: Values): SearchOverrides {\n  return {\n    preset: values.preset as string | undefined,\n    include: values.include as string[] | undefined,\n    exclude: values.exclude as string[] | undefined,\n    where: values.where as string | undefined,\n    noExclude: values['no-exclude'] === true,\n  };\n}\n\nexport function formatOf(values: Values): 'table' | 'json' {\n  return values.format === 'json' ? 'json' : 'table';\n}\n\n// Per-command parseArgs: strict (a foreign flag exits 2), and every table gets --help for free.\nexport function parse(argv: string[], usage: string, options: ParseArgsOptionsConfig): { values: Values; positionals: string[] } {\n  let values: Values;\n  let positionals: string[];\n  try {\n    ({ values, positionals } = parseArgs({\n      args: argv,\n      options: { ...options, help: { type: 'boolean', default: false, short: 'h' } },\n      strict: true,\n      allowPositionals: true,\n    }));\n  } catch (err) {\n    console.error((err as Error).message);\n    console.error(usage);\n    process.exit(2);\n  }\n  if (values.help) {\n    console.log(usage);\n    process.exit(0);\n  }\n  return { values, positionals };\n}\n\n// --k must be a positive integer: SQLite reads a bound LIMIT of -1 as \"unlimited\" and 0 as\n// \"nothing\", and parseInt would silently truncate \"5.9\" -- all three are caller mistakes\n// worth a usage error, matching the config-level SavedSearch validation.\nexport function parseK(k: string | undefined, usageError: (message: string) => never): number | undefined {\n  if (k === undefined) return undefined;\n  const parsed = Number(k);\n  if (!Number.isInteger(parsed) || parsed <= 0) usageError(`--k expects a positive integer, got \"${k}\"`);\n  return parsed;\n}\n\n// Shared open-query-close envelope for commands that touch the tree.\n\nexport function printWarnings(warnings: string[]): void {\n  for (const w of warnings) console.warn(w);\n}\n\nexport async function withDb(ctx: Ctx, configPath: string | undefined, fn: (db: OpenResult['db'], cfg: ResolvedConfig) => void | Promise<void>): Promise<void> {\n  const cfg = ctx.resolveConfig(configPath);\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  try {\n    await fn(db, cfg);\n  } finally {\n    db.close();\n  }\n}\n\n// `--preset` on SQL binds the scope rather than applying it: the statement joins `scope`\n// itself. Filtering behind the query's back is not available -- the obvious version, temp views\n// shadowing the base tables, cannot cover `content`, because FTS5 `MATCH` uses the table name\n// as a hidden column and a view has none. A flag that silently scoped three tables of four\n// would look scoped and not be. See plans/vault-field-report-fixes.md item F.\nfunction bindScope(db: OpenResult['db'], cfg: ResolvedConfig, preset: string): void {\n  const { name } = resolvePreset(cfg, preset); // unknown names throw, listing what is declared\n  db.exec('DROP TABLE IF EXISTS temp.scope');\n  db.exec('CREATE TEMP TABLE scope (\"path\" TEXT PRIMARY KEY)');\n  db.prepare('INSERT INTO temp.scope (\"path\") SELECT \"path\" FROM preset_files WHERE preset = ?').run(name);\n}\n\n// An unbound `?` silently binds NULL, so mismatched param counts fail loudly instead.\nexport function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: 'table' | 'json', label: string, preset?: string): void {\n  const placeholderCount = (sql.match(/\\?/g) ?? []).length;\n  if (params.length !== placeholderCount) {\n    console.error(`${label} expects ${placeholderCount} parameter(s), got ${params.length}`);\n    process.exit(2);\n  }\n  // Naming a preset and never joining it would return the whole index while reading as scoped,\n  // which is the one hazard of binding rather than applying. Refuse instead.\n  if (preset !== undefined && !/\\bscope\\b/i.test(sql)) {\n    console.error(`--preset binds a temporary \"scope\" table of the preset's paths, and ${label} never joins it, so the preset would have no effect`);\n    console.error(`add: JOIN scope ON scope.\"path\" = <table>.\"path\"`);\n    process.exit(2);\n  }\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  if (preset !== undefined) bindScope(db, cfg, preset);\n  let rows: Row[];\n  try {\n    rows = db.prepare(sql).all(...params) as Row[];\n  } catch (err) {\n    db.close();\n    // A saved query or ad-hoc SQL can carry `content MATCH ?` too, so the same FTS5\n    // punctuation trap applies -- the bound parameters are the search terms there. SQL\n    // without MATCH gets its error verbatim; search advice on a plain typo would mislead.\n    if (/\\bMATCH\\b/i.test(sql)) throw searchError(err as Error, params.join(' '));\n    throw err;\n  }\n  printRows(rows, format);\n  db.close();\n}\n"],"names":["parseArgs","resolvePreset","open","printRows","searchError","FORMAT","format","type","default","CONFIG","config","SCOPE","where","preset","include","multiple","exclude","SEARCH_FLAGS","k","scopeOf","values","noExclude","formatOf","parse","argv","usage","options","positionals","args","help","short","strict","allowPositionals","err","console","error","message","process","exit","log","parseK","usageError","undefined","parsed","Number","isInteger","printWarnings","warnings","w","warn","withDb","ctx","configPath","fn","cfg","resolveConfig","db","close","bindScope","name","exec","prepare","run","runSql","sql","params","label","placeholderCount","match","length","test","rows","all","join"],"mappings":"AACA,SAASA,SAAS,QAAQ,YAAY;AAEtC,SAASC,aAAa,QAAQ,eAAe;AAE7C,SAASC,IAAI,QAAQ,WAAW;AAEhC,SAASC,SAAS,QAAQ,eAAe;AACzC,SAASC,WAAW,QAAQ,qBAAqB;AAGjD,+FAA+F;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEC,MAAM;QAAUC,SAAS;IAAQ;AAAE,EAAE;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEH,MAAM;IAAS;AAAE,EAAE;AAC7E,yFAAyF;AACzF,sDAAsD;AACtD,OAAO,MAAMI,QAAgC;IAC3CC,OAAO;QAAEL,MAAM;IAAS;IACxBM,QAAQ;QAAEN,MAAM;IAAS;IACzBO,SAAS;QAAEP,MAAM;QAAUQ,UAAU;IAAK;IAC1CC,SAAS;QAAET,MAAM;QAAUQ,UAAU;IAAK;IAC1C,yFAAyF;IACzF,gFAAgF;IAChF,cAAc;QAAER,MAAM;QAAWC,SAAS;IAAM;AAClD,EAAE;AACF,OAAO,MAAMS,eAAuC;IAAE,GAAGN,KAAK;IAAEO,GAAG;QAAEX,MAAM;IAAS;AAAE,EAAE;AAIxF,4FAA4F;AAC5F,0FAA0F;AAC1F,kBAAkB;AAClB,OAAO,SAASY,QAAQC,MAAc;IACpC,OAAO;QACLP,QAAQO,OAAOP,MAAM;QACrBC,SAASM,OAAON,OAAO;QACvBE,SAASI,OAAOJ,OAAO;QACvBJ,OAAOQ,OAAOR,KAAK;QACnBS,WAAWD,MAAM,CAAC,aAAa,KAAK;IACtC;AACF;AAEA,OAAO,SAASE,SAASF,MAAc;IACrC,OAAOA,OAAOd,MAAM,KAAK,SAAS,SAAS;AAC7C;AAEA,gGAAgG;AAChG,OAAO,SAASiB,MAAMC,IAAc,EAAEC,KAAa,EAAEC,OAA+B;IAClF,IAAIN;IACJ,IAAIO;IACJ,IAAI;QACD,CAAA,EAAEP,MAAM,EAAEO,WAAW,EAAE,GAAG3B,UAAU;YACnC4B,MAAMJ;YACNE,SAAS;gBAAE,GAAGA,OAAO;gBAAEG,MAAM;oBAAEtB,MAAM;oBAAWC,SAAS;oBAAOsB,OAAO;gBAAI;YAAE;YAC7EC,QAAQ;YACRC,kBAAkB;QACpB,EAAC;IACH,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,AAACF,IAAcG,OAAO;QACpCF,QAAQC,KAAK,CAACV;QACdY,QAAQC,IAAI,CAAC;IACf;IACA,IAAIlB,OAAOS,IAAI,EAAE;QACfK,QAAQK,GAAG,CAACd;QACZY,QAAQC,IAAI,CAAC;IACf;IACA,OAAO;QAAElB;QAAQO;IAAY;AAC/B;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,yEAAyE;AACzE,OAAO,SAASa,OAAOtB,CAAqB,EAAEuB,UAAsC;IAClF,IAAIvB,MAAMwB,WAAW,OAAOA;IAC5B,MAAMC,SAASC,OAAO1B;IACtB,IAAI,CAAC0B,OAAOC,SAAS,CAACF,WAAWA,UAAU,GAAGF,WAAW,CAAC,qCAAqC,EAAEvB,EAAE,CAAC,CAAC;IACrG,OAAOyB;AACT;AAEA,qEAAqE;AAErE,OAAO,SAASG,cAAcC,QAAkB;IAC9C,KAAK,MAAMC,KAAKD,SAAUb,QAAQe,IAAI,CAACD;AACzC;AAEA,OAAO,eAAeE,OAAOC,GAAQ,EAAEC,UAA8B,EAAEC,EAAuE;IAC5I,MAAMC,MAAMH,IAAII,aAAa,CAACH;IAC9B,MAAM,EAAEI,EAAE,EAAET,QAAQ,EAAE,GAAG7C,KAAKoD;IAC9BR,cAAcC;IACd,IAAI;QACF,MAAMM,GAAGG,IAAIF;IACf,SAAU;QACRE,GAAGC,KAAK;IACV;AACF;AAEA,yFAAyF;AACzF,gGAAgG;AAChG,8FAA8F;AAC9F,2FAA2F;AAC3F,8EAA8E;AAC9E,SAASC,UAAUF,EAAoB,EAAEF,GAAmB,EAAEzC,MAAc;IAC1E,MAAM,EAAE8C,IAAI,EAAE,GAAG1D,cAAcqD,KAAKzC,SAAS,gDAAgD;IAC7F2C,GAAGI,IAAI,CAAC;IACRJ,GAAGI,IAAI,CAAC;IACRJ,GAAGK,OAAO,CAAC,oFAAoFC,GAAG,CAACH;AACrG;AAEA,sFAAsF;AACtF,OAAO,SAASI,OAAOT,GAAmB,EAAEU,GAAW,EAAEC,MAAgB,EAAE3D,MAAwB,EAAE4D,KAAa,EAAErD,MAAe;QACvGmD;IAA1B,MAAMG,mBAAmB,EAACH,aAAAA,IAAII,KAAK,CAAC,oBAAVJ,wBAAAA,aAAoB,EAAE,EAAEK,MAAM;IACxD,IAAIJ,OAAOI,MAAM,KAAKF,kBAAkB;QACtCjC,QAAQC,KAAK,CAAC,GAAG+B,MAAM,SAAS,EAAEC,iBAAiB,mBAAmB,EAAEF,OAAOI,MAAM,EAAE;QACvFhC,QAAQC,IAAI,CAAC;IACf;IACA,6FAA6F;IAC7F,2EAA2E;IAC3E,IAAIzB,WAAW6B,aAAa,CAAC,aAAa4B,IAAI,CAACN,MAAM;QACnD9B,QAAQC,KAAK,CAAC,CAAC,oEAAoE,EAAE+B,MAAM,mDAAmD,CAAC;QAC/IhC,QAAQC,KAAK,CAAC,CAAC,gDAAgD,CAAC;QAChEE,QAAQC,IAAI,CAAC;IACf;IACA,MAAM,EAAEkB,EAAE,EAAET,QAAQ,EAAE,GAAG7C,KAAKoD;IAC9BR,cAAcC;IACd,IAAIlC,WAAW6B,WAAWgB,UAAUF,IAAIF,KAAKzC;IAC7C,IAAI0D;IACJ,IAAI;QACFA,OAAOf,GAAGK,OAAO,CAACG,KAAKQ,GAAG,IAAIP;IAChC,EAAE,OAAOhC,KAAK;QACZuB,GAAGC,KAAK;QACR,gFAAgF;QAChF,mFAAmF;QACnF,sFAAsF;QACtF,IAAI,aAAaa,IAAI,CAACN,MAAM,MAAM5D,YAAY6B,KAAcgC,OAAOQ,IAAI,CAAC;QACxE,MAAMxC;IACR;IACA9B,UAAUoE,MAAMjE;IAChBkD,GAAGC,KAAK;AACV"}

package/dist/esm/cli/sql.js

@@ -4,11 +4,14 @@ const sqlCmd = (ctx)=>{
     const usage = `usage: ${ctx.name} ${USAGE.sql}`;
     const { values, positionals } = parse(ctx.argv, usage, {
         ...FORMAT,
-        ...CONFIG
+        ...CONFIG,
+        preset: {
+            type: 'string'
+        }
     });
     const [sql, ...params] = positionals;
     if (!sql) ctx.usageError(usage);
     const format = formatOf(values);
-    runSql(ctx.resolveConfig(values.config), sql, params, format, 'ad-hoc statement');
+    runSql(ctx.resolveConfig(values.config), sql, params, format, 'ad-hoc statement', values.preset);
 };
 export default sqlCmd;

package/dist/esm/cli/sql.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/sql.ts"],"sourcesContent":["import { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, formatOf, parse, runSql } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst sqlCmd: Command = (ctx) => {\n  const usage = `usage: ${ctx.name} ${USAGE.sql}`;\n  const { values, positionals } = parse(ctx.argv, usage, { ...FORMAT, ...CONFIG });\n  const [sql, ...params] = positionals;\n  if (!sql) ctx.usageError(usage);\n  const format = formatOf(values);\n  runSql(ctx.resolveConfig(values.config as string | undefined), sql, params, format, 'ad-hoc statement');\n};\nexport default sqlCmd;\n"],"names":["USAGE","CONFIG","FORMAT","formatOf","parse","runSql","sqlCmd","ctx","usage","name","sql","values","positionals","argv","params","usageError","format","resolveConfig","config"],"mappings":"AAAA,SAASA,KAAK,QAAQ,aAAa;AACnC,SAASC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAEC,KAAK,EAAEC,MAAM,QAAQ,cAAc;AAGtE,MAAMC,SAAkB,CAACC;IACvB,MAAMC,QAAQ,CAAC,OAAO,EAAED,IAAIE,IAAI,CAAC,CAAC,EAAET,MAAMU,GAAG,EAAE;IAC/C,MAAM,EAAEC,MAAM,EAAEC,WAAW,EAAE,GAAGR,MAAMG,IAAIM,IAAI,EAAEL,OAAO;QAAE,GAAGN,MAAM;QAAE,GAAGD,MAAM;IAAC;IAC9E,MAAM,CAACS,KAAK,GAAGI,OAAO,GAAGF;IACzB,IAAI,CAACF,KAAKH,IAAIQ,UAAU,CAACP;IACzB,MAAMQ,SAASb,SAASQ;IACxBN,OAAOE,IAAIU,aAAa,CAACN,OAAOO,MAAM,GAAyBR,KAAKI,QAAQE,QAAQ;AACtF;AACA,eAAeV,OAAO"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/sql.ts"],"sourcesContent":["import { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, formatOf, parse, runSql } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst sqlCmd: Command = (ctx) => {\n  const usage = `usage: ${ctx.name} ${USAGE.sql}`;\n  const { values, positionals } = parse(ctx.argv, usage, { ...FORMAT, ...CONFIG, preset: { type: 'string' } });\n  const [sql, ...params] = positionals;\n  if (!sql) ctx.usageError(usage);\n  const format = formatOf(values);\n  runSql(ctx.resolveConfig(values.config as string | undefined), sql, params, format, 'ad-hoc statement', values.preset as string | undefined);\n};\nexport default sqlCmd;\n"],"names":["USAGE","CONFIG","FORMAT","formatOf","parse","runSql","sqlCmd","ctx","usage","name","sql","values","positionals","argv","preset","type","params","usageError","format","resolveConfig","config"],"mappings":"AAAA,SAASA,KAAK,QAAQ,aAAa;AACnC,SAASC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAEC,KAAK,EAAEC,MAAM,QAAQ,cAAc;AAGtE,MAAMC,SAAkB,CAACC;IACvB,MAAMC,QAAQ,CAAC,OAAO,EAAED,IAAIE,IAAI,CAAC,CAAC,EAAET,MAAMU,GAAG,EAAE;IAC/C,MAAM,EAAEC,MAAM,EAAEC,WAAW,EAAE,GAAGR,MAAMG,IAAIM,IAAI,EAAEL,OAAO;QAAE,GAAGN,MAAM;QAAE,GAAGD,MAAM;QAAEa,QAAQ;YAAEC,MAAM;QAAS;IAAE;IAC1G,MAAM,CAACL,KAAK,GAAGM,OAAO,GAAGJ;IACzB,IAAI,CAACF,KAAKH,IAAIU,UAAU,CAACT;IACzB,MAAMU,SAASf,SAASQ;IACxBN,OAAOE,IAAIY,aAAa,CAACR,OAAOS,MAAM,GAAyBV,KAAKM,QAAQE,QAAQ,oBAAoBP,OAAOG,MAAM;AACvH;AACA,eAAeR,OAAO"}

package/dist/esm/cli/status.d.ts

@@ -1,3 +1,4 @@
 import type { Command } from './types.js';
+export declare function redactUrl(url: string): string;
 declare const status: Command;
 export default status;

package/dist/esm/cli/status.js

@@ -1,12 +1,17 @@
+import { dirname } from 'node:path';
 import { presetCoverage } from '../commands.js';
-import { embedConfig, featureStates } from '../config.js';
-import { docCount, getMeta, open } from '../db.js';
-import { isDownloadable, MODEL_FILENAMES, modelPresent } from '../features/embed.js';
+import { embedConfig, featureStates, SUPPORTED_CONFIG_VERSION } from '../config.js';
+import { docCount, getMeta, open, SCHEMA_VERSION } from '../db.js';
+import { isDownloadable, MODEL_FILENAMES, modelDir, modelPresent } from '../features/embed.js';
 import { featuresLine, presetsLines } from '../output.js';
 import { USAGE } from './index.js';
 import { CONFIG, FORMAT, formatOf, parse, printWarnings } from './shared.js';
+// status names what you cannot read off the config file: locations sense chose, values it
+// derived, state it stored. Config content is not echoed -- the config line says which file to
+// read, and duplicating its globs here would create two places that disagree after an edit.
+// The exception is the embed block's url and key, whose effect is remote and otherwise invisible.
 // Vectors are computed on the first semantic query, not at reconcile, so a tree can have a
-// semantic-on preset with nothing embedded yet -- reported here because a fast `rebuild` on
+// semantic-on preset with nothing embedded yet -- reported here because a fast reconcile on
 // such a tree otherwise reads as "the feature isn't on".
 function vectorState(db) {
     try {
@@ -16,7 +21,25 @@ function vectorState(db) {
         return null;
     }
 }
+function parseErrorCount(db) {
+    const row = db.prepare(`SELECT COUNT(*) AS n FROM frontmatter WHERE "_parse_error" IS NOT NULL`).get();
+    return row.n;
+}
+// A URL can carry credentials in its userinfo (https://user:pass@host/v1). status output gets
+// pasted into issues and transcripts, so it is always safe to paste.
+export function redactUrl(url) {
+    try {
+        const parsed = new URL(url);
+        if (!parsed.username && !parsed.password) return url;
+        parsed.username = '';
+        parsed.password = '';
+        return `${parsed.toString().replace(/\/$/, '')} (credentials redacted)`;
+    } catch  {
+        return url;
+    }
+}
 const status = (ctx)=>{
+    var _process_env_e_key, _cfg_migratedFrom, _cfg_unknownKeys, _cfg_queries;
     const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.status}`, {
         ...FORMAT,
         ...CONFIG
@@ -35,35 +58,69 @@ const status = (ctx)=>{
     // transaction for, floored at 30s -- read back from the connection rather than
     // recomputed here, so this always reports what open() actually set.
     const busyTimeoutMs = db.prepare('PRAGMA busy_timeout').get().timeout;
+    // The env var holds the token; only its name and whether it is set are ever reported.
+    const keyEnv = (e === null || e === void 0 ? void 0 : e.key) ? {
+        name: e.key,
+        set: ((_process_env_e_key = process.env[e.key]) !== null && _process_env_e_key !== void 0 ? _process_env_e_key : '') !== ''
+    } : null;
     const result = {
+        config: cfg.configPath,
+        configVersion: SUPPORTED_CONFIG_VERSION,
+        migratedFrom: (_cfg_migratedFrom = cfg.migratedFrom) !== null && _cfg_migratedFrom !== void 0 ? _cfg_migratedFrom : null,
+        unknownConfigKeys: (_cfg_unknownKeys = cfg.unknownKeys) !== null && _cfg_unknownKeys !== void 0 ? _cfg_unknownKeys : null,
+        tree: cfg.baseDir,
+        cache: dirname(dbPath),
         db: dbPath,
+        cacheSchema: SCHEMA_VERSION,
         docs: docCount(db),
+        unparseableFrontmatter: parseErrorCount(db),
         features: features.on,
         featuresOff: features.off,
         embed: e ? {
             type: e.type,
             model: e.model,
+            dir: e.type === 'static' ? modelDir(e.model) : null,
+            url: e.url ? redactUrl(e.url) : null,
+            keyEnv,
             downloaded: hasModel,
             ...vectors !== null && vectors !== void 0 ? vectors : {}
         } : null,
         presets,
+        queries: Object.keys((_cfg_queries = cfg.queries) !== null && _cfg_queries !== void 0 ? _cfg_queries : {}).length,
+        watcherPid: getMeta(db, 'watch_pid'),
         watcherHeartbeatSecondsAgo: heartbeat ? Math.round((Date.now() - Date.parse(heartbeat)) / 1000) : null,
         busyTimeoutMs
     };
     if (format === 'json') {
         console.log(JSON.stringify(result, null, 2));
     } else {
-        console.log(`db: ${result.db}`);
-        console.log(`docs: ${result.docs}`);
+        const migrated = cfg.migratedFrom !== undefined ? `, migrated from v${cfg.migratedFrom} on this run` : '';
+        console.log(`config:   ${result.config} (v${SUPPORTED_CONFIG_VERSION}${migrated})`);
+        // A load-time stderr warning has scrolled away by the time anyone looks here.
+        if (cfg.unknownKeys) console.log(`          unknown keys, ignored: ${cfg.unknownKeys.join(', ')}`);
+        console.log(`tree:     ${result.tree}`);
+        console.log(`cache:    ${result.cache} (schema ${SCHEMA_VERSION}; delete this directory to reset)`);
+        console.log(`docs:     ${result.docs}${result.unparseableFrontmatter > 0 ? `  (${result.unparseableFrontmatter} with unparseable frontmatter: WHERE _parse_error IS NOT NULL)` : ''}`);
+        console.log('');
         console.log(featuresLine(features));
-        // One authoritative line, off states included: an absent model reads as a degraded
-        // search, not as an empty corpus.
-        if (!e) console.log('embed: off (no preset asks for vectors)');
-        else if (!hasModel) console.log(`embed: ${e.type} ${e.model} — unavailable, so search runs on words and links only (${isDownloadable(e.model) ? `run \`${ctx.name} download\`` : `no ${MODEL_FILENAMES} in that directory`})`);
-        else console.log(`embed: ${e.type} ${e.model}${vectors ? ` (vectors: ${vectors.embedded} embedded, ${vectors.pending} pending — embedded on the first semantic search)` : ''}`);
+        // One authoritative block, off states included: an absent model reads as a degraded search,
+        // not as an empty corpus.
+        if (!e) console.log('embed:    off (no preset asks for vectors)');
+        else {
+            console.log(`embed:    ${e.type} ${e.model}`);
+            if (e.type === 'static') console.log(`          model:   ${modelDir(e.model)} (${hasModel ? 'present' : `missing, ${isDownloadable(e.model) ? `run \`${ctx.name} download\`` : `no ${MODEL_FILENAMES} in that directory`}`})`);
+            if (e.url) console.log(`          url:     ${redactUrl(e.url)}`);
+            if (keyEnv) console.log(`          key:     env ${keyEnv.name} (${keyEnv.set ? 'set' : 'NOT SET'})`);
+            if (!hasModel) console.log('          vectors: off until the model is present; search runs on words and links');
+            else if (vectors) console.log(`          vectors: ${vectors.embedded} embedded, ${vectors.pending} pending (embedded on the first semantic search)`);
+        }
+        console.log('');
         for (const line of presetsLines(presets))console.log(line);
-        console.log(result.watcherHeartbeatSecondsAgo === null ? 'watcher: no watcher' : `watcher: last heartbeat ${result.watcherHeartbeatSecondsAgo}s ago`);
-        console.log(`busy_timeout: ${result.busyTimeoutMs}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)`);
+        console.log('');
+        console.log(`queries:  ${result.queries} saved (${ctx.name} --list)`);
+        const pid = result.watcherPid ? `pid ${result.watcherPid}, ` : '';
+        console.log(result.watcherHeartbeatSecondsAgo === null ? 'watcher:  none' : `watcher:  ${pid}last heartbeat ${result.watcherHeartbeatSecondsAgo}s ago`);
+        console.log(`sqlite:   busy_timeout ${result.busyTimeoutMs}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)`);
     }
     db.close();
 };