sensemaking 0.16.0 → 0.17.0

package/dist/esm/{scan.js → scan/frontmatter.js}

@@ -1,9 +1,4 @@
-import { globSync, readFileSync, statSync } from 'node:fs';
-import { join, sep } from 'node:path';
-import removeMarkdown from 'remove-markdown';
 import { isCollection, parseDocument, visit } from 'yaml';
-import { embedEnabled, presetHasSignal, presetNames } from './config/index.js';
-// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
 // Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell
 // a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.
 export const RESERVED_COLUMNS = new Set([
@@ -26,76 +21,10 @@ export const RESERVED_COLUMNS = new Set([
 const ACCEPTED_YAML_CODES = new Set([
     'BAD_SCALAR_START'
 ]);
-function normalizeText(value) {
+export function normalizeText(value) {
     if (value === null || value === undefined) return '';
     return String(value).replace(/\s+/g, ' ').trim();
 }
-// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms
-// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.
-function stripText(value) {
-    const withoutWikilinks = value.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2').replace(/\[\[([^\]]+)\]\]/g, '$1');
-    // remove-markdown deletes footnote definitions wholesale ("[^1]: text" -> ""), so unwrap
-    // them (and drop inline refs) before it runs; the definition text stays indexed.
-    const withoutFootnotes = withoutWikilinks.replace(/^\[\^[^\]]+\]:[ \t]?/gm, '').replace(/\[\^[^\]]+\]/g, '');
-    const withoutMarkdown = removeMarkdown(withoutFootnotes);
-    const withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
-    return normalizeText(withoutTables);
-}
-// Presets are views, not partitions: they overlap freely, and a file's covering set (not one
-// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.
-export function toPosixPath(relPath, separator = sep) {
-    return separator === '\\' ? relPath.split(separator).join('/') : relPath;
-}
-// Every command pays listFiles before it answers (the freshness check stats each file), so
-// per-file work here is the hottest path in the package. Everything derivable from the config
-// alone is computed once, above the loop.
-const NO_THROW = {
-    throwIfNoEntry: false
-};
-export function listFiles(cfg, baseDir) {
-    const coverage = new Map();
-    const posixNeeded = sep === '\\';
-    for (const name of presetNames(cfg)){
-        const preset = cfg.presets[name];
-        for (const matched of globSync(preset.include, {
-            cwd: baseDir,
-            exclude: preset.exclude
-        })){
-            var _coverage_get;
-            const relPath = posixNeeded ? toPosixPath(matched) : matched;
-            const set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
-            set.add(name);
-            coverage.set(relPath, set);
-        }
-    }
-    // Which presets want vectors is a property of the config, not of any file.
-    const embedding = embedEnabled(cfg);
-    const vectorPresets = embedding ? new Set(presetNames(cfg).filter((name)=>presetHasSignal(cfg, name, 'vectors'))) : null;
-    const files = [];
-    for (const relPath of [
-        ...coverage.keys()
-    ].sort()){
-        const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls
-        // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so
-        // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).
-        const st = statSync(absPath, NO_THROW);
-        if (!(st === null || st === void 0 ? void 0 : st.isFile())) continue;
-        const presets = [
-            ...coverage.get(relPath)
-        ].sort();
-        const embed = vectorPresets !== null && presets.some((name)=>vectorPresets.has(name));
-        files.push({
-            relPath,
-            absPath,
-            mtimeMs: st.mtimeMs,
-            ctimeMs: st.birthtimeMs,
-            size: st.size,
-            presets,
-            embed
-        });
-    }
-    return files;
-}
 // SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601
 // allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.
 const ISO_DATETIME = /^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?)(Z|[+-]\d{2}(?::?\d{2})?)?$/;
@@ -118,7 +47,7 @@ export function normalizeDate(value) {
 }
 // Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches
 // and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.
-function mapValue(value) {
+export function mapValue(value) {
     if (value === null || value === undefined) return null;
     if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
     if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;
@@ -126,7 +55,7 @@ function mapValue(value) {
     return JSON.stringify(value);
 }
 // The delimiter split is all this package used gray-matter for.
-function splitFrontmatter(raw) {
+export function splitFrontmatter(raw) {
     const open = raw.match(/^---\r?\n/);
     if (!open) return {
         fm: null,
@@ -172,7 +101,7 @@ function warnStringifiedKeys(relPath, doc, warnings) {
 function firstLine(message) {
     return message.split('\n')[0].replace(/:\s*$/, '');
 }
-function parseFrontmatter(relPath, fm, warnings) {
+export function parseFrontmatter(relPath, fm, warnings) {
     // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one
     // that carries information, with the file it came from.
     const doc = parseDocument(fm, {
@@ -219,51 +148,3 @@ function parseFrontmatter(relPath, fm, warnings) {
         parseError: null
     };
 }
-export function parseFile(file, extractors = []) {
-    const raw = readFileSync(file.absPath, 'utf8');
-    const warnings = [];
-    const { fm, body: content } = splitFrontmatter(raw);
-    const { data, parseError } = fm === null ? {
-        data: {},
-        parseError: null
-    } : parseFrontmatter(file.relPath, fm, warnings);
-    const mapped = {};
-    for (const key of Object.keys(data)){
-        if (RESERVED_COLUMNS.has(key)) {
-            warnings.push(`warning: ${file.relPath} has a frontmatter key named "${key}", which is reserved; ignoring it`);
-            continue;
-        }
-        const value = mapValue(data[key]);
-        if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {
-            warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);
-        }
-        mapped[key] = value;
-    }
-    // title/summary are plain YAML strings -- whitespace-collapse only;
-    // the prose gets the full markdown strip.
-    const search = {
-        title: normalizeText(data.title),
-        summary: normalizeText(data.summary),
-        text: stripText(content)
-    };
-    return {
-        doc: {
-            relPath: file.relPath,
-            mtimeMs: file.mtimeMs,
-            ctimeMs: file.ctimeMs,
-            size: file.size,
-            presets: file.presets,
-            data: mapped,
-            parseError,
-            search,
-            extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
-                var _f_extract;
-                return [
-                    f.name,
-                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search, data)
-                ];
-            }))
-        },
-        warnings
-    };
-}

package/dist/esm/scan/frontmatter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4DAA4D;AAC5D,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gEAAgE;AAChE,OAAO,SAAS6B,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,MAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKpC,WAAW,OAAO;QAAEgC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC/C,MAAM6C,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACpD,aAAaoD,KAAKC,GAAG,GAAG,OAAO7C;YACpCyC,QAAQ;YACR,OAAO/C,MAAMoD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAChD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASiD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAM9C,cAAcuC,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAC3D,oBAAoB4D,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQrC,MAAM,GAAG,GAAG;QACtB,MAAM2C,SAASN,QAAQrC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEqC,QAAQrC,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAM4C,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS7D,WAAW,OAAO;QAAE6D,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}

package/dist/esm/scan/index.d.ts

@@ -0,0 +1,25 @@
+import type { Config } from '../config/index.js';
+import type { Feature } from '../features/types.js';
+import type { FileStat } from './list.js';
+export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
+export type { FileStat } from './list.js';
+export { listFiles, toPosixPath } from './list.js';
+export interface ParsedDoc {
+    relPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    data: Record<string, string | number | bigint | null>;
+    parseError: string | null;
+    search: {
+        title: string;
+        summary: string;
+        text: string;
+    };
+    extracted: Record<string, unknown>;
+}
+export declare function parseFile(file: FileStat, extractors?: Feature[], cfg?: Config): {
+    doc: ParsedDoc;
+    warnings: string[];
+};

package/dist/esm/scan/index.js

@@ -0,0 +1,58 @@
+import { readFileSync } from 'node:fs';
+import { parse } from '../chunk/index.js';
+import { textFromBlocks } from '../text/strip.js';
+import { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.js';
+// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
+export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
+export { listFiles, toPosixPath } from './list.js';
+export function parseFile(file, extractors = [], cfg) {
+    const raw = readFileSync(file.absPath, 'utf8');
+    const warnings = [];
+    const { fm, body: content } = splitFrontmatter(raw);
+    const { data, parseError } = fm === null ? {
+        data: {},
+        parseError: null
+    } : parseFrontmatter(file.relPath, fm, warnings);
+    const mapped = {};
+    for (const key of Object.keys(data)){
+        if (RESERVED_COLUMNS.has(key)) {
+            warnings.push(`warning: ${file.relPath} has a frontmatter key named "${key}", which is reserved; ignoring it`);
+            continue;
+        }
+        const value = mapValue(data[key]);
+        if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {
+            warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);
+        }
+        mapped[key] = value;
+    }
+    // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature
+    // that needs a parse tree (embed's chunker) both read this same block list.
+    const blocks = parse(content);
+    // title/summary are plain YAML strings -- whitespace-collapse only;
+    // the prose gets the full markdown strip.
+    const search = {
+        title: normalizeText(data.title),
+        summary: normalizeText(data.summary),
+        text: textFromBlocks(blocks)
+    };
+    return {
+        doc: {
+            relPath: file.relPath,
+            mtimeMs: file.mtimeMs,
+            ctimeMs: file.ctimeMs,
+            size: file.size,
+            presets: file.presets,
+            data: mapped,
+            parseError,
+            search,
+            extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
+                var _f_extract;
+                return [
+                    f.name,
+                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search, data, cfg, blocks)
+                ];
+            }))
+        },
+        warnings
+    };
+}

package/dist/esm/scan/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,6EAA6E;AAE7E,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}

package/dist/esm/scan/list.d.ts

@@ -0,0 +1,12 @@
+import type { Config } from '../config/index.js';
+export interface FileStat {
+    relPath: string;
+    absPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    embed: boolean;
+}
+export declare function toPosixPath(relPath: string, separator?: string): string;
+export declare function listFiles(cfg: Config, baseDir: string): FileStat[];

package/dist/esm/scan/list.js

@@ -0,0 +1,58 @@
+import { globSync, statSync } from 'node:fs';
+import { join, sep } from 'node:path';
+import { embedEnabled, presetHasSignal, presetNames } from '../config/index.js';
+// Presets are views, not partitions: they overlap freely, and a file's covering set (not one
+// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.
+export function toPosixPath(relPath, separator = sep) {
+    return separator === '\\' ? relPath.split(separator).join('/') : relPath;
+}
+// Every command pays listFiles before it answers (the freshness check stats each file), so
+// per-file work here is the hottest path in the package. Everything derivable from the config
+// alone is computed once, above the loop.
+const NO_THROW = {
+    throwIfNoEntry: false
+};
+export function listFiles(cfg, baseDir) {
+    const coverage = new Map();
+    const posixNeeded = sep === '\\';
+    for (const name of presetNames(cfg)){
+        const preset = cfg.presets[name];
+        for (const matched of globSync(preset.include, {
+            cwd: baseDir,
+            exclude: preset.exclude
+        })){
+            var _coverage_get;
+            const relPath = posixNeeded ? toPosixPath(matched) : matched;
+            const set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
+            set.add(name);
+            coverage.set(relPath, set);
+        }
+    }
+    // Which presets want vectors is a property of the config, not of any file.
+    const embedding = embedEnabled(cfg);
+    const vectorPresets = embedding ? new Set(presetNames(cfg).filter((name)=>presetHasSignal(cfg, name, 'vectors'))) : null;
+    const files = [];
+    for (const relPath of [
+        ...coverage.keys()
+    ].sort()){
+        const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls
+        // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so
+        // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).
+        const st = statSync(absPath, NO_THROW);
+        if (!(st === null || st === void 0 ? void 0 : st.isFile())) continue;
+        const presets = [
+            ...coverage.get(relPath)
+        ].sort();
+        const embed = vectorPresets !== null && presets.some((name)=>vectorPresets.has(name));
+        files.push({
+            relPath,
+            absPath,
+            mtimeMs: st.mtimeMs,
+            ctimeMs: st.birthtimeMs,
+            size: st.size,
+            presets,
+            embed
+        });
+    }
+    return files;
+}

package/dist/esm/scan/list.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/list.ts"],"sourcesContent":["import { globSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedEnabled, presetHasSignal, presetNames } from '../config/index.ts';\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  ctimeMs: number; // filesystem birthtime; a clone or copy resets it, like _mtime\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset's signals include vectors\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const vectorPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetHasSignal(cfg, name, 'vectors'))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = vectorPresets !== null && presets.some((name) => vectorPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, ctimeMs: st.birthtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n"],"names":["globSync","statSync","join","sep","embedEnabled","presetHasSignal","presetNames","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","Set","add","embedding","vectorPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","ctimeMs","birthtimeMs","size"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,QAAQ,QAAQ,UAAU;AAC7C,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AAEtC,SAASC,YAAY,EAAEC,eAAe,EAAEC,WAAW,QAAQ,qBAAqB;AAYhF,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBN,GAAG;IAClE,OAAOM,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWP,IAAI,CAAC,OAAOM;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAcf,QAAQ;IAC5B,KAAK,MAAMgB,QAAQb,YAAYQ,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWtB,SAASoB,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIY;YACzCF,IAAIG,GAAG,CAACV;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMI,YAAY1B,aAAaU;IAC/B,MAAMiB,gBAAgBD,YAAY,IAAIF,IAAItB,YAAYQ,KAAKkB,MAAM,CAAC,CAACb,OAASd,gBAAgBS,KAAKK,MAAM,eAAe;IAEtH,MAAMc,QAAoB,EAAE;IAC5B,KAAK,MAAMzB,WAAW;WAAIQ,SAASkB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlC,KAAKa,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM6B,KAAKpC,SAASmC,SAASzB;QAC7B,IAAI,EAAC0B,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMjB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC2B,IAAI;QAChE,MAAMI,QAAQR,kBAAkB,QAAQV,QAAQmB,IAAI,CAAC,CAACrB,OAASY,cAAcU,GAAG,CAACtB;QACjFc,MAAMS,IAAI,CAAC;YAAElC;YAAS4B;YAASO,SAASN,GAAGM,OAAO;YAAEC,SAASP,GAAGQ,WAAW;YAAEC,MAAMT,GAAGS,IAAI;YAAEzB;YAASkB;QAAM;IAC7G;IACA,OAAON;AACT"}

package/dist/esm/text/segment.d.ts

@@ -0,0 +1,3 @@
+export declare const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}";
+export declare function segmentField(text: string): string;
+export declare function segmentMatch(terms: string): string;

package/dist/esm/{segment.js → text/segment.js}

@@ -9,11 +9,11 @@
 // characters -- so grapheme mode is used below; no dictionary, no word mode.
 // Scripts written without word spaces: a closed set of writing systems.
 // Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.
-const SCRIPTS = '\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}';
+export const UNSPACED_SCRIPTS = '\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}';
 // A run is script BASE characters with their combining marks attached; a bare mark after a
 // Latin letter (decomposed é) never starts one.
-const RUN = new RegExp(`((?:[${SCRIPTS}]\\p{M}*)+)`, 'gu');
-const HAS_RUN = new RegExp(`[${SCRIPTS}]`, 'u');
+const RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\p{M}*)+)`, 'gu');
+const HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');
 // Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.
 // Built once (construction cost amortizes) and used for both index and query splitting.
 const GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, {

package/dist/esm/text/segment.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own.\n//\n// Segmentation is context-free: seg(query) appears inside seg(document) whenever the query is\n// a substring of the document. That is the contract -- substring semantics, what `grep` and\n// `LIKE '%..%'` give -- and it is what makes index and query agree on every input, always.\n// Intl.Segmenter's word mode is context-dependent (`东京都政府` splits as `东 | 京都 | 政府`\n// while the query `东京` splits as one word) and is rejected for that reason. Its grapheme\n// mode has no such dependency -- UAX #29 cluster boundaries never look past adjacent\n// characters -- so grapheme mode is used below; no dictionary, no word mode.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built once (construction cost amortizes) and used for both index and query splitting.\nconst GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  return Array.from(GRAPHEME_SEGMENTER.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run (the common case, paid for by\n// nothing). Otherwise each run explodes into its graphemes, barrier-delimited from its\n// neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// An unqualified run's target: raw title/summary/text drop punctuation as unicode61's token\n// separator, so two adjacent single-grapheme phrase positions match across punctuation the same\n// as across a real gap (`数数` vs `数。数`) -- a false positive only the barriered `_seg` columns\n// are safe from. FTS5's column-set filter, not parens+OR, so the group still composes under\n// AND/OR/NOT/juxtaposition exactly like the single-column qualifier form below.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching how segmentField\n// indexed it. A qualifier ahead of such a run maps to its `_seg` column; unqualified maps to all\n// three (SIDECAR_COLUMNS), since raw columns cannot express the contract. An author's own quoted\n// phrase is their explicit escape hatch to FTS5's native syntax, and passes through byte-identical.\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","RUN","RegExp","HAS_RUN","GRAPHEME_SEGMENTER","Intl","Segmenter","undefined","granularity","PUNCTUATION","BARRIER","graphemes","run","Array","from","segment","s","splitOnPunctuation","groups","g","test","push","length","filter","segmentField","text","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","segmentMatch","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,8DAA8D;AAC9D,EAAE;AACF,8FAA8F;AAC9F,4FAA4F;AAC5F,2FAA2F;AAC3F,mFAAmF;AACnF,yFAAyF;AACzF,qFAAqF;AACrF,6EAA6E;AAE7E,wEAAwE;AACxE,kFAAkF;AAClF,OAAO,MAAMA,mBAAmB,wGAAwG;AACxI,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,MAAM,IAAIC,OAAO,CAAC,KAAK,EAAEF,iBAAiB,WAAW,CAAC,EAAE;AAC9D,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEF,iBAAiB,CAAC,CAAC,EAAE;AACpD,6FAA6F;AAC7F,wFAAwF;AACxF,MAAMI,qBAAqB,IAAIC,KAAKC,SAAS,CAACC,WAAW;IAAEC,aAAa;AAAW;AACnF,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,MAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5B,OAAOC,MAAMC,IAAI,CAACV,mBAAmBW,OAAO,CAACH,MAAM,CAACI,IAAMA,EAAED,OAAO;AACrE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBL,GAAW;IACrC,MAAMM,SAAqB;QAAC,EAAE;KAAC;IAC/B,KAAK,MAAMC,KAAKR,UAAUC,KAAM;QAC9B,IAAIH,YAAYW,IAAI,CAACD,IAAID,OAAOG,IAAI,CAAC,EAAE;aAClCH,MAAM,CAACA,OAAOI,MAAM,GAAG,EAAE,CAACD,IAAI,CAACF;IACtC;IACA,OAAOD,OAAOK,MAAM,CAAC,CAACJ,IAAMA,EAAEG,MAAM,GAAG;AACzC;AAEA,yFAAyF;AACzF,uFAAuF;AACvF,wDAAwD;AACxD,OAAO,SAASE,aAAaC,IAAY;IACvC,IAAI,CAACtB,QAAQiB,IAAI,CAACK,OAAO,OAAO;IAChC,MAAMC,MAAMD,KAAKE,OAAO,CAAC1B,KAAK,CAACW;QAC7B,MAAMgB,OAAOX,mBAAmBL,KAC7BiB,GAAG,CAAC,CAACV,IAAMA,EAAEW,IAAI,CAAC,MAClBA,IAAI,CAAC,CAAC,CAAC,EAAEpB,QAAQ,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,EAAEA,QAAQ,CAAC,EAAEkB,KAAK,CAAC,EAAElB,QAAQ,CAAC,CAAC;IAC1C;IACA,OAAOgB,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,gGAAgG;AAChG,6FAA6F;AAC7F,4FAA4F;AAC5F,gFAAgF;AAChF,MAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAAStB,GAAW,EAAEuB,YAAoB;IACjD,OAAOlB,mBAAmBL,KACvBiB,GAAG,CAAC,CAACV,IAAM,GAAGgB,eAAehB,EAAEG,MAAM,GAAG,IAAI,CAAC,CAAC,EAAEH,EAAEW,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGX,CAAC,CAAC,EAAE,EAAE,EACvEW,IAAI,CAAC;AACV;AAEA,4FAA4F;AAC5F,iGAAiG;AACjG,iGAAiG;AACjG,oGAAoG;AACpG,OAAO,SAASM,aAAaC,KAAa;IACxC,IAAI,CAAClC,QAAQiB,IAAI,CAACiB,QAAQ,OAAOA;IACjC,IAAIX,MAAM;IACV,IAAIY,SAAS;IACb,MAAMC,SAASF,MAAMG,KAAK,CAACvC,MAAM,2CAA2C;IAC5E,IAAK,IAAIwC,IAAI,GAAGA,IAAIF,OAAOjB,MAAM,EAAEmB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAMC,MAAMH,MAAM,CAACE,EAAE,CAAE,IAAIC,OAAO,KAAKJ,SAAS,CAACA;YACtDZ,OAAOa,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVZ,OAAOa,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,MAAME,IAAIjB,IAAIkB,KAAK,CAACZ;QACpB,IAAIW,GAAG;YACLjB,MAAM,GAAGA,IAAImB,KAAK,CAAC,GAAGF,EAAEG,KAAK,IAAIH,CAAC,CAAC,EAAE,GAAGT,SAASK,MAAM,CAACE,EAAE,EAAE,GAAGE,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG;QACtF,OAAO;YACLjB,OAAOQ,SAASK,MAAM,CAACE,EAAE,EAAER;QAC7B;IACF;IACA,OAAOP;AACT"}

package/dist/esm/text/strip.d.ts

@@ -0,0 +1,3 @@
+import type { Block } from '../chunk/index.js';
+export declare function textFromBlocks(blocks: Block[]): string;
+export declare function stripText(value: string): string;

package/dist/esm/text/strip.js

@@ -0,0 +1,12 @@
+import { extractText, parse } from '../chunk/index.js';
+import { normalizeText } from '../scan/frontmatter.js';
+// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses
+// (src/chunk) -- words and vectors read one definition of "the prose of this note".
+export function textFromBlocks(blocks) {
+    const text = blocks.map((block)=>extractText(block.node)).filter((s)=>s.length > 0).join('\n');
+    return normalizeText(text);
+}
+// Thin parse + delegate, for callers with no blocks of their own already.
+export function stripText(value) {
+    return textFromBlocks(parse(value));
+}

package/dist/esm/text/strip.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractText, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  const text = blocks\n    .map((block) => extractText(block.node))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n  return normalizeText(text);\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractText","parse","normalizeText","textFromBlocks","blocks","text","map","block","node","filter","s","length","join","stripText","value"],"mappings":"AACA,SAASA,WAAW,EAAEC,KAAK,QAAQ,oBAAoB;AACvD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,4FAA4F;AAC5F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,MAAMC,OAAOD,OACVE,GAAG,CAAC,CAACC,QAAUP,YAAYO,MAAMC,IAAI,GACrCC,MAAM,CAAC,CAACC,IAAMA,EAAEC,MAAM,GAAG,GACzBC,IAAI,CAAC;IACR,OAAOV,cAAcG;AACvB;AAEA,0EAA0E;AAC1E,OAAO,SAASQ,UAAUC,KAAa;IACrC,OAAOX,eAAeF,MAAMa;AAC9B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -38,6 +38,8 @@
     "agent-memory",
     "memory-consolidation",
     "llm-wiki",
+    "second-brain",
+    "company-brain",
     "dreaming",
     "cli"
   ],
@@ -78,10 +80,21 @@
   "dependencies": {
     "@huggingface/tokenizers": "^0.1.3",
     "franc-min": "^6.2.0",
-    "remove-markdown": "^0.6.4",
+    "mdast-util-from-markdown": "^2.0.3",
+    "mdast-util-gfm-autolink-literal": "^2.0.0",
+    "mdast-util-gfm-footnote": "^2.0.0",
+    "mdast-util-gfm-strikethrough": "^2.0.0",
+    "mdast-util-gfm-table": "^2.0.0",
+    "mdast-util-gfm-task-list-item": "^2.0.0",
+    "micromark-extension-gfm-autolink-literal": "^2.0.0",
+    "micromark-extension-gfm-footnote": "^2.0.0",
+    "micromark-extension-gfm-strikethrough": "^2.0.0",
+    "micromark-extension-gfm-table": "^2.0.0",
+    "micromark-extension-gfm-task-list-item": "^2.0.0",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
+    "@types/mdast": "^4.0.4",
     "@types/mocha": "*",
     "@types/node": "*",
     "cr": "^0.1.0",

package/schema.json

@@ -91,7 +91,14 @@
         "model": { "type": "string" },
         "provider": { "type": "string", "enum": ["static", "openai", "cohere"] },
         "url": { "type": "string" },
-        "key": { "type": "string" }
+        "key": { "type": "string" },
+        "languages": {
+          "type": "array",
+          "items": { "type": "string" },
+          "minItems": 1,
+          "description": "Languages the model supports, overriding its own card declaration for the language-fit check."
+        },
+        "chunkTokens": { "type": "integer", "minimum": 1, "description": "Chunk size ceiling in estimated tokens for small-context models; default 500." }
       }
     },
     "content": {

package/skills/sense/SKILL.md

@@ -31,7 +31,7 @@ Output defaults to a table, built for humans; `--format json` returns the same r
 | `links` | connections authors wrote | what the tree's own structure treats as related: context the matching text never restates | notes nobody linked |
 | `vectors` | per-chunk meaning-vectors | notes sharing no words with the query: paraphrase, the category for an instance, a concept restated | proving presence -- a vector row cannot show the words occur anywhere; `hit` can |
 
-Each signal in the map carries a weight (`{"words": 1, "links": 1, "vectors": 1}` is the default when the key is omitted entirely); presence turns a signal on, and the number scales its share of the fused ranking. Equal weight (1 for everything) is what every number elsewhere in this doc describes. A weight above 1 pulls the fused ranking toward that signal's own ordering: measured on nfcorpus with the default static model and on MIRACL zh with an HTTP encoder, equal-weight fusion helps English nfcorpus but ranks the encoder's MIRACL-zh results below its own cosine-only ranking (BENCHMARKING.md, 2026-08-27 weight-sweep table) -- there is no single weight that is right for both, so a preset that leans on a strong encoder is a candidate for a higher `vectors` weight, checked against that table rather than assumed.
+Each signal in the map carries a weight (`{"words": 1, "links": 1, "vectors": 1}` is the default when the key is omitted entirely); presence turns a signal on, and the number scales its share of the fused ranking. Equal weight (1 for everything) is what every number elsewhere in this doc describes. A weight above 1 pulls the fused ranking toward that signal's own ordering: measured on nfcorpus with the default static model and on MIRACL zh with an HTTP encoder, equal-weight fusion helps English nfcorpus but ranks the encoder's MIRACL-zh results below its own cosine-only ranking (benchmark/reports/2026-08-27-embedding-model-selection.md, weight-sweep table) -- there is no single weight that is right for both, so a preset that leans on a strong encoder is a candidate for a higher `vectors` weight, checked against that table rather than assumed.
 
 Which signal carries a search is a property of the query, and the ends of the range are measured (BENCHMARKING.md, "Retrieval quality"): on the vocabulary-gap corpus, 31% of queries have no relevant lexical row in their top 10 and vector rows are the only recall; on a corpus whose queries quote their documents, words alone hit 99.7% and vectors add nothing. Real trees sit between. An exact identifier is words territory; "notes about X" where X could be phrased many ways is where vector rows carry; "how do these connect" is the link graph (`path`, `peek`).
 

package/skills/sense-setup/SKILL.md

@@ -30,9 +30,9 @@ A preset is a named, self-contained bundle of settings. `default` (required) is
 - Presets may overlap; they are views, not partitions.
 - Global `features` (`links`, `sections`, `rank`) still toggle tree-wide; most trees never touch them.
 
-**Vectors take two decisions, in two places.** The top-level `"embed": { "model", "provider": "static"|"openai"|"cohere", "url", "key" }` block names the model and says whether the tree has vectors at all. A preset's `signals` says which engines that scope uses: a layer searched for exact wording (ingested sources, archives, generated output) declares `"signals": {"words": 1, "links": 1}`, costs no embedding, and its searches run on words and links. That is the main scale lever, and it is the llm-wiki split: compiled pages searched by meaning, raw sources searched for the phrasing you are citing. Each named signal's number is its RRF weight, not a toggle -- 1 is the default and reproduces equal-weight fusion; a preset can instead raise one signal's number, e.g. `{"words": 1, "vectors": 4}`, to shift the fused ranking toward that signal without dropping the others. Whether that helps is corpus- and model-contingent, not a fixed rule: BENCHMARKING.md's 2026-08-27 weight-sweep table measures equal weight against {0.5, 1, 2, 4} and vectors-only on nfcorpus and on MIRACL zh with an HTTP encoder, and the two corpora do not agree on which weight wins. `static` is the built-in pure-JS Model2Vec loader and handles paraphrase and reworded concepts; tight domain jargon ("heart attack" for "myocardial infarction") is where an `openai`- or `cohere`-shaped encoder model tends to do better, measured in BENCHMARKING.md, "Retrieval quality". Naming the model in the config is the consent to fetch it: the first vector search downloads it once per machine into `~/.sense/models` (or `sense download` prefetches) (huggingface_hub's cache layout, one snapshot directory per resolved revision), so several models coexist and switching between them rebuilds the index rather than mixing vector spaces. A `model` holding a path instead of a Hugging Face id points at a local directory, which `sense download` reports as nothing to fetch. The first search after that embeds the tree (progress on stderr; minutes on tens of thousands of notes, seconds on small trees). A config edit that changes coverage or features rebuilds the cache, vectors included, so settle presets before the first vector search on a large tree or that embedding run is paid twice.
+**Vectors take two decisions, in two places.** The top-level `"embed": { "model", "provider": "static"|"openai"|"cohere", "url", "key", "chunkTokens" }` block names the model and says whether the tree has vectors at all. `chunkTokens` is a chunk size ceiling in estimated tokens for small-context models; default 500. A preset's `signals` says which engines that scope uses: a layer searched for exact wording (ingested sources, archives, generated output) declares `"signals": {"words": 1, "links": 1}`, costs no embedding, and its searches run on words and links. That is the main scale lever, and it is the llm-wiki split: compiled pages searched by meaning, raw sources searched for the phrasing you are citing. Each named signal's number is its RRF weight, not a toggle -- 1 is the default and reproduces equal-weight fusion; a preset can instead raise one signal's number, e.g. `{"words": 1, "vectors": 4}`, to shift the fused ranking toward that signal without dropping the others. Whether that helps is corpus- and model-contingent, not a fixed rule: benchmark/reports/2026-08-27-embedding-model-selection.md's weight-sweep table measures equal weight against {0.5, 1, 2, 4} and vectors-only on nfcorpus and on MIRACL zh with an HTTP encoder, and the two corpora do not agree on which weight wins. `static` is the built-in pure-JS Model2Vec loader and handles paraphrase and reworded concepts; tight domain jargon ("heart attack" for "myocardial infarction") is where an `openai`- or `cohere`-shaped encoder model tends to do better, measured in the same report's encoder-tier tables. Naming the model in the config is the consent to fetch it: the first vector search downloads it once per machine into `~/.sense/models` (or `sense download` prefetches) (huggingface_hub's cache layout, one snapshot directory per resolved revision), so several models coexist and switching between them rebuilds the index rather than mixing vector spaces. A `model` holding a path instead of a Hugging Face id points at a local directory, which `sense download` reports as nothing to fetch. The first search after that embeds the tree (progress on stderr; minutes on tens of thousands of notes, seconds on small trees). A config edit that changes coverage or features rebuilds the cache, vectors included, so settle presets before the first vector search on a large tree or that embedding run is paid twice.
 
-**Finding a model.** The default, `potion-retrieval-32M`, is English only. A tree whose text is mostly a language the model does not declare fails loudly at embed time (`EMBED_MODEL_MISMATCH`, naming the fix), and `sense status` shows the detected language mix beside the model's declared languages. A model whose card declares no languages leaves that check off, and a mismatched pairing then degrades silently -- nearest-neighbour search always returns a neighbour regardless of fit -- so check the card's languages yourself when the tree is not English. Picking a model is a lookup against the source, not a name to memorize: for a static model, filter Hugging Face's model2vec library for the target language and read the card for its declared languages, its safetensors shape, and F32 weights (an int8 republish fails the loader's dtype check by design). For an encoder reached over HTTP, Ollama's embedding-model library and LM Studio's catalog list each model's languages and context length; both serve an OpenAI-shaped endpoint, so `provider: "openai"` with `url` pointing at the local port reaches either with no other config. Cohere's hosted models are reached with `provider: "cohere"` instead. One dated measurement stands in for a name table that would go stale: `potion-retrieval-32M` was the best-measured static English retrieval model as of 2026-08, per BENCHMARKING.md's bake-off.
+**Finding a model.** The default, `potion-retrieval-32M`, is English only. A tree whose text is mostly a language the model does not declare fails loudly at embed time (`EMBED_MODEL_MISMATCH`, naming the fix), and `sense status` shows the detected language mix beside the model's declared languages. A model whose card declares no languages leaves that check off, and a mismatched pairing then degrades silently -- nearest-neighbour search always returns a neighbour regardless of fit -- so check the card's languages yourself when the tree is not English. Picking a model is a lookup against the source, not a name to memorize: for a static model, filter Hugging Face's model2vec library for the target language and read the card for its declared languages, its safetensors shape, and F32 weights (an int8 republish fails the loader's dtype check by design). For an encoder reached over HTTP, Ollama's embedding-model library and LM Studio's catalog list each model's languages and context length; both serve an OpenAI-shaped endpoint, so `provider: "openai"` with `url` pointing at the local port reaches either with no other config. Cohere's hosted models are reached with `provider: "cohere"` instead. One dated measurement stands in for a name table that would go stale: `potion-retrieval-32M` was the best-measured static English retrieval model as of 2026-08, per benchmark/reports/2026-08-27-embedding-model-selection.md's static-ladder table.
 
 **Large vaults**: everything except the vector build is measured linear to 100k notes with no tuning (BENCHMARKING.md). The knobs that matter are `k` (more, smaller results; rows carry `lines` section ranges, so agents read sections, not files) and `"signals": {"words": 1, "links": 1}` on the layers that do not earn vectors.
 
@@ -47,6 +47,6 @@ These belong to the tree's owner. sense works with any of them and reads no inst
 - **Language.** No decision needed. A language written without word spaces (Chinese, Japanese, Thai, Khmer, Lao, Burmese) is indexed per grapheme and searched as an ordered grapheme phrase, so `sense search "全文"` finds what it should. This is substring semantics, what `grep` gives: a query matches wherever its exact text occurs, including inside a longer run, and needs no minimum length. A language written with spaces is left exactly as it was, storing nothing extra, and its stemming is unaffected. The one place this does not reach is a hand-written `content MATCH '...'`, which cannot be rewritten for its author: pass the terms through `segment()` there. `content.tokenize` is a separate lever with a different purpose, substring matching inside a Latin word (`trigram`) or keeping hyphenated terms whole (`unicode61 tokenchars '-_'`); naming one turns the grapheme-phrase scheme off, since the tree has then chosen its own scheme. Changing it rebuilds the text index only; vectors, links, and sections are kept.
 - **Summaries.** A one-line `summary:` is optional and pays twice: it shows in every result row (often answering a question with no file read) and is a weighted search field ranked above body text. The cost is writing and maintaining the line as notes change.
 - **Folder shape.** Globs find the files, paths are queryable text, links resolve by basename at any depth, but presets make folders meaningful: a folder is the natural unit that gets its own coverage and settings.
-- **Note size.** Many small notes: precise search hits, whole-file reads stay cheap, more links to maintain. Fewer large notes: `sections`, `peek`, and the `lines` column carry the cost down to line-range reads. Both work.
+- **Note size.** Many small notes: precise search hits, whole-file reads stay cheap, more links to maintain. Fewer large notes: `sections`, `peek`, and the `lines` column carry the cost down to line-range reads. Both work. A long section no longer becomes one oversized vector either: chunking splits at headings and caps each chunk around 500 estimated tokens (CJK counted 1:1, spaced scripts ~4:1), so an oversized note degrades to more, smaller chunks rather than one truncated one; `embed.chunkTokens` lowers that cap further for a small-context model.
 - **Recurring questions.** Save a scenario an agent will repeat under `queries`, naming the verb it runs: `{ "sql": "..." }` for filters and reports, or `{ "search": "...", "preset": "raw", "k": 5 }` for a ranked search. Either runs as `sense <name>`, and running one is how it is validated: a typo'd column or an unknown preset errors and exits nonzero. A parameterised entry validates with any argument, since SQL is prepared before parameters bind.
 - **Where decisions live.** Choices that should outlive one conversation can be recorded in the agent's own instruction or skill files, or in a note in the tree itself; a one-off search over an existing corpus needs none of that.

package/dist/cjs/column-hint.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/column-hint.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\n\n// A frontmatter key with punctuation (`plugin-id`, `a.b`) is a real column, but naming it\n// unquoted in SQL parses as an expression: `plugin-id` is `plugin - id`, and SQLite's error\n// names a fragment (`plugin`) the user never wrote. This maps that fragment back to the\n// columns it could have come from, so the error names the actual fix instead.\n\nfunction startsWithBoundary(name: string, prefix: string): boolean {\n  return name.length > prefix.length && name.startsWith(prefix) && /\\W/.test(name[prefix.length]);\n}\n\nexport function columnHint(db: DatabaseSync, err: Error): Error {\n  const match = /no such column: (\\S+)/.exec(err.message);\n  if (!match) return err;\n  const token = match[1];\n  // `f.plugin` -> `plugin`; left as-is when there's no leading alias segment, which also\n  // covers a column literally named `a.b` -- the full-token check below matches that case\n  // directly, so stripping here never needs to special-case it.\n  const aliasStripped = token.replace(/^[A-Za-z_]\\w*\\./, '');\n  const forms = new Set([token, aliasStripped]);\n  const columns = (db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>).map((c) => c.name);\n  const candidates = new Set<string>();\n  for (const name of columns) {\n    for (const form of forms) {\n      if (name === form || startsWithBoundary(name, form)) candidates.add(name);\n    }\n  }\n  if (candidates.size === 0) return err;\n  const quoted = [...candidates].map((name) => `\"${name.split('\"').join('\"\"')}\"`).join(', ');\n  return new Error(`${err.message} -- ${quoted} needs double quotes in SQL: unquoted, SQLite parses the punctuation as an operator instead of column syntax.`);\n}\n"],"names":["columnHint","startsWithBoundary","name","prefix","length","startsWith","test","db","err","match","exec","message","token","aliasStripped","replace","forms","Set","columns","prepare","all","map","c","candidates","form","add","size","quoted","split","join","Error"],"mappings":";;;;+BAWgBA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAThB,0FAA0F;AAC1F,4FAA4F;AAC5F,wFAAwF;AACxF,8EAA8E;AAE9E,SAASC,mBAAmBC,IAAY,EAAEC,MAAc;IACtD,OAAOD,KAAKE,MAAM,GAAGD,OAAOC,MAAM,IAAIF,KAAKG,UAAU,CAACF,WAAW,KAAKG,IAAI,CAACJ,IAAI,CAACC,OAAOC,MAAM,CAAC;AAChG;AAEO,SAASJ,WAAWO,EAAgB,EAAEC,GAAU;IACrD,IAAMC,QAAQ,wBAAwBC,IAAI,CAACF,IAAIG,OAAO;IACtD,IAAI,CAACF,OAAO,OAAOD;IACnB,IAAMI,QAAQH,KAAK,CAAC,EAAE;IACtB,uFAAuF;IACvF,wFAAwF;IACxF,8DAA8D;IAC9D,IAAMI,gBAAgBD,MAAME,OAAO,CAAC,mBAAmB;IACvD,IAAMC,QAAQ,IAAIC,IAAI;QAACJ;QAAOC;KAAc;IAC5C,IAAMI,UAAU,AAACV,GAAGW,OAAO,CAAC,kCAAkCC,GAAG,GAA+BC,GAAG,CAAC,SAACC;eAAMA,EAAEnB,IAAI;;IACjH,IAAMoB,aAAa,IAAIN;QAClB,kCAAA,2BAAA;;QAAL,QAAK,YAAcC,4BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMf,OAAN;gBACE,mCAAA,4BAAA;;gBAAL,QAAK,aAAca,0BAAd,UAAA,8BAAA,SAAA,0BAAA,kCAAqB;oBAArB,IAAMQ,OAAN;oBACH,IAAIrB,SAASqB,QAAQtB,mBAAmBC,MAAMqB,OAAOD,WAAWE,GAAG,CAACtB;gBACtE;;gBAFK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAGP;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,IAAIoB,WAAWG,IAAI,KAAK,GAAG,OAAOjB;IAClC,IAAMkB,SAAS,AAAC,qBAAGJ,YAAYF,GAAG,CAAC,SAAClB;eAAS,AAAC,IAA8B,OAA3BA,KAAKyB,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;OAAIA,IAAI,CAAC;IACrF,OAAO,IAAIC,MAAM,AAAC,GAAoBH,OAAlBlB,IAAIG,OAAO,EAAC,QAAa,OAAPe,QAAO;AAC/C"}

package/dist/cjs/fences.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column\n// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see\n// test/unit/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself\n// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the\n// opener's, holding nothing but trailing spaces after the run.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n\n// Masks <!-- ... --> spans that may cross line boundaries, one line at a time so callers can\n// interleave it with their own per-line state. In isolation an unclosed <!-- stays open forever\n// (mask() has no notion of a line boundary that should kill it); maskRegions() below is what\n// calls close() at the point an unclosed comment actually dies -- a blank line at paragraph\n// level, or its enclosing HTML block's end.\nexport interface CommentTracker {\n  mask(line: string): string;\n  close(): void; // force-ends an unclosed comment (paragraph blank line, or its HTML block ending)\n  readonly inComment: boolean;\n}\n\nexport function commentTracker(): CommentTracker {\n  let inComment = false;\n  return {\n    mask(line: string): string {\n      let out = '';\n      let i = 0;\n      while (i < line.length) {\n        if (inComment) {\n          const end = line.indexOf('-->', i);\n          if (end === -1) {\n            out += ' '.repeat(line.length - i);\n            i = line.length;\n          } else {\n            out += ' '.repeat(end + 3 - i);\n            i = end + 3;\n            inComment = false;\n          }\n          continue;\n        }\n        const start = line.indexOf('<!--', i);\n        if (start === -1) {\n          out += line.slice(i);\n          i = line.length;\n        } else {\n          out += `${line.slice(i, start)}    `;\n          i = start + 4;\n          inComment = true;\n        }\n      }\n      return out;\n    },\n    close() {\n      inComment = false;\n    },\n    get inComment() {\n      return inComment;\n    },\n  };\n}\n\n// CommonMark's HTML-block type-6 list (fixed by the spec, not a drifting enumeration): a line\n// starting with an open or close tag of one of these, at column 0 (leading whitespace/`>`\n// blockquote markers tolerated), opens a block that swallows following lines -- tags, links,\n// and comments alike -- until a blank line closes it.\nconst HTML_BLOCK_TAGS = new Set([\n  'address',\n  'article',\n  'aside',\n  'base',\n  'basefont',\n  'blockquote',\n  'body',\n  'caption',\n  'center',\n  'col',\n  'colgroup',\n  'dd',\n  'details',\n  'dialog',\n  'dir',\n  'div',\n  'dl',\n  'dt',\n  'fieldset',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'frame',\n  'frameset',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'head',\n  'header',\n  'hr',\n  'html',\n  'iframe',\n  'legend',\n  'li',\n  'link',\n  'main',\n  'menu',\n  'menuitem',\n  'nav',\n  'noframes',\n  'ol',\n  'optgroup',\n  'option',\n  'p',\n  'param',\n  'search',\n  'section',\n  'summary',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'title',\n  'tr',\n  'track',\n  'ul',\n]);\n// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,\n// not on a blank line, and that line is the last one swallowed.\nconst HTML_PRE_TAGS = new Set(['script', 'pre', 'style', 'textarea']);\nconst HTML_PRE_CLOSE_RE = /<\\/(?:script|pre|style|textarea)>/i; // any of the four closers ends the block, not only the tag that opened it\n// An opening or closing tag at column 0, tag name captured for the lookups above.\nconst HTML_BLOCK_OPEN_RE = /^<\\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \\t]|\\/?>|$)/;\nconst BLANK_LINE_RE = /^[ \\t>]*$/; // whitespace/blockquote-markers only -- a paragraph break, and what ends a type-6 block\n\n// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines\n// preserved, so line numbers and offsets survive) plus inline code spans masked. The single\n// region masker for tags/links: fence and HTML-block delimiter+content lines are opaque, an\n// unclosed comment dies at its containing block's end (or the next blank line at paragraph\n// level -- never silently to end of file), and a closed comment masks exactly its span.\n// Bodies without a backtick/tilde/`<` fast-path out untouched.\nexport function maskRegions(body: string): string {\n  if (!/[`~<]/.test(body)) return body;\n  const fence = fenceTracker();\n  const comment = commentTracker();\n  let inHtmlBlock = false;\n  let htmlBlockClose: RegExp | null = null; // set while inside a type-1 (script/pre/style/textarea) block\n  return body\n    .split('\\n')\n    .map((line) => {\n      if (inHtmlBlock) {\n        if (htmlBlockClose) {\n          if (htmlBlockClose.test(line)) {\n            inHtmlBlock = false;\n            htmlBlockClose = null;\n          }\n          return ' '.repeat(line.length);\n        }\n        if (BLANK_LINE_RE.test(line)) {\n          // Block content never feeds the comment tracker, so an inner <!-- was only ever\n          // masked text; nothing to close here.\n          inHtmlBlock = false;\n          return line;\n        }\n        return ' '.repeat(line.length);\n      }\n      if (!comment.inComment) {\n        const isDelim = fence.feed(line);\n        if (isDelim || fence.inFence) return ' '.repeat(line.length);\n      }\n      const uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;\n      if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line\n      // Indented or blockquoted HTML blocks still swallow their content in Obsidian, so the\n      // opener test runs after stripping leading whitespace and > markers.\n      const stripped = uncommented.replace(/^[ \\t>]*/, '');\n      if (stripped[0] === '<') {\n        const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);\n        if (openMatch) {\n          const tagName = openMatch[1].toLowerCase();\n          const isClosingTag = stripped[1] === '/';\n          if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {\n            // The end condition can be met on the opening line itself: <script>x</script>\n            // is a one-line block and the next line parses (Obsidian-verified).\n            if (!HTML_PRE_CLOSE_RE.test(stripped.slice(openMatch[0].length))) {\n              inHtmlBlock = true;\n              htmlBlockClose = HTML_PRE_CLOSE_RE;\n            }\n            return ' '.repeat(line.length);\n          }\n          if (HTML_BLOCK_TAGS.has(tagName)) {\n            inHtmlBlock = true;\n            return ' '.repeat(line.length);\n          }\n        }\n      }\n      // Inline code spans hide their content too: `[[x]]` in prose is not a link.\n      return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;\n    })\n    .join('\\n');\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N -- a\n// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).\n// Masked with spaces so column positions and tag-boundary whitespace are unaffected.\nexport function maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n"],"names":["commentTracker","fenceTracker","maskCodeSpans","maskRegions","BACKTICK_OPEN","TILDE_OPEN","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim","inComment","mask","out","i","end","indexOf","repeat","start","slice","close","HTML_BLOCK_TAGS","Set","HTML_PRE_TAGS","HTML_PRE_CLOSE_RE","HTML_BLOCK_OPEN_RE","BLANK_LINE_RE","body","test","fence","comment","inHtmlBlock","htmlBlockClose","split","map","isDelim","uncommented","stripped","replace","openMatch","tagName","toLowerCase","isClosingTag","has","join","j","n","k","closeStart","closeEnd"],"mappings":"AAAA,+FAA+F;AAC/F,8FAA8F;AAC9F,+FAA+F;AAC/F,yFAAyF;AACzF,+DAA+D;;;;;;;;;;;;QA0D/CA;eAAAA;;QAhDAC;eAAAA;;QAyOAC;eAAAA;;QA/DAC;eAAAA;;;AA7KhB,IAAMC,gBAAgB;AACtB,IAAMC,aAAa;AAEZ,SAASJ;IACd,IAAIK,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAAA,SAAAA,KAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,IAAMK,KAAKP,cAAcQ,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,IAAMC,KAAKV,WAAWO,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,IAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,IAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAaO,SAASN;IACd,IAAImB,YAAY;IAChB,OAAO;QACLC,MAAAA,SAAAA,KAAKV,IAAY;YACf,IAAIW,MAAM;YACV,IAAIC,IAAI;YACR,MAAOA,IAAIZ,KAAKI,MAAM,CAAE;gBACtB,IAAIK,WAAW;oBACb,IAAMI,MAAMb,KAAKc,OAAO,CAAC,OAAOF;oBAChC,IAAIC,QAAQ,CAAC,GAAG;wBACdF,OAAO,IAAII,MAAM,CAACf,KAAKI,MAAM,GAAGQ;wBAChCA,IAAIZ,KAAKI,MAAM;oBACjB,OAAO;wBACLO,OAAO,IAAII,MAAM,CAACF,MAAM,IAAID;wBAC5BA,IAAIC,MAAM;wBACVJ,YAAY;oBACd;oBACA;gBACF;gBACA,IAAMO,QAAQhB,KAAKc,OAAO,CAAC,QAAQF;gBACnC,IAAII,UAAU,CAAC,GAAG;oBAChBL,OAAOX,KAAKiB,KAAK,CAACL;oBAClBA,IAAIZ,KAAKI,MAAM;gBACjB,OAAO;oBACLO,OAAO,AAAC,GAAuB,OAArBX,KAAKiB,KAAK,CAACL,GAAGI,QAAO;oBAC/BJ,IAAII,QAAQ;oBACZP,YAAY;gBACd;YACF;YACA,OAAOE;QACT;QACAO,OAAAA,SAAAA;YACET,YAAY;QACd;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,6FAA6F;AAC7F,sDAAsD;AACtD,IAAMU,kBAAkB,IAAIC,IAAI;IAC9B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AACD,8FAA8F;AAC9F,gEAAgE;AAChE,IAAMC,gBAAgB,IAAID,IAAI;IAAC;IAAU;IAAO;IAAS;CAAW;AACpE,IAAME,oBAAoB,sCAAsC,0EAA0E;AAC1I,kFAAkF;AAClF,IAAMC,qBAAqB;AAC3B,IAAMC,gBAAgB,aAAa,wFAAwF;AAQpH,SAAS/B,YAAYgC,IAAY;IACtC,IAAI,CAAC,QAAQC,IAAI,CAACD,OAAO,OAAOA;IAChC,IAAME,QAAQpC;IACd,IAAMqC,UAAUtC;IAChB,IAAIuC,cAAc;IAClB,IAAIC,iBAAgC,MAAM,8DAA8D;IACxG,OAAOL,KACJM,KAAK,CAAC,MACNC,GAAG,CAAC,SAAChC;QACJ,IAAI6B,aAAa;YACf,IAAIC,gBAAgB;gBAClB,IAAIA,eAAeJ,IAAI,CAAC1B,OAAO;oBAC7B6B,cAAc;oBACdC,iBAAiB;gBACnB;gBACA,OAAO,IAAIf,MAAM,CAACf,KAAKI,MAAM;YAC/B;YACA,IAAIoB,cAAcE,IAAI,CAAC1B,OAAO;gBAC5B,gFAAgF;gBAChF,sCAAsC;gBACtC6B,cAAc;gBACd,OAAO7B;YACT;YACA,OAAO,IAAIe,MAAM,CAACf,KAAKI,MAAM;QAC/B;QACA,IAAI,CAACwB,QAAQnB,SAAS,EAAE;YACtB,IAAMwB,UAAUN,MAAM5B,IAAI,CAACC;YAC3B,IAAIiC,WAAWN,MAAM/B,OAAO,EAAE,OAAO,IAAImB,MAAM,CAACf,KAAKI,MAAM;QAC7D;QACA,IAAM8B,cAAcN,QAAQnB,SAAS,IAAIT,KAAKG,QAAQ,CAAC,UAAUyB,QAAQlB,IAAI,CAACV,QAAQA;QACtF,IAAI4B,QAAQnB,SAAS,IAAIe,cAAcE,IAAI,CAAC1B,OAAO4B,QAAQV,KAAK,IAAI,+CAA+C;QACnH,sFAAsF;QACtF,qEAAqE;QACrE,IAAMiB,WAAWD,YAAYE,OAAO,CAAC,YAAY;QACjD,IAAID,QAAQ,CAAC,EAAE,KAAK,KAAK;YACvB,IAAME,YAAYd,mBAAmBrB,IAAI,CAACiC;YAC1C,IAAIE,WAAW;gBACb,IAAMC,UAAUD,SAAS,CAAC,EAAE,CAACE,WAAW;gBACxC,IAAMC,eAAeL,QAAQ,CAAC,EAAE,KAAK;gBACrC,IAAI,CAACK,gBAAgBnB,cAAcoB,GAAG,CAACH,UAAU;oBAC/C,8EAA8E;oBAC9E,oEAAoE;oBACpE,IAAI,CAAChB,kBAAkBI,IAAI,CAACS,SAASlB,KAAK,CAACoB,SAAS,CAAC,EAAE,CAACjC,MAAM,IAAI;wBAChEyB,cAAc;wBACdC,iBAAiBR;oBACnB;oBACA,OAAO,IAAIP,MAAM,CAACf,KAAKI,MAAM;gBAC/B;gBACA,IAAIe,gBAAgBsB,GAAG,CAACH,UAAU;oBAChCT,cAAc;oBACd,OAAO,IAAId,MAAM,CAACf,KAAKI,MAAM;gBAC/B;YACF;QACF;QACA,4EAA4E;QAC5E,OAAO8B,YAAY/B,QAAQ,CAAC,OAAOX,cAAc0C,eAAeA;IAClE,GACCQ,IAAI,CAAC;AACV;AAKO,SAASlD,cAAcQ,IAAY;IACxC,IAAIW,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIZ,KAAKI,MAAM,CAAE;QACtB,IAAIJ,IAAI,CAACY,EAAE,KAAK,KAAK;YACnBD,OAAOX,IAAI,CAACY,EAAE;YACdA;YACA;QACF;QACA,IAAI+B,IAAI/B;QACR,MAAOZ,IAAI,CAAC2C,EAAE,KAAK,IAAKA;QACxB,IAAMC,IAAID,IAAI/B;QACd,IAAIiC,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAI7C,KAAKI,MAAM,CAAE;YACtB,IAAIJ,IAAI,CAAC6C,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAItC,IAAIsC;YACR,MAAO7C,IAAI,CAACO,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIsC,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAWxC;gBACX;YACF;YACAsC,IAAItC;QACN;QACA,IAAIuC,cAAc,GAAG;YACnBnC,OAAO,IAAII,MAAM,CAACgC,WAAWnC;YAC7BA,IAAImC;QACN,OAAO;YACLpC,OAAOX,KAAKiB,KAAK,CAACL,GAAG+B;YACrB/B,IAAI+B;QACN;IACF;IACA,OAAOhC;AACT"}