sensemaking 0.15.4 → 0.17.0

package/dist/esm/{scan.js → scan/frontmatter.js}

@@ -1,9 +1,4 @@
-import { globSync, readFileSync, statSync } from 'node:fs';
-import { join, sep } from 'node:path';
-import removeMarkdown from 'remove-markdown';
 import { isCollection, parseDocument, visit } from 'yaml';
-import { embedEnabled, presetNames, presetSemanticEnabled } from './config/index.js';
-// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
 // Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell
 // a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.
 export const RESERVED_COLUMNS = new Set([
@@ -26,73 +21,10 @@ export const RESERVED_COLUMNS = new Set([
 const ACCEPTED_YAML_CODES = new Set([
     'BAD_SCALAR_START'
 ]);
-function normalizeText(value) {
+export function normalizeText(value) {
     if (value === null || value === undefined) return '';
     return String(value).replace(/\s+/g, ' ').trim();
 }
-// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms
-// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.
-function stripText(value) {
-    const withoutWikilinks = value.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2').replace(/\[\[([^\]]+)\]\]/g, '$1');
-    const withoutMarkdown = removeMarkdown(withoutWikilinks);
-    const withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
-    return normalizeText(withoutTables);
-}
-// Presets are views, not partitions: they overlap freely, and a file's covering set (not one
-// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.
-export function toPosixPath(relPath, separator = sep) {
-    return separator === '\\' ? relPath.split(separator).join('/') : relPath;
-}
-// Every command pays listFiles before it answers (the freshness check stats each file), so
-// per-file work here is the hottest path in the package. Everything derivable from the config
-// alone is computed once, above the loop.
-const NO_THROW = {
-    throwIfNoEntry: false
-};
-export function listFiles(cfg, baseDir) {
-    const coverage = new Map();
-    const posixNeeded = sep === '\\';
-    for (const name of presetNames(cfg)){
-        const preset = cfg.presets[name];
-        for (const matched of globSync(preset.include, {
-            cwd: baseDir,
-            exclude: preset.exclude
-        })){
-            var _coverage_get;
-            const relPath = posixNeeded ? toPosixPath(matched) : matched;
-            const set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
-            set.add(name);
-            coverage.set(relPath, set);
-        }
-    }
-    // Which presets want vectors is a property of the config, not of any file.
-    const embedding = embedEnabled(cfg);
-    const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name)=>presetSemanticEnabled(cfg, name))) : null;
-    const files = [];
-    for (const relPath of [
-        ...coverage.keys()
-    ].sort()){
-        const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls
-        // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so
-        // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).
-        const st = statSync(absPath, NO_THROW);
-        if (!(st === null || st === void 0 ? void 0 : st.isFile())) continue;
-        const presets = [
-            ...coverage.get(relPath)
-        ].sort();
-        const embed = semanticPresets !== null && presets.some((name)=>semanticPresets.has(name));
-        files.push({
-            relPath,
-            absPath,
-            mtimeMs: st.mtimeMs,
-            ctimeMs: st.birthtimeMs,
-            size: st.size,
-            presets,
-            embed
-        });
-    }
-    return files;
-}
 // SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601
 // allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.
 const ISO_DATETIME = /^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?)(Z|[+-]\d{2}(?::?\d{2})?)?$/;
@@ -115,7 +47,7 @@ export function normalizeDate(value) {
 }
 // Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches
 // and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.
-function mapValue(value) {
+export function mapValue(value) {
     if (value === null || value === undefined) return null;
     if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
     if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;
@@ -123,7 +55,7 @@ function mapValue(value) {
     return JSON.stringify(value);
 }
 // The delimiter split is all this package used gray-matter for.
-function splitFrontmatter(raw) {
+export function splitFrontmatter(raw) {
     const open = raw.match(/^---\r?\n/);
     if (!open) return {
         fm: null,
@@ -169,7 +101,7 @@ function warnStringifiedKeys(relPath, doc, warnings) {
 function firstLine(message) {
     return message.split('\n')[0].replace(/:\s*$/, '');
 }
-function parseFrontmatter(relPath, fm, warnings) {
+export function parseFrontmatter(relPath, fm, warnings) {
     // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one
     // that carries information, with the file it came from.
     const doc = parseDocument(fm, {
@@ -216,51 +148,3 @@ function parseFrontmatter(relPath, fm, warnings) {
         parseError: null
     };
 }
-export function parseFile(file, extractors = []) {
-    const raw = readFileSync(file.absPath, 'utf8');
-    const warnings = [];
-    const { fm, body: content } = splitFrontmatter(raw);
-    const { data, parseError } = fm === null ? {
-        data: {},
-        parseError: null
-    } : parseFrontmatter(file.relPath, fm, warnings);
-    const mapped = {};
-    for (const key of Object.keys(data)){
-        if (RESERVED_COLUMNS.has(key)) {
-            warnings.push(`warning: ${file.relPath} has a frontmatter key named "${key}", which is reserved; ignoring it`);
-            continue;
-        }
-        const value = mapValue(data[key]);
-        if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {
-            warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);
-        }
-        mapped[key] = value;
-    }
-    // title/summary are plain YAML strings -- whitespace-collapse only;
-    // the prose gets the full markdown strip.
-    const search = {
-        title: normalizeText(data.title),
-        summary: normalizeText(data.summary),
-        text: stripText(content)
-    };
-    return {
-        doc: {
-            relPath: file.relPath,
-            mtimeMs: file.mtimeMs,
-            ctimeMs: file.ctimeMs,
-            size: file.size,
-            presets: file.presets,
-            data: mapped,
-            parseError,
-            search,
-            extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
-                var _f_extract;
-                return [
-                    f.name,
-                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search, data)
-                ];
-            }))
-        },
-        warnings
-    };
-}

package/dist/esm/scan/frontmatter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["isCollection","parseDocument","visit","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","Pair","_key","pair","key","BREAK","push","firstLine","message","split","parseFrontmatter","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":"AAAA,SAASA,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAExI,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4DAA4D;AAC5D,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,OAAO,SAASE,cAAcC,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBP,KAAa;IAC7C,OAAOM,kBAAkBE,IAAI,CAACR;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASS,cAAcT,KAAa;IACzC,MAAMU,IAAIL,aAAaM,IAAI,CAACX;IAC5B,IAAIU,MAAM,MAAM,OAAOV;IACvB,MAAM,GAAGY,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAASb,aAAaa,SAAS,MAAM,KAAKA,KAAKX,OAAO,CAAC,KAAK;IAC3E,MAAMa,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAenB,QAAQmB;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,OAAO,SAASK,SAASxB,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOoB,OAAOM,aAAa,CAAC1B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOS,cAAcT;IACpD,OAAO2B,KAAKC,SAAS,CAAC5B;AACxB;AAEA,gEAAgE;AAChE,OAAO,SAAS6B,iBAAiBC,GAAW;IAC1C,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,MAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKpC,WAAW,OAAO;QAAEgC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC/C,MAAM6C,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACpD,aAAaoD,KAAKC,GAAG,GAAG,OAAO7C;YACpCyC,QAAQ;YACR,OAAO/C,MAAMoD,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASU,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAChD,OAAO,CAAC,SAAS;AACjD;AAEA,OAAO,SAASiD,iBAAiBb,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAM9C,cAAcuC,IAAI;QAAEoB,UAAU;IAAS;IACnD,MAAMC,UAAUd,IAAIe,MAAM,CAACC,MAAM,CAAC,CAACC,MAAQ,CAAC3D,oBAAoB4D,GAAG,CAACD,IAAIE,IAAI;IAC5E,IAAIL,QAAQrC,MAAM,GAAG,GAAG;QACtB,MAAM2C,SAASN,QAAQrC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEqC,QAAQrC,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAM4C,aAAa,GAAGZ,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,IAAIU,QAAQ;QAC9DnB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOtB,IAAIuB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMI,aAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDT,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,sDAAsD,EAAEsB,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS7D,WAAW,OAAO;QAAE6D,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBpB,SAASO,IAAI,CAAC,CAAC,SAAS,EAAET,QAAQ,CAAC,EAAEsB,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAvB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEqB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}

package/dist/esm/scan/index.d.ts

@@ -0,0 +1,25 @@
+import type { Config } from '../config/index.js';
+import type { Feature } from '../features/types.js';
+import type { FileStat } from './list.js';
+export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
+export type { FileStat } from './list.js';
+export { listFiles, toPosixPath } from './list.js';
+export interface ParsedDoc {
+    relPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    data: Record<string, string | number | bigint | null>;
+    parseError: string | null;
+    search: {
+        title: string;
+        summary: string;
+        text: string;
+    };
+    extracted: Record<string, unknown>;
+}
+export declare function parseFile(file: FileStat, extractors?: Feature[], cfg?: Config): {
+    doc: ParsedDoc;
+    warnings: string[];
+};

package/dist/esm/scan/index.js

@@ -0,0 +1,58 @@
+import { readFileSync } from 'node:fs';
+import { parse } from '../chunk/index.js';
+import { textFromBlocks } from '../text/strip.js';
+import { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.js';
+// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
+export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
+export { listFiles, toPosixPath } from './list.js';
+export function parseFile(file, extractors = [], cfg) {
+    const raw = readFileSync(file.absPath, 'utf8');
+    const warnings = [];
+    const { fm, body: content } = splitFrontmatter(raw);
+    const { data, parseError } = fm === null ? {
+        data: {},
+        parseError: null
+    } : parseFrontmatter(file.relPath, fm, warnings);
+    const mapped = {};
+    for (const key of Object.keys(data)){
+        if (RESERVED_COLUMNS.has(key)) {
+            warnings.push(`warning: ${file.relPath} has a frontmatter key named "${key}", which is reserved; ignoring it`);
+            continue;
+        }
+        const value = mapValue(data[key]);
+        if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {
+            warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);
+        }
+        mapped[key] = value;
+    }
+    // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature
+    // that needs a parse tree (embed's chunker) both read this same block list.
+    const blocks = parse(content);
+    // title/summary are plain YAML strings -- whitespace-collapse only;
+    // the prose gets the full markdown strip.
+    const search = {
+        title: normalizeText(data.title),
+        summary: normalizeText(data.summary),
+        text: textFromBlocks(blocks)
+    };
+    return {
+        doc: {
+            relPath: file.relPath,
+            mtimeMs: file.mtimeMs,
+            ctimeMs: file.ctimeMs,
+            size: file.size,
+            presets: file.presets,
+            data: mapped,
+            parseError,
+            search,
+            extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
+                var _f_extract;
+                return [
+                    f.name,
+                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search, data, cfg, blocks)
+                ];
+            }))
+        },
+        warnings
+    };
+}

package/dist/esm/scan/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","parse","textFromBlocks","looksLikeDatetime","mapValue","normalizeText","parseFrontmatter","RESERVED_COLUMNS","splitFrontmatter","normalizeDate","listFiles","toPosixPath","parseFile","file","extractors","cfg","raw","absPath","warnings","fm","body","content","data","parseError","relPath","mapped","key","Object","keys","has","push","value","Number","isNaN","Date","blocks","search","title","summary","text","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,KAAK,QAAQ,oBAAoB;AAG1C,SAASC,cAAc,QAAQ,mBAAmB;AAClD,SAASC,iBAAiB,EAAEC,QAAQ,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAQ,mBAAmB;AAGpI,6EAA6E;AAE7E,SAASL,iBAAiB,EAAEM,aAAa,EAAEF,gBAAgB,QAAQ,mBAAmB;AAEtF,SAASG,SAAS,EAAEC,WAAW,QAAQ,YAAY;AAkBnD,OAAO,SAASC,UAAUC,IAAc,EAAEC,aAAwB,EAAE,EAAEC,GAAY;IAChF,MAAMC,MAAMhB,aAAaa,KAAKI,OAAO,EAAE;IACvC,MAAMC,WAAqB,EAAE;IAE7B,MAAM,EAAEC,EAAE,EAAEC,MAAMC,OAAO,EAAE,GAAGb,iBAAiBQ;IAC/C,MAAM,EAAEM,IAAI,EAAEC,UAAU,EAAE,GAAGJ,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIjB,iBAAiBO,KAAKW,OAAO,EAAEL,IAAID;IAC1I,MAAMO,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACN,MAAO;QACnC,IAAIf,iBAAiBsB,GAAG,CAACH,MAAM;YAC7BR,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,8BAA8B,EAAEE,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMK,QAAQ3B,SAASkB,IAAI,CAACI,IAAI;QAChC,IAAI,OAAOK,UAAU,YAAY5B,kBAAkB4B,UAAUC,OAAOC,KAAK,CAACC,KAAKjC,KAAK,CAAC8B,SAAS;YAC5Fb,SAASY,IAAI,CAAC,CAAC,SAAS,EAAEjB,KAAKW,OAAO,CAAC,EAAE,EAAEE,IAAI,sBAAsB,EAAEK,MAAM,8CAA8C,CAAC;QAC9H;QACAN,MAAM,CAACC,IAAI,GAAGK;IAChB;IAEA,yFAAyF;IACzF,4EAA4E;IAC5E,MAAMI,SAASlC,MAAMoB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMe,SAAS;QAAEC,OAAOhC,cAAciB,KAAKe,KAAK;QAAGC,SAASjC,cAAciB,KAAKgB,OAAO;QAAGC,MAAMrC,eAAeiC;IAAQ;IAEtH,OAAO;QACLK,KAAK;YACHhB,SAASX,KAAKW,OAAO;YACrBiB,SAAS5B,KAAK4B,OAAO;YACrBC,SAAS7B,KAAK6B,OAAO;YACrBC,MAAM9B,KAAK8B,IAAI;YACfC,SAAS/B,KAAK+B,OAAO;YACrBtB,MAAMG;YACNF;YACAa;YACAS,WAAWlB,OAAOmB,WAAW,CAAChC,WAAWiC,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhC,KAAKK,SAASe,QAAQd,MAAMP,KAAKoB;iBAAQ;;QAC7I;QACAjB;IACF;AACF"}

package/dist/esm/scan/list.d.ts

@@ -0,0 +1,12 @@
+import type { Config } from '../config/index.js';
+export interface FileStat {
+    relPath: string;
+    absPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    embed: boolean;
+}
+export declare function toPosixPath(relPath: string, separator?: string): string;
+export declare function listFiles(cfg: Config, baseDir: string): FileStat[];

package/dist/esm/scan/list.js

@@ -0,0 +1,58 @@
+import { globSync, statSync } from 'node:fs';
+import { join, sep } from 'node:path';
+import { embedEnabled, presetHasSignal, presetNames } from '../config/index.js';
+// Presets are views, not partitions: they overlap freely, and a file's covering set (not one
+// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.
+export function toPosixPath(relPath, separator = sep) {
+    return separator === '\\' ? relPath.split(separator).join('/') : relPath;
+}
+// Every command pays listFiles before it answers (the freshness check stats each file), so
+// per-file work here is the hottest path in the package. Everything derivable from the config
+// alone is computed once, above the loop.
+const NO_THROW = {
+    throwIfNoEntry: false
+};
+export function listFiles(cfg, baseDir) {
+    const coverage = new Map();
+    const posixNeeded = sep === '\\';
+    for (const name of presetNames(cfg)){
+        const preset = cfg.presets[name];
+        for (const matched of globSync(preset.include, {
+            cwd: baseDir,
+            exclude: preset.exclude
+        })){
+            var _coverage_get;
+            const relPath = posixNeeded ? toPosixPath(matched) : matched;
+            const set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
+            set.add(name);
+            coverage.set(relPath, set);
+        }
+    }
+    // Which presets want vectors is a property of the config, not of any file.
+    const embedding = embedEnabled(cfg);
+    const vectorPresets = embedding ? new Set(presetNames(cfg).filter((name)=>presetHasSignal(cfg, name, 'vectors'))) : null;
+    const files = [];
+    for (const relPath of [
+        ...coverage.keys()
+    ].sort()){
+        const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls
+        // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so
+        // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).
+        const st = statSync(absPath, NO_THROW);
+        if (!(st === null || st === void 0 ? void 0 : st.isFile())) continue;
+        const presets = [
+            ...coverage.get(relPath)
+        ].sort();
+        const embed = vectorPresets !== null && presets.some((name)=>vectorPresets.has(name));
+        files.push({
+            relPath,
+            absPath,
+            mtimeMs: st.mtimeMs,
+            ctimeMs: st.birthtimeMs,
+            size: st.size,
+            presets,
+            embed
+        });
+    }
+    return files;
+}

package/dist/esm/scan/list.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/list.ts"],"sourcesContent":["import { globSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedEnabled, presetHasSignal, presetNames } from '../config/index.ts';\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  ctimeMs: number; // filesystem birthtime; a clone or copy resets it, like _mtime\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset's signals include vectors\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const vectorPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetHasSignal(cfg, name, 'vectors'))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = vectorPresets !== null && presets.some((name) => vectorPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, ctimeMs: st.birthtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n"],"names":["globSync","statSync","join","sep","embedEnabled","presetHasSignal","presetNames","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","Set","add","embedding","vectorPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","ctimeMs","birthtimeMs","size"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,QAAQ,QAAQ,UAAU;AAC7C,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AAEtC,SAASC,YAAY,EAAEC,eAAe,EAAEC,WAAW,QAAQ,qBAAqB;AAYhF,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBN,GAAG;IAClE,OAAOM,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWP,IAAI,CAAC,OAAOM;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAcf,QAAQ;IAC5B,KAAK,MAAMgB,QAAQb,YAAYQ,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWtB,SAASoB,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIY;YACzCF,IAAIG,GAAG,CAACV;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMI,YAAY1B,aAAaU;IAC/B,MAAMiB,gBAAgBD,YAAY,IAAIF,IAAItB,YAAYQ,KAAKkB,MAAM,CAAC,CAACb,OAASd,gBAAgBS,KAAKK,MAAM,eAAe;IAEtH,MAAMc,QAAoB,EAAE;IAC5B,KAAK,MAAMzB,WAAW;WAAIQ,SAASkB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlC,KAAKa,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM6B,KAAKpC,SAASmC,SAASzB;QAC7B,IAAI,EAAC0B,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMjB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC2B,IAAI;QAChE,MAAMI,QAAQR,kBAAkB,QAAQV,QAAQmB,IAAI,CAAC,CAACrB,OAASY,cAAcU,GAAG,CAACtB;QACjFc,MAAMS,IAAI,CAAC;YAAElC;YAAS4B;YAASO,SAASN,GAAGM,OAAO;YAAEC,SAASP,GAAGQ,WAAW;YAAEC,MAAMT,GAAGS,IAAI;YAAEzB;YAASkB;QAAM;IAC7G;IACA,OAAON;AACT"}

package/dist/esm/text/segment.d.ts

@@ -0,0 +1,3 @@
+export declare const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}";
+export declare function segmentField(text: string): string;
+export declare function segmentMatch(terms: string): string;

package/dist/esm/text/segment.js

@@ -0,0 +1,95 @@
+// Word boundaries FTS5's tokenizers cannot find on their own.
+//
+// Segmentation is context-free: seg(query) appears inside seg(document) whenever the query is
+// a substring of the document. That is the contract -- substring semantics, what `grep` and
+// `LIKE '%..%'` give -- and it is what makes index and query agree on every input, always.
+// Intl.Segmenter's word mode is context-dependent (`东京都政府` splits as `东 | 京都 | 政府`
+// while the query `东京` splits as one word) and is rejected for that reason. Its grapheme
+// mode has no such dependency -- UAX #29 cluster boundaries never look past adjacent
+// characters -- so grapheme mode is used below; no dictionary, no word mode.
+// Scripts written without word spaces: a closed set of writing systems.
+// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.
+export const UNSPACED_SCRIPTS = '\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}';
+// A run is script BASE characters with their combining marks attached; a bare mark after a
+// Latin letter (decomposed é) never starts one.
+const RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\p{M}*)+)`, 'gu');
+const HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');
+// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.
+// Built once (construction cost amortizes) and used for both index and query splitting.
+const GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, {
+    granularity: 'grapheme'
+});
+// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions
+// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.
+const PUNCTUATION = /\p{P}/u;
+// Token barrier between separate runs (or a punctuation split within one) so their graphemes
+// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.
+const BARRIER = 'ꟷ';
+function graphemes(run) {
+    return Array.from(GRAPHEME_SEGMENTER.segment(run), (s)=>s.segment);
+}
+// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,
+// matching unicode61). The one place punctuation is classified, so index and query agree.
+function splitOnPunctuation(run) {
+    const groups = [
+        []
+    ];
+    for (const g of graphemes(run)){
+        if (PUNCTUATION.test(g)) groups.push([]);
+        else groups[groups.length - 1].push(g);
+    }
+    return groups.filter((g)=>g.length > 0);
+}
+// Index side: '' when the field has no unspaced-script run (the common case, paid for by
+// nothing). Otherwise each run explodes into its graphemes, barrier-delimited from its
+// neighbors and from a punctuation split within itself.
+export function segmentField(text) {
+    if (!HAS_RUN.test(text)) return '';
+    const out = text.replace(RUN, (run)=>{
+        const body = splitOnPunctuation(run).map((g)=>g.join(' ')).join(` ${BARRIER} `);
+        return ` ${BARRIER} ${body} ${BARRIER} `;
+    });
+    return out.replace(/\s+/g, ' ').trim();
+}
+// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a
+// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.
+const QUALIFIER = /(^|[\s(])(-?)(title|summary|text)\s*:\s*$/;
+// An unqualified run's target: raw title/summary/text drop punctuation as unicode61's token
+// separator, so two adjacent single-grapheme phrase positions match across punctuation the same
+// as across a real gap (`数数` vs `数。数`) -- a false positive only the barriered `_seg` columns
+// are safe from. FTS5's column-set filter, not parens+OR, so the group still composes under
+// AND/OR/NOT/juxtaposition exactly like the single-column qualifier form below.
+const SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';
+// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),
+// space-joined -- the same split points segmentField barriers, so query and index agree.
+function runQuery(run, columnPrefix) {
+    return splitOnPunctuation(run).map((g)=>`${columnPrefix}${g.length > 1 ? `"${g.join(' ')}"` : g[0]}`).join(' ');
+}
+// Query side: each unspaced run becomes phrases of its graphemes, matching how segmentField
+// indexed it. A qualifier ahead of such a run maps to its `_seg` column; unqualified maps to all
+// three (SIDECAR_COLUMNS), since raw columns cannot express the contract. An author's own quoted
+// phrase is their explicit escape hatch to FTS5's native syntax, and passes through byte-identical.
+export function segmentMatch(terms) {
+    if (!HAS_RUN.test(terms)) return terms;
+    let out = '';
+    let quoted = false;
+    const pieces = terms.split(RUN); // split keeps captured runs at odd indices
+    for(let i = 0; i < pieces.length; i++){
+        if (i % 2 === 0) {
+            for (const ch of pieces[i])if (ch === '"') quoted = !quoted;
+            out += pieces[i];
+            continue;
+        }
+        if (quoted) {
+            out += pieces[i]; // an author's phrase is matched as written
+            continue;
+        }
+        const m = out.match(QUALIFIER);
+        if (m) {
+            out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;
+        } else {
+            out += runQuery(pieces[i], SIDECAR_COLUMNS);
+        }
+    }
+    return out;
+}

package/dist/esm/text/segment.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own.\n//\n// Segmentation is context-free: seg(query) appears inside seg(document) whenever the query is\n// a substring of the document. That is the contract -- substring semantics, what `grep` and\n// `LIKE '%..%'` give -- and it is what makes index and query agree on every input, always.\n// Intl.Segmenter's word mode is context-dependent (`东京都政府` splits as `东 | 京都 | 政府`\n// while the query `东京` splits as one word) and is rejected for that reason. Its grapheme\n// mode has no such dependency -- UAX #29 cluster boundaries never look past adjacent\n// characters -- so grapheme mode is used below; no dictionary, no word mode.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built once (construction cost amortizes) and used for both index and query splitting.\nconst GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  return Array.from(GRAPHEME_SEGMENTER.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run (the common case, paid for by\n// nothing). Otherwise each run explodes into its graphemes, barrier-delimited from its\n// neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// An unqualified run's target: raw title/summary/text drop punctuation as unicode61's token\n// separator, so two adjacent single-grapheme phrase positions match across punctuation the same\n// as across a real gap (`数数` vs `数。数`) -- a false positive only the barriered `_seg` columns\n// are safe from. FTS5's column-set filter, not parens+OR, so the group still composes under\n// AND/OR/NOT/juxtaposition exactly like the single-column qualifier form below.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching how segmentField\n// indexed it. A qualifier ahead of such a run maps to its `_seg` column; unqualified maps to all\n// three (SIDECAR_COLUMNS), since raw columns cannot express the contract. An author's own quoted\n// phrase is their explicit escape hatch to FTS5's native syntax, and passes through byte-identical.\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","RUN","RegExp","HAS_RUN","GRAPHEME_SEGMENTER","Intl","Segmenter","undefined","granularity","PUNCTUATION","BARRIER","graphemes","run","Array","from","segment","s","splitOnPunctuation","groups","g","test","push","length","filter","segmentField","text","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","segmentMatch","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,8DAA8D;AAC9D,EAAE;AACF,8FAA8F;AAC9F,4FAA4F;AAC5F,2FAA2F;AAC3F,mFAAmF;AACnF,yFAAyF;AACzF,qFAAqF;AACrF,6EAA6E;AAE7E,wEAAwE;AACxE,kFAAkF;AAClF,OAAO,MAAMA,mBAAmB,wGAAwG;AACxI,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,MAAM,IAAIC,OAAO,CAAC,KAAK,EAAEF,iBAAiB,WAAW,CAAC,EAAE;AAC9D,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEF,iBAAiB,CAAC,CAAC,EAAE;AACpD,6FAA6F;AAC7F,wFAAwF;AACxF,MAAMI,qBAAqB,IAAIC,KAAKC,SAAS,CAACC,WAAW;IAAEC,aAAa;AAAW;AACnF,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,MAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5B,OAAOC,MAAMC,IAAI,CAACV,mBAAmBW,OAAO,CAACH,MAAM,CAACI,IAAMA,EAAED,OAAO;AACrE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBL,GAAW;IACrC,MAAMM,SAAqB;QAAC,EAAE;KAAC;IAC/B,KAAK,MAAMC,KAAKR,UAAUC,KAAM;QAC9B,IAAIH,YAAYW,IAAI,CAACD,IAAID,OAAOG,IAAI,CAAC,EAAE;aAClCH,MAAM,CAACA,OAAOI,MAAM,GAAG,EAAE,CAACD,IAAI,CAACF;IACtC;IACA,OAAOD,OAAOK,MAAM,CAAC,CAACJ,IAAMA,EAAEG,MAAM,GAAG;AACzC;AAEA,yFAAyF;AACzF,uFAAuF;AACvF,wDAAwD;AACxD,OAAO,SAASE,aAAaC,IAAY;IACvC,IAAI,CAACtB,QAAQiB,IAAI,CAACK,OAAO,OAAO;IAChC,MAAMC,MAAMD,KAAKE,OAAO,CAAC1B,KAAK,CAACW;QAC7B,MAAMgB,OAAOX,mBAAmBL,KAC7BiB,GAAG,CAAC,CAACV,IAAMA,EAAEW,IAAI,CAAC,MAClBA,IAAI,CAAC,CAAC,CAAC,EAAEpB,QAAQ,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,EAAEA,QAAQ,CAAC,EAAEkB,KAAK,CAAC,EAAElB,QAAQ,CAAC,CAAC;IAC1C;IACA,OAAOgB,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,gGAAgG;AAChG,6FAA6F;AAC7F,4FAA4F;AAC5F,gFAAgF;AAChF,MAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAAStB,GAAW,EAAEuB,YAAoB;IACjD,OAAOlB,mBAAmBL,KACvBiB,GAAG,CAAC,CAACV,IAAM,GAAGgB,eAAehB,EAAEG,MAAM,GAAG,IAAI,CAAC,CAAC,EAAEH,EAAEW,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGX,CAAC,CAAC,EAAE,EAAE,EACvEW,IAAI,CAAC;AACV;AAEA,4FAA4F;AAC5F,iGAAiG;AACjG,iGAAiG;AACjG,oGAAoG;AACpG,OAAO,SAASM,aAAaC,KAAa;IACxC,IAAI,CAAClC,QAAQiB,IAAI,CAACiB,QAAQ,OAAOA;IACjC,IAAIX,MAAM;IACV,IAAIY,SAAS;IACb,MAAMC,SAASF,MAAMG,KAAK,CAACvC,MAAM,2CAA2C;IAC5E,IAAK,IAAIwC,IAAI,GAAGA,IAAIF,OAAOjB,MAAM,EAAEmB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAMC,MAAMH,MAAM,CAACE,EAAE,CAAE,IAAIC,OAAO,KAAKJ,SAAS,CAACA;YACtDZ,OAAOa,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVZ,OAAOa,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,MAAME,IAAIjB,IAAIkB,KAAK,CAACZ;QACpB,IAAIW,GAAG;YACLjB,MAAM,GAAGA,IAAImB,KAAK,CAAC,GAAGF,EAAEG,KAAK,IAAIH,CAAC,CAAC,EAAE,GAAGT,SAASK,MAAM,CAACE,EAAE,EAAE,GAAGE,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG;QACtF,OAAO;YACLjB,OAAOQ,SAASK,MAAM,CAACE,EAAE,EAAER;QAC7B;IACF;IACA,OAAOP;AACT"}

package/dist/esm/text/strip.d.ts

@@ -0,0 +1,3 @@
+import type { Block } from '../chunk/index.js';
+export declare function textFromBlocks(blocks: Block[]): string;
+export declare function stripText(value: string): string;

package/dist/esm/text/strip.js

@@ -0,0 +1,12 @@
+import { extractText, parse } from '../chunk/index.js';
+import { normalizeText } from '../scan/frontmatter.js';
+// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses
+// (src/chunk) -- words and vectors read one definition of "the prose of this note".
+export function textFromBlocks(blocks) {
+    const text = blocks.map((block)=>extractText(block.node)).filter((s)=>s.length > 0).join('\n');
+    return normalizeText(text);
+}
+// Thin parse + delegate, for callers with no blocks of their own already.
+export function stripText(value) {
+    return textFromBlocks(parse(value));
+}

package/dist/esm/text/strip.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractText, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  const text = blocks\n    .map((block) => extractText(block.node))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n  return normalizeText(text);\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractText","parse","normalizeText","textFromBlocks","blocks","text","map","block","node","filter","s","length","join","stripText","value"],"mappings":"AACA,SAASA,WAAW,EAAEC,KAAK,QAAQ,oBAAoB;AACvD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,4FAA4F;AAC5F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,MAAMC,OAAOD,OACVE,GAAG,CAAC,CAACC,QAAUP,YAAYO,MAAMC,IAAI,GACrCC,MAAM,CAAC,CAACC,IAAMA,EAAEC,MAAM,GAAG,GACzBC,IAAI,CAAC;IACR,OAAOV,cAAcG;AACvB;AAEA,0EAA0E;AAC1E,OAAO,SAASQ,UAAUC,KAAa;IACrC,OAAOX,eAAeF,MAAMa;AAC9B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.15.4",
+  "version": "0.17.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -38,6 +38,8 @@
     "agent-memory",
     "memory-consolidation",
     "llm-wiki",
+    "second-brain",
+    "company-brain",
     "dreaming",
     "cli"
   ],
@@ -77,17 +79,36 @@
   },
   "dependencies": {
     "@huggingface/tokenizers": "^0.1.3",
-    "remove-markdown": "^0.6.4",
+    "franc-min": "^6.2.0",
+    "mdast-util-from-markdown": "^2.0.3",
+    "mdast-util-gfm-autolink-literal": "^2.0.0",
+    "mdast-util-gfm-footnote": "^2.0.0",
+    "mdast-util-gfm-strikethrough": "^2.0.0",
+    "mdast-util-gfm-table": "^2.0.0",
+    "mdast-util-gfm-task-list-item": "^2.0.0",
+    "micromark-extension-gfm-autolink-literal": "^2.0.0",
+    "micromark-extension-gfm-footnote": "^2.0.0",
+    "micromark-extension-gfm-strikethrough": "^2.0.0",
+    "micromark-extension-gfm-table": "^2.0.0",
+    "micromark-extension-gfm-task-list-item": "^2.0.0",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
+    "@types/mdast": "^4.0.4",
     "@types/mocha": "*",
     "@types/node": "*",
+    "cr": "^0.1.0",
     "node-version-use": "*",
     "ts-dev-stack": "*",
     "tsds-config": "*"
   },
   "engines": {
     "node": ">=22.20"
+  },
+  "allowScripts": {
+    "node-semvers": true,
+    "node-filename-to-dist-paths": true,
+    "node-version-use": true,
+    "thread-sleep-compat": true
   }
 }