sensemaking 0.16.0 → 0.17.0

package/dist/cjs/{search-error.js → output/search-error.js}

@@ -8,7 +8,7 @@ Object.defineProperty(exports, "searchError", {
         return searchError;
     }
 });
-var _errorsts = require("./errors.js");
+var _errorsts = require("../errors.js");
 // FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and
 // errors `no such column: to` -- true about the parse, misleading about the input. Matched as
 // "not a bareword": an operator list is a list to keep current, and missing a character costs

package/dist/cjs/output/search-error.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output/search-error.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\n\n// FTS5 reads punctuation as syntax, so `end-to-end` parses as a filter on column `to` and\n// errors `no such column: to` -- true about the parse, misleading about the input. Matched as\n// \"not a bareword\": an operator list is a list to keep current, and missing a character costs\n// the remedy, not the error.\nconst FTS5_PUNCTUATION = /[^\\p{L}\\p{N}_\\s]/u;\n\nexport function searchError(err: Error, terms: string, scope?: string): Error {\n  const message = err.message;\n  if (!/no such column|fts5: syntax error|malformed MATCH/.test(message)) return err;\n  const suspects = (terms.match(/\\S+/g) ?? []).filter((t) => !t.startsWith('\"') && FTS5_PUNCTUATION.test(t));\n  // Blame the terms only when the failing token actually came from one -- a typo'd column\n  // in --where (or the tree's default scope) raises \"no such column\" through this same\n  // statement, and naming a term for it would state a false fact about the input.\n  const col = /no such column: (\\S+)/.exec(message)?.[1];\n  const fromTerms = col === undefined ? suspects.length > 0 : suspects.some((t) => t.split(/[^\\p{L}\\p{N}]+/u).includes(col));\n  if (fromTerms && suspects.length > 0) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the punctuation in ${suspects.map((t) => `\\`${t}\\``).join(', ')} is FTS5 syntax, not literal text; search for it literally by double-quoting: '\"${suspects[0]}\"'. Searchable columns are title, summary, text.`);\n  }\n  if (col !== undefined && scope !== undefined) {\n    return new SenseError('SEARCH_SYNTAX', `${message} -- the where condition (${scope}) references it; frontmatter columns are listed by sense sql \"SELECT name FROM pragma_table_info('frontmatter')\".`);\n  }\n  return new SenseError('SEARCH_SYNTAX', `${message} -- searchable columns are title, summary, text; frontmatter fields are queried with --where or sense sql (list them with pragma_table_info('frontmatter')).`);\n}\n"],"names":["searchError","FTS5_PUNCTUATION","err","terms","scope","message","test","suspects","match","filter","t","startsWith","col","exec","fromTerms","undefined","length","some","split","includes","SenseError","map","join"],"mappings":";;;;+BAQgBA;;;eAAAA;;;wBARW;AAE3B,0FAA0F;AAC1F,8FAA8F;AAC9F,8FAA8F;AAC9F,6BAA6B;AAC7B,IAAMC,mBAAmB;AAElB,SAASD,YAAYE,GAAU,EAAEC,KAAa,EAAEC,KAAc;QAGjDD;QAIN;IANZ,IAAME,UAAUH,IAAIG,OAAO;IAC3B,IAAI,CAAC,oDAAoDC,IAAI,CAACD,UAAU,OAAOH;IAC/E,IAAMK,WAAW,EAACJ,eAAAA,MAAMK,KAAK,CAAC,qBAAZL,0BAAAA,eAAuB,EAAE,EAAEM,MAAM,CAAC,SAACC;eAAM,CAACA,EAAEC,UAAU,CAAC,QAAQV,iBAAiBK,IAAI,CAACI;;IACvG,wFAAwF;IACxF,qFAAqF;IACrF,gFAAgF;IAChF,IAAME,OAAM,QAAA,wBAAwBC,IAAI,CAACR,sBAA7B,4BAAA,KAAuC,CAAC,EAAE;IACtD,IAAMS,YAAYF,QAAQG,YAAYR,SAASS,MAAM,GAAG,IAAIT,SAASU,IAAI,CAAC,SAACP;eAAMA,EAAEQ,KAAK,CAAC,8maAAmBC,QAAQ,CAACP;;IACrH,IAAIE,aAAaP,SAASS,MAAM,GAAG,GAAG;QACpC,OAAO,IAAII,oBAAU,CAAC,iBAAiB,AAAC,GAAmCb,OAAjCF,SAAQ,2BAAsJE,OAA7HA,SAASc,GAAG,CAAC,SAACX;mBAAM,AAAC,IAAM,OAAFA,GAAE;WAAKY,IAAI,CAAC,OAAM,qFAA8F,OAAZf,QAAQ,CAAC,EAAE,EAAC;IACtN;IACA,IAAIK,QAAQG,aAAaX,UAAUW,WAAW;QAC5C,OAAO,IAAIK,oBAAU,CAAC,iBAAiB,AAAC,GAAqChB,OAAnCC,SAAQ,6BAAiC,OAAND,OAAM;IACrF;IACA,OAAO,IAAIgB,oBAAU,CAAC,iBAAiB,AAAC,GAAU,OAARf,SAAQ;AACpD"}

package/dist/cjs/scan/frontmatter.d.cts

@@ -0,0 +1,13 @@
+export declare const RESERVED_COLUMNS: Set<string>;
+export declare function normalizeText(value: unknown): string;
+export declare function looksLikeDatetime(value: string): boolean;
+export declare function normalizeDate(value: string): string;
+export declare function mapValue(value: unknown): string | number | bigint | null;
+export declare function splitFrontmatter(raw: string): {
+    fm: string | null;
+    body: string;
+};
+export declare function parseFrontmatter(relPath: string, fm: string, warnings: string[]): {
+    data: Record<string, unknown>;
+    parseError: string | null;
+};

package/dist/cjs/scan/frontmatter.d.ts

@@ -0,0 +1,13 @@
+export declare const RESERVED_COLUMNS: Set<string>;
+export declare function normalizeText(value: unknown): string;
+export declare function looksLikeDatetime(value: string): boolean;
+export declare function normalizeDate(value: string): string;
+export declare function mapValue(value: unknown): string | number | bigint | null;
+export declare function splitFrontmatter(raw: string): {
+    fm: string | null;
+    body: string;
+};
+export declare function parseFrontmatter(relPath: string, fm: string, warnings: string[]): {
+    data: Record<string, unknown>;
+    parseError: string | null;
+};

package/dist/cjs/scan/frontmatter.js

@@ -0,0 +1,229 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get RESERVED_COLUMNS () {
+        return RESERVED_COLUMNS;
+    },
+    get looksLikeDatetime () {
+        return looksLikeDatetime;
+    },
+    get mapValue () {
+        return mapValue;
+    },
+    get normalizeDate () {
+        return normalizeDate;
+    },
+    get normalizeText () {
+        return normalizeText;
+    },
+    get parseFrontmatter () {
+        return parseFrontmatter;
+    },
+    get splitFrontmatter () {
+        return splitFrontmatter;
+    }
+});
+var _yaml = require("yaml");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
+function _type_of(obj) {
+    "@swc/helpers - typeof";
+    return obj && typeof Symbol !== "undefined" && obj.constructor === Symbol ? "symbol" : typeof obj;
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var RESERVED_COLUMNS = new Set([
+    'path',
+    '_mtime',
+    '_ctime',
+    '_size',
+    '_rank',
+    '_parse_error',
+    'content',
+    'links',
+    'sections'
+]);
+// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than
+// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain
+// scalar for future use, so they can never be valid and the text can only be what was typed
+// (`aliases: [@handle]` -> ["@handle"]). Every other code has a second reading -- an unquoted
+// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks
+// one value in silence -- so it writes values nobody wrote.
+var ACCEPTED_YAML_CODES = new Set([
+    'BAD_SCALAR_START'
+]);
+function normalizeText(value) {
+    if (value === null || value === undefined) return '';
+    return String(value).replace(/\s+/g, ' ').trim();
+}
+// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601
+// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.
+var ISO_DATETIME = /^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?)(Z|[+-]\d{2}(?::?\d{2})?)?$/;
+// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it
+// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.
+var MEANT_AS_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d/;
+function looksLikeDatetime(value) {
+    return MEANT_AS_DATETIME.test(value);
+}
+function normalizeDate(value) {
+    var m = ISO_DATETIME.exec(value);
+    if (m === null) return value;
+    var _m = _sliced_to_array(m, 4), date = _m[1], time = _m[2], zone = _m[3];
+    var digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');
+    var offset = digits === '' ? zone !== null && zone !== void 0 ? zone : '' : digits.length === 3 ? "".concat(digits, ":00") : "".concat(digits.slice(0, 3), ":").concat(digits.slice(3));
+    var normalized = "".concat(date, "T").concat(time).concat(offset);
+    return Number.isNaN(Date.parse(normalized)) ? value : normalized;
+}
+function mapValue(value) {
+    if (value === null || value === undefined) return null;
+    if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
+    if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;
+    if (typeof value === 'string') return normalizeDate(value);
+    return JSON.stringify(value);
+}
+function splitFrontmatter(raw) {
+    var open = raw.match(/^---\r?\n/);
+    if (!open) return {
+        fm: null,
+        body: raw
+    };
+    var rest = raw.slice(open[0].length);
+    var close = rest.match(/^---\r?(\n|$)/m);
+    if (!close || close.index === undefined) return {
+        fm: null,
+        body: raw
+    };
+    return {
+        fm: rest.slice(0, close.index),
+        body: rest.slice(close.index + close[0].length)
+    };
+}
+// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid
+// YAML for a flow map used as a mapping key, so it raises no error and stores
+// {"{ date }": null}. No error code can catch that, but yaml notices the stringified key, so
+// this reports it with the path instead (yaml's own warning has none, fires once per document,
+// and is what trains readers to discard stderr).
+function warnStringifiedKeys(relPath, doc, warnings) {
+    var found = false;
+    // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside
+    // the flow map that `{{...}}` parses as.
+    (0, _yaml.visit)(doc, {
+        Pair: function Pair(_key, pair) {
+            if (!(0, _yaml.isCollection)(pair.key)) return undefined;
+            found = true;
+            return _yaml.visit.BREAK;
+        }
+    });
+    // One per file: a template repeats the same mistake on every field it stamps.
+    if (found) warnings.push("warning: ".concat(relPath, " frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}"));
+}
+// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).
+// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the
+// reason. Recovering it would write values nobody wrote, which is worse than absence because
+// no query can see it. The file is still indexed -- content, links and sections never touch
+// frontmatter -- so a broken note stays searchable while it is being hunted for.
+// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus
+// the colon that introduced the part being dropped.
+function firstLine(message) {
+    return message.split('\n')[0].replace(/:\s*$/, '');
+}
+function parseFrontmatter(relPath, fm, warnings) {
+    // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one
+    // that carries information, with the file it came from.
+    var doc = (0, _yaml.parseDocument)(fm, {
+        logLevel: 'silent'
+    });
+    var refused = doc.errors.filter(function(err) {
+        return !ACCEPTED_YAML_CODES.has(err.code);
+    });
+    if (refused.length > 0) {
+        var detail = refused.length > 1 ? " (and ".concat(refused.length - 1, " more)") : '';
+        var parseError = "".concat(firstLine(refused[0].message)).concat(detail);
+        warnings.push("warning: ".concat(relPath, " frontmatter did not parse, so none of it is indexed: ").concat(parseError));
+        return {
+            data: {},
+            parseError: parseError
+        };
+    }
+    var data;
+    try {
+        data = doc.toJS();
+    } catch (err) {
+        // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on
+        // materialisation. An empty error list is not a successful parse.
+        var parseError1 = firstLine(err.message);
+        warnings.push("warning: ".concat(relPath, " frontmatter did not parse, so none of it is indexed: ").concat(parseError1));
+        return {
+            data: {},
+            parseError: parseError1
+        };
+    }
+    if (data === null || data === undefined) return {
+        data: {},
+        parseError: null
+    };
+    if ((typeof data === "undefined" ? "undefined" : _type_of(data)) !== 'object' || Array.isArray(data)) {
+        var parseError2 = 'frontmatter is not a key-value mapping';
+        warnings.push("warning: ".concat(relPath, " ").concat(parseError2, "; none of it is indexed"));
+        return {
+            data: {},
+            parseError: parseError2
+        };
+    }
+    warnStringifiedKeys(relPath, doc, warnings);
+    return {
+        data: data,
+        parseError: null
+    };
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/scan/frontmatter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/frontmatter.ts"],"sourcesContent":["import { isCollection, parseDocument, visit } from 'yaml';\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nexport function normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nexport function mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nexport function splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nexport function parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n"],"names":["RESERVED_COLUMNS","looksLikeDatetime","mapValue","normalizeDate","normalizeText","parseFrontmatter","splitFrontmatter","Set","ACCEPTED_YAML_CODES","value","undefined","String","replace","trim","ISO_DATETIME","MEANT_AS_DATETIME","test","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","BigInt","isSafeInteger","JSON","stringify","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","relPath","doc","warnings","found","visit","Pair","_key","pair","isCollection","key","BREAK","push","firstLine","message","split","parseDocument","logLevel","refused","errors","filter","err","has","code","detail","parseError","data","toJS","Array","isArray"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QAuBGC;eAAAA;;QAkBAC;eAAAA;;QAZAC;eAAAA;;QAnBAC;eAAAA;;QAgFAC;eAAAA;;QAxCAC;eAAAA;;;oBAtDmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAI5C,IAAMN,mBAAmB,IAAIO,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW;AAEtI,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4DAA4D;AAC5D,IAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAEjD,SAASH,cAAcK,KAAc;IAC1C,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,iGAAiG;AACjG,mGAAmG;AACnG,IAAMC,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,oBAAoB;AAEnB,SAASd,kBAAkBQ,KAAa;IAC7C,OAAOM,kBAAkBC,IAAI,CAACP;AAChC;AAIO,SAASN,cAAcM,KAAa;IACzC,IAAMQ,IAAIH,aAAaI,IAAI,CAACT;IAC5B,IAAIQ,MAAM,MAAM,OAAOR;IACvB,IAA6BQ,sBAAAA,OAApBE,OAAoBF,OAAdG,OAAcH,OAARI,OAAQJ;IAC7B,IAAMK,SAASD,SAASX,aAAaW,SAAS,MAAM,KAAKA,KAAKT,OAAO,CAAC,KAAK;IAC3E,IAAMW,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,AAAC,GAAS,OAAPF,QAAO,SAAO,AAAC,GAAwBA,OAAtBA,OAAOG,KAAK,CAAC,GAAG,IAAG,KAAmB,OAAhBH,OAAOG,KAAK,CAAC;IAC3H,IAAMC,aAAa,AAAC,GAAUN,OAARD,MAAK,KAAUI,OAAPH,MAAc,OAAPG;IACrC,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAejB,QAAQiB;AACxD;AAIO,SAASxB,SAASO,KAAc;IACrC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOsB,OAAOtB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOkB,OAAOK,aAAa,CAACvB,SAASsB,OAAOtB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAON,cAAcM;IACpD,OAAOwB,KAAKC,SAAS,CAACzB;AACxB;AAGO,SAASH,iBAAiB6B,GAAW;IAC1C,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIV,KAAK,CAACW,IAAI,CAAC,EAAE,CAACZ,MAAM;IACrC,IAAMiB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKhC,WAAW,OAAO;QAAE4B,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKf,KAAK,CAAC,GAAGgB,MAAMC,KAAK;QAAGH,MAAMC,KAAKf,KAAK,CAACgB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACjB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASmB,oBAAoBC,OAAe,EAAEC,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCC,IAAAA,WAAK,EAACH,KAAK;QACTI,MAAAA,SAAAA,KAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACC,IAAAA,kBAAY,EAACD,KAAKE,GAAG,GAAG,OAAO3C;YACpCqC,QAAQ;YACR,OAAOC,WAAK,CAACM,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIP,OAAOD,SAASS,IAAI,CAAC,AAAC,YAAmB,OAARX,SAAQ;AAC/C;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASY,UAAUC,OAAe;IAChC,OAAOA,QAAQC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC9C,OAAO,CAAC,SAAS;AACjD;AAEO,SAASP,iBAAiBuC,OAAe,EAAEN,EAAU,EAAEQ,QAAkB;IAC9E,yFAAyF;IACzF,wDAAwD;IACxD,IAAMD,MAAMc,IAAAA,mBAAa,EAACrB,IAAI;QAAEsB,UAAU;IAAS;IACnD,IAAMC,UAAUhB,IAAIiB,MAAM,CAACC,MAAM,CAAC,SAACC;eAAQ,CAACxD,oBAAoByD,GAAG,CAACD,IAAIE,IAAI;;IAC5E,IAAIL,QAAQrC,MAAM,GAAG,GAAG;QACtB,IAAM2C,SAASN,QAAQrC,MAAM,GAAG,IAAI,AAAC,SAA2B,OAAnBqC,QAAQrC,MAAM,GAAG,GAAE,YAAU;QAC1E,IAAM4C,aAAa,AAAC,GAAkCD,OAAhCX,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,GAAW,OAAPU;QACtDrB,SAASS,IAAI,CAAC,AAAC,YAA2Ea,OAAhExB,SAAQ,0DAAmE,OAAXwB;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOxB,IAAIyB,IAAI;IACjB,EAAE,OAAON,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,IAAMI,cAAaZ,UAAU,AAACQ,IAAcP,OAAO;QACnDX,SAASS,IAAI,CAAC,AAAC,YAA2Ea,OAAhExB,SAAQ,0DAAmE,OAAXwB;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS3D,WAAW,OAAO;QAAE2D,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,CAAA,OAAOC,qCAAP,SAAOA,KAAG,MAAM,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,IAAMD,cAAa;QACnBtB,SAASS,IAAI,CAAC,AAAC,YAAsBa,OAAXxB,SAAQ,KAAc,OAAXwB,aAAW;QAChD,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IACAzB,oBAAoBC,SAASC,KAAKC;IAClC,OAAO;QAAEuB,MAAMA;QAAiCD,YAAY;IAAK;AACnE"}

package/dist/cjs/scan/index.d.cts

@@ -0,0 +1,25 @@
+import type { Config } from '../config/index.js';
+import type { Feature } from '../features/types.js';
+import type { FileStat } from './list.js';
+export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
+export type { FileStat } from './list.js';
+export { listFiles, toPosixPath } from './list.js';
+export interface ParsedDoc {
+    relPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    data: Record<string, string | number | bigint | null>;
+    parseError: string | null;
+    search: {
+        title: string;
+        summary: string;
+        text: string;
+    };
+    extracted: Record<string, unknown>;
+}
+export declare function parseFile(file: FileStat, extractors?: Feature[], cfg?: Config): {
+    doc: ParsedDoc;
+    warnings: string[];
+};

package/dist/cjs/scan/index.d.ts

@@ -0,0 +1,25 @@
+import type { Config } from '../config/index.js';
+import type { Feature } from '../features/types.js';
+import type { FileStat } from './list.js';
+export { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.js';
+export type { FileStat } from './list.js';
+export { listFiles, toPosixPath } from './list.js';
+export interface ParsedDoc {
+    relPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    data: Record<string, string | number | bigint | null>;
+    parseError: string | null;
+    search: {
+        title: string;
+        summary: string;
+        text: string;
+    };
+    extracted: Record<string, unknown>;
+}
+export declare function parseFile(file: FileStat, extractors?: Feature[], cfg?: Config): {
+    doc: ParsedDoc;
+    warnings: string[];
+};

package/dist/cjs/scan/index.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get RESERVED_COLUMNS () {
+        return _frontmatterts.RESERVED_COLUMNS;
+    },
+    get listFiles () {
+        return _listts.listFiles;
+    },
+    get looksLikeDatetime () {
+        return _frontmatterts.looksLikeDatetime;
+    },
+    get normalizeDate () {
+        return _frontmatterts.normalizeDate;
+    },
+    get parseFile () {
+        return parseFile;
+    },
+    get toPosixPath () {
+        return _listts.toPosixPath;
+    }
+});
+var _nodefs = require("node:fs");
+var _indexts = require("../chunk/index.js");
+var _stripts = require("../text/strip.js");
+var _frontmatterts = require("./frontmatter.js");
+var _listts = require("./list.js");
+function parseFile(file) {
+    var extractors = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : [], cfg = arguments.length > 2 ? arguments[2] : void 0;
+    var raw = (0, _nodefs.readFileSync)(file.absPath, 'utf8');
+    var warnings = [];
+    var _splitFrontmatter = (0, _frontmatterts.splitFrontmatter)(raw), fm = _splitFrontmatter.fm, content = _splitFrontmatter.body;
+    var _ref = fm === null ? {
+        data: {},
+        parseError: null
+    } : (0, _frontmatterts.parseFrontmatter)(file.relPath, fm, warnings), data = _ref.data, parseError = _ref.parseError;
+    var mapped = {};
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = Object.keys(data)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var key = _step.value;
+            if (_frontmatterts.RESERVED_COLUMNS.has(key)) {
+                warnings.push("warning: ".concat(file.relPath, ' has a frontmatter key named "').concat(key, '", which is reserved; ignoring it'));
+                continue;
+            }
+            var value = (0, _frontmatterts.mapValue)(data[key]);
+            if (typeof value === 'string' && (0, _frontmatterts.looksLikeDatetime)(value) && Number.isNaN(Date.parse(value))) {
+                warnings.push("warning: ".concat(file.relPath, ": ").concat(key, " is not a valid date (").concat(value, "), so it is invisible to every date comparison"));
+            }
+            mapped[key] = value;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature
+    // that needs a parse tree (embed's chunker) both read this same block list.
+    var blocks = (0, _indexts.parse)(content);
+    // title/summary are plain YAML strings -- whitespace-collapse only;
+    // the prose gets the full markdown strip.
+    var search = {
+        title: (0, _frontmatterts.normalizeText)(data.title),
+        summary: (0, _frontmatterts.normalizeText)(data.summary),
+        text: (0, _stripts.textFromBlocks)(blocks)
+    };
+    return {
+        doc: {
+            relPath: file.relPath,
+            mtimeMs: file.mtimeMs,
+            ctimeMs: file.ctimeMs,
+            size: file.size,
+            presets: file.presets,
+            data: mapped,
+            parseError: parseError,
+            search: search,
+            extracted: Object.fromEntries(extractors.filter(function(f) {
+                return f.extract;
+            }).map(function(f) {
+                var _f_extract;
+                return [
+                    f.name,
+                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search, data, cfg, blocks)
+                ];
+            }))
+        },
+        warnings: warnings
+    };
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/scan/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","looksLikeDatetime","normalizeDate","parseFile","toPosixPath","file","extractors","cfg","raw","readFileSync","absPath","warnings","splitFrontmatter","fm","body","content","data","parseError","parseFrontmatter","relPath","mapped","Object","keys","key","has","push","value","mapValue","Number","isNaN","Date","parse","blocks","search","title","normalizeText","summary","text","textFromBlocks","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":";;;;;;;;;;;QAU2CA;eAAAA,+BAAgB;;QAElDC;eAAAA,iBAAS;;QAFTC;eAAAA,gCAAiB;;QAAEC;eAAAA,4BAAa;;QAoBzBC;eAAAA;;QAlBIC;eAAAA,mBAAW;;;sBAZF;uBACP;uBAGS;6BACkF;sBAO1E;AAkBhC,SAASD,UAAUE,IAAc;QAAEC,aAAAA,iEAAwB,EAAE,EAAEC;IACpE,IAAMC,MAAMC,IAAAA,oBAAY,EAACJ,KAAKK,OAAO,EAAE;IACvC,IAAMC,WAAqB,EAAE;IAE7B,IAA8BC,oBAAAA,IAAAA,+BAAgB,EAACJ,MAAvCK,KAAsBD,kBAAtBC,IAAIC,AAAMC,UAAYH,kBAAlBE;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIC,IAAAA,+BAAgB,EAACb,KAAKc,OAAO,EAAEN,IAAIF,WAAlIK,OAAqBH,KAArBG,MAAMC,aAAeJ,KAAfI;IACd,IAAMG,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAOC,IAAI,CAACN,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMO,MAAN;YACH,IAAIxB,+BAAgB,CAACyB,GAAG,CAACD,MAAM;gBAC7BZ,SAASc,IAAI,CAAC,AAAC,YAAwDF,OAA7ClB,KAAKc,OAAO,EAAC,kCAAoC,OAAJI,KAAI;gBAC3E;YACF;YACA,IAAMG,QAAQC,IAAAA,uBAAQ,EAACX,IAAI,CAACO,IAAI;YAChC,IAAI,OAAOG,UAAU,YAAYzB,IAAAA,gCAAiB,EAACyB,UAAUE,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACL,SAAS;gBAC5Ff,SAASc,IAAI,CAAC,AAAC,YAA4BF,OAAjBlB,KAAKc,OAAO,EAAC,MAAgCO,OAA5BH,KAAI,0BAA8B,OAANG,OAAM;YAC/E;YACAN,MAAM,CAACG,IAAI,GAAGG;QAChB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,yFAAyF;IACzF,4EAA4E;IAC5E,IAAMM,SAASD,IAAAA,cAAK,EAAChB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMkB,SAAS;QAAEC,OAAOC,IAAAA,4BAAa,EAACnB,KAAKkB,KAAK;QAAGE,SAASD,IAAAA,4BAAa,EAACnB,KAAKoB,OAAO;QAAGC,MAAMC,IAAAA,uBAAc,EAACN;IAAQ;IAEtH,OAAO;QACLO,KAAK;YACHpB,SAASd,KAAKc,OAAO;YACrBqB,SAASnC,KAAKmC,OAAO;YACrBC,SAASpC,KAAKoC,OAAO;YACrBC,MAAMrC,KAAKqC,IAAI;YACfC,SAAStC,KAAKsC,OAAO;YACrB3B,MAAMI;YACNH,YAAAA;YACAgB,QAAAA;YACAW,WAAWvB,OAAOwB,WAAW,CAACvC,WAAWwC,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYvC,KAAKO,SAASkB,QAAQjB,MAAMT,KAAKyB;iBAAQ;;QAC7I;QACArB,UAAAA;IACF;AACF"}

package/dist/cjs/scan/list.d.cts

@@ -0,0 +1,12 @@
+import type { Config } from '../config/index.js';
+export interface FileStat {
+    relPath: string;
+    absPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    embed: boolean;
+}
+export declare function toPosixPath(relPath: string, separator?: string): string;
+export declare function listFiles(cfg: Config, baseDir: string): FileStat[];

package/dist/cjs/scan/list.d.ts

@@ -0,0 +1,12 @@
+import type { Config } from '../config/index.js';
+export interface FileStat {
+    relPath: string;
+    absPath: string;
+    mtimeMs: number;
+    ctimeMs: number;
+    size: number;
+    presets: string[];
+    embed: boolean;
+}
+export declare function toPosixPath(relPath: string, separator?: string): string;
+export declare function listFiles(cfg: Config, baseDir: string): FileStat[];