sensemaking 0.19.2 → 0.21.0

package/schema.json

@@ -101,17 +101,6 @@
         "chunkTokens": { "type": "integer", "minimum": 1, "description": "Chunk size ceiling in estimated tokens for small-context models; default 500." }
       }
     },
-    "content": {
-      "type": "object",
-      "additionalProperties": false,
-      "description": "Settings for the `content` FTS5 table. Changing anything here rebuilds the content table: FTS5 cannot alter a tokenizer in place, and the prose is stored nowhere else, so every file's text is re-read. Vectors, links, and sections are file-derived and tokenizer-independent, so they are kept. The rebuild announces itself on stderr, naming the content tokenizer as the cause.",
-      "properties": {
-        "tokenize": {
-          "type": "string",
-          "description": "FTS5 tokenizer for word search, defaulting to `porter unicode61`. Languages written without word spaces (Chinese, Japanese, Thai, Khmer, Lao, Burmese) need no decision here: their text is indexed per grapheme and searched as an ordered grapheme phrase against machine-written sidecar columns, which is substring semantics -- what grep gives, a query matches wherever its exact text occurs, including inside a longer run, and needs no minimum length. What this setting is for is substring matching inside a Latin word, `trigram`, which indexes every three-character window, at the cost of stemming (`running` stops matching `run`), weaker bm25 relevance because trigrams are not words, a larger index, and a floor of three characters. `unicode61 tokenchars '-_'` is the smaller lever, keeping hyphenated terms whole. Naming any tokenizer turns the grapheme-phrase scheme off, on the reasoning that the tree has chosen its own scheme. Built-in choices are unicode61, ascii, porter, and trigram, each with their own options; the value is validated by building a throwaway table with it, so whatever the linked SQLite accepts is accepted here and anything else errors carrying SQLite's own message."
-        }
-      }
-    },
     "store": {
       "type": "string",
       "enum": ["sqlite", "duckdb", "turso"],

package/dist/cjs/store/sqlite/signature.js

@@ -1,380 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", {
-    value: true
-});
-function _export(target, all) {
-    for(var name in all)Object.defineProperty(target, name, {
-        enumerable: true,
-        get: Object.getOwnPropertyDescriptor(all, name).get
-    });
-}
-_export(exports, {
-    get changedSignatureKeys () {
-        return changedSignatureKeys;
-    },
-    get embedIdentityAdopted () {
-        return embedIdentityAdopted;
-    },
-    get rebuildContentTable () {
-        return rebuildContentTable;
-    },
-    get signatureDiff () {
-        return signatureDiff;
-    }
-});
-var _indexts = require("../../config/index.js");
-var _indexts1 = require("../../scan/index.js");
-var _transactionts = require("../transaction.js");
-var _reconcilets = require("./reconcile.js");
-function _array_like_to_array(arr, len) {
-    if (len == null || len > arr.length) len = arr.length;
-    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
-    return arr2;
-}
-function _array_with_holes(arr) {
-    if (Array.isArray(arr)) return arr;
-}
-function _array_without_holes(arr) {
-    if (Array.isArray(arr)) return _array_like_to_array(arr);
-}
-function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
-    try {
-        var info = gen[key](arg);
-        var value = info.value;
-    } catch (error) {
-        reject(error);
-        return;
-    }
-    if (info.done) resolve(value);
-    else Promise.resolve(value).then(_next, _throw);
-}
-function _async_to_generator(fn) {
-    return function() {
-        var self = this, args = arguments;
-        return new Promise(function(resolve, reject) {
-            var gen = fn.apply(self, args);
-            function _next(value) {
-                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
-            }
-            function _throw(err) {
-                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
-            }
-            _next(undefined);
-        });
-    };
-}
-function _iterable_to_array(iter) {
-    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
-        return Array.from(iter);
-    }
-}
-function _iterable_to_array_limit(arr, i) {
-    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
-    if (_i == null) return;
-    var _arr = [];
-    var _n = true;
-    var _d = false;
-    var _s, _e;
-    try {
-        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
-            _arr.push(_s.value);
-            if (i && _arr.length === i) break;
-        }
-    } catch (err) {
-        _d = true;
-        _e = err;
-    } finally{
-        try {
-            if (!_n && _i["return"] != null) _i["return"]();
-        } finally{
-            if (_d) throw _e;
-        }
-    }
-    return _arr;
-}
-function _non_iterable_rest() {
-    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
-}
-function _non_iterable_spread() {
-    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
-}
-function _sliced_to_array(arr, i) {
-    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
-}
-function _to_consumable_array(arr) {
-    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
-}
-function _ts_generator(thisArg, body) {
-    var f, y, t, _ = {
-        label: 0,
-        sent: function() {
-            if (t[0] & 1) throw t[1];
-            return t[1];
-        },
-        trys: [],
-        ops: []
-    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
-    return d(g, "next", {
-        value: verb(0)
-    }), d(g, "throw", {
-        value: verb(1)
-    }), d(g, "return", {
-        value: verb(2)
-    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
-        value: function() {
-            return this;
-        }
-    }), g;
-    function verb(n) {
-        return function(v) {
-            return step([
-                n,
-                v
-            ]);
-        };
-    }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while(g && (g = 0, op[0] && (_ = 0)), _)try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [
-                op[0] & 2,
-                t.value
-            ];
-            switch(op[0]){
-                case 0:
-                case 1:
-                    t = op;
-                    break;
-                case 4:
-                    _.label++;
-                    return {
-                        value: op[1],
-                        done: false
-                    };
-                case 5:
-                    _.label++;
-                    y = op[1];
-                    op = [
-                        0
-                    ];
-                    continue;
-                case 7:
-                    op = _.ops.pop();
-                    _.trys.pop();
-                    continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
-                        _ = 0;
-                        continue;
-                    }
-                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
-                        _.label = op[1];
-                        break;
-                    }
-                    if (op[0] === 6 && _.label < t[1]) {
-                        _.label = t[1];
-                        t = op;
-                        break;
-                    }
-                    if (t && _.label < t[2]) {
-                        _.label = t[2];
-                        _.ops.push(op);
-                        break;
-                    }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop();
-                    continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) {
-            op = [
-                6,
-                e
-            ];
-            y = 0;
-        } finally{
-            f = t = 0;
-        }
-        if (op[0] & 5) throw op[1];
-        return {
-            value: op[0] ? op[1] : void 0,
-            done: true
-        };
-    }
-}
-function _unsupported_iterable_to_array(o, minLen) {
-    if (!o) return;
-    if (typeof o === "string") return _array_like_to_array(o, minLen);
-    var n = Object.prototype.toString.call(o).slice(8, -1);
-    if (n === "Object" && o.constructor) n = o.constructor.name;
-    if (n === "Map" || n === "Set") return Array.from(n);
-    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
-}
-function changedSignatureKeys(before, after) {
-    var keyOf = function keyOf(part) {
-        return part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
-    };
-    var parse = function parse(sig) {
-        return new Map(sig.split('|').map(function(part) {
-            return [
-                keyOf(part),
-                part
-            ];
-        }));
-    };
-    var a = parse(before);
-    var b = parse(after);
-    var changed = new Set();
-    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-    try {
-        for(var _iterator = b[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-            var _step_value = _sliced_to_array(_step.value, 2), key = _step_value[0], val = _step_value[1];
-            if (a.get(key) !== val) changed.add(key);
-        }
-    } catch (err) {
-        _didIteratorError = true;
-        _iteratorError = err;
-    } finally{
-        try {
-            if (!_iteratorNormalCompletion && _iterator.return != null) {
-                _iterator.return();
-            }
-        } finally{
-            if (_didIteratorError) {
-                throw _iteratorError;
-            }
-        }
-    }
-    var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
-    try {
-        for(var _iterator1 = a.keys()[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
-            var key1 = _step1.value;
-            if (!b.has(key1)) changed.add(key1);
-        }
-    } catch (err) {
-        _didIteratorError1 = true;
-        _iteratorError1 = err;
-    } finally{
-        try {
-            if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
-                _iterator1.return();
-            }
-        } finally{
-            if (_didIteratorError1) {
-                throw _iteratorError1;
-            }
-        }
-    }
-    return changed;
-}
-function embedIdentityAdopted(before, after) {
-    var embedPart = function embedPart(sig) {
-        return sig.split('|').find(function(p) {
-            return p.startsWith('embed:');
-        });
-    };
-    var b = embedPart(before);
-    var a = embedPart(after);
-    if (b === undefined || a === undefined) return false;
-    var at = a.indexOf('@');
-    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
-}
-function signatureDiff(before, after) {
-    var changed = changedSignatureKeys(before, after);
-    var label = function label(key) {
-        return key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? 'preset "'.concat(key.slice(7), '"') : 'features';
-    };
-    return changed.size === 0 ? 'features' : _to_consumable_array(changed).map(label).join(', ');
-}
-function rebuildContentTable(conn, cfg, baseDir) {
-    return _async_to_generator(function() {
-        var stmt, known, _, files, segmenting, warnings, rows, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, _warnings, _parseFile, doc, fileWarnings;
-        return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    return [
-                        4,
-                        conn.prepare('SELECT "path" FROM frontmatter')
-                    ];
-                case 1:
-                    stmt = _state.sent();
-                    _ = Set.bind;
-                    return [
-                        4,
-                        stmt.all()
-                    ];
-                case 2:
-                    known = new (_.apply(Set, [
-                        void 0,
-                        _state.sent().map(function(r) {
-                            return r.path;
-                        })
-                    ]));
-                    files = (0, _indexts1.listFiles)(cfg, baseDir).filter(function(f) {
-                        return known.has(f.relPath);
-                    });
-                    segmenting = (0, _indexts.contentTokenize)(cfg) === undefined;
-                    warnings = [];
-                    rows = [];
-                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                    try {
-                        for(_iterator = files[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                            file = _step.value;
-                            ;
-                            _parseFile = (0, _indexts1.parseFile)(file), doc = _parseFile.doc, fileWarnings = _parseFile.warnings;
-                            (_warnings = warnings).push.apply(_warnings, _to_consumable_array(fileWarnings));
-                            rows.push((0, _reconcilets.contentRow)(doc, segmenting));
-                        }
-                    } catch (err) {
-                        _didIteratorError = true;
-                        _iteratorError = err;
-                    } finally{
-                        try {
-                            if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                _iterator.return();
-                            }
-                        } finally{
-                            if (_didIteratorError) {
-                                throw _iteratorError;
-                            }
-                        }
-                    }
-                    return [
-                        4,
-                        (0, _transactionts.withTransaction)(conn, function() {
-                            return _async_to_generator(function() {
-                                return _ts_generator(this, function(_state) {
-                                    switch(_state.label){
-                                        case 0:
-                                            if (!(rows.length > 0)) return [
-                                                3,
-                                                2
-                                            ];
-                                            return [
-                                                4,
-                                                conn.runBatch(_reconcilets.INSERT_CONTENT_SQL, rows)
-                                            ];
-                                        case 1:
-                                            _state.sent();
-                                            _state.label = 2;
-                                        case 2:
-                                            return [
-                                                2
-                                            ];
-                                    }
-                                });
-                            })();
-                        }, _transactionts.BEGIN_WRITE)
-                    ];
-                case 3:
-                    _state.sent();
-                    return [
-                        2,
-                        warnings
-                    ];
-            }
-        });
-    })();
-}
-/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/sqlite/signature.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/signature.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { listFiles, parseFile } from '../../scan/index.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\nimport { contentRow, INSERT_CONTENT_SQL } from './reconcile.ts';\n\n// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:\n// global features, embed provider, tokenize, then one segment per preset). Never called from\n// reconcile() itself.\n\n// Segment keys that moved between two feature signatures.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","signatureDiff","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedPart","find","p","undefined","at","indexOf","label","size","conn","cfg","baseDir","stmt","known","files","segmenting","warnings","rows","file","parseFile","doc","fileWarnings","prepare","all","r","path","listFiles","filter","f","relPath","contentTokenize","push","contentRow","withTransaction","length","runBatch","INSERT_CONTENT_SQL","BEGIN_WRITE"],"mappings":";;;;;;;;;;;QAYgBA;eAAAA;;QAaAC;eAAAA;;QAkBMC;eAAAA;;QARNC;eAAAA;;;uBAlCgB;wBACK;6BACQ;2BAEE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOxC,SAASH,qBAAqBI,MAAc,EAAEC,KAAa;IAChE,IAAMC,QAAQ,eAACC;eAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAMG,QAAQ,eAACC;eAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,SAACR;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMS,IAAIJ,MAAMR;IAChB,IAAMa,IAAIL,MAAMP;IAChB,IAAMa,UAAU,IAAIC;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBF,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOG,sBAAKC;YAAW,IAAIL,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAaJ,EAAEQ,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMJ,OAAN;YAAuB,IAAI,CAACH,EAAEQ,GAAG,CAACL,OAAMF,QAAQK,GAAG,CAACH;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOF;AACT;AAIO,SAASjB,qBAAqBG,MAAc,EAAEC,KAAa;IAChE,IAAMqB,YAAY,mBAACb;eAAgBA,IAAIJ,KAAK,CAAC,KAAKkB,IAAI,CAAC,SAACC;mBAAMA,EAAEpB,UAAU,CAAC;;;IAC3E,IAAMS,IAAIS,UAAUtB;IACpB,IAAMY,IAAIU,UAAUrB;IACpB,IAAIY,MAAMY,aAAab,MAAMa,WAAW,OAAO;IAC/C,IAAMC,KAAKd,EAAEe,OAAO,CAAC;IACrB,OAAOd,EAAEc,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKd,EAAEN,KAAK,CAAC,GAAGoB,QAAQb;AAClE;AAGO,SAASd,cAAcC,MAAc,EAAEC,KAAa;IACzD,IAAMa,UAAUlB,qBAAqBI,QAAQC;IAC7C,IAAM2B,QAAQ,eAACZ;eAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIZ,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbY,IAAIV,KAAK,CAAC,IAAG,OAAK;;IACzK,OAAOQ,QAAQe,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAGf,SAASH,GAAG,CAACiB,OAAOrB,IAAI,CAAC;AACxE;AAIO,SAAeT,oBAAoBgC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YAChFC,MACAC,UACAC,OACAC,YACAC,UACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAETF,WADwCG,YAAhCC,KAAeC;;;;oBAPZ;;wBAAMZ,KAAKa,OAAO,CAAC;;;oBAA1BV,OAAO;wBACKlB;oBAAM;;wBAAMkB,KAAKW,GAAG;;;oBAAhCV,QAAQ,IAAA,CAAA,EAAA,MAAInB;;wBAAM,cAA8CJ,GAAG,CAAC,SAACkC;mCAAMA,EAAEC,IAAI;;sBAAC;oBAClFX,QAAQY,IAAAA,mBAAS,EAAChB,KAAKC,SAASgB,MAAM,CAAC,SAACC;+BAAMf,MAAMb,GAAG,CAAC4B,EAAEC,OAAO;;oBACjEd,aAAae,IAAAA,wBAAe,EAACpB,SAASN;oBACtCY;oBACAC;oBACD,kCAAA,2BAAA;;wBAAL,IAAK,YAAcH,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAfI,OAAN;;4BACqCC,aAAAA,IAAAA,mBAAS,EAACD,OAA1CE,MAAgCD,WAAhCC,KAAeC,eAAiBF,WAA3BH;4BACbA,CAAAA,YAAAA,UAASe,IAAI,OAAbf,WAAc,qBAAGK;4BACjBJ,KAAKc,IAAI,CAACC,IAAAA,uBAAU,EAACZ,KAAKL;wBAC5B;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKL;;wBAAMkB,IAAAA,8BAAe,EACnBxB,MACA;;;;;iDACMQ,CAAAA,KAAKiB,MAAM,GAAG,CAAA,GAAdjB;;;;4CAAiB;;gDAAMR,KAAK0B,QAAQ,CAACC,+BAAkB,EAAEnB;;;4CAAxC;;;;;;;;4BACvB;2BACAoB,0BAAW;;;oBALb;oBAOA;;wBAAOrB;;;;IACT"}

package/dist/esm/store/sqlite/signature.js

@@ -1,58 +0,0 @@
-import { contentTokenize } from '../../config/index.js';
-import { listFiles, parseFile } from '../../scan/index.js';
-import { BEGIN_WRITE, withTransaction } from '../transaction.js';
-import { contentRow, INSERT_CONTENT_SQL } from './reconcile.js';
-// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:
-// global features, embed provider, tokenize, then one segment per preset). Never called from
-// reconcile() itself.
-// Segment keys that moved between two feature signatures.
-export function changedSignatureKeys(before, after) {
-    const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
-    const parse = (sig)=>new Map(sig.split('|').map((part)=>[
-                keyOf(part),
-                part
-            ]));
-    const a = parse(before);
-    const b = parse(after);
-    const changed = new Set();
-    for (const [key, val] of b)if (a.get(key) !== val) changed.add(key);
-    for (const key of a.keys())if (!b.has(key)) changed.add(key);
-    return changed;
-}
-// Whether the embed segment only gained its resolved weight identity: same provider and
-// model, no identity recorded before, one now -- adopted into meta without a rebuild.
-export function embedIdentityAdopted(before, after) {
-    const embedPart = (sig)=>sig.split('|').find((p)=>p.startsWith('embed:'));
-    const b = embedPart(before);
-    const a = embedPart(after);
-    if (b === undefined || a === undefined) return false;
-    const at = a.indexOf('@');
-    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
-}
-// Names what moved, for the rebuild notice.
-export function signatureDiff(before, after) {
-    const changed = changedSignatureKeys(before, after);
-    const label = (key)=>key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
-    return changed.size === 0 ? 'features' : [
-        ...changed
-    ].map(label).join(', ');
-}
-// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)
-// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.
-export async function rebuildContentTable(conn, cfg, baseDir) {
-    const stmt = await conn.prepare('SELECT "path" FROM frontmatter');
-    const known = new Set((await stmt.all()).map((r)=>r.path));
-    const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
-    const segmenting = contentTokenize(cfg) === undefined;
-    const warnings = [];
-    const rows = [];
-    for (const file of files){
-        const { doc, warnings: fileWarnings } = parseFile(file);
-        warnings.push(...fileWarnings);
-        rows.push(contentRow(doc, segmenting));
-    }
-    await withTransaction(conn, async ()=>{
-        if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
-    }, BEGIN_WRITE);
-    return warnings;
-}

package/dist/esm/store/sqlite/signature.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/signature.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { listFiles, parseFile } from '../../scan/index.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\nimport { contentRow, INSERT_CONTENT_SQL } from './reconcile.ts';\n\n// open()-lifecycle helpers for a config-signature change (see config.featureSignature's format:\n// global features, embed provider, tokenize, then one segment per preset). Never called from\n// reconcile() itself.\n\n// Segment keys that moved between two feature signatures.\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["contentTokenize","listFiles","parseFile","BEGIN_WRITE","withTransaction","contentRow","INSERT_CONTENT_SQL","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","join","parse","sig","Map","map","a","b","changed","Set","key","val","get","add","keys","has","embedIdentityAdopted","embedPart","find","p","undefined","at","indexOf","signatureDiff","label","size","rebuildContentTable","conn","cfg","baseDir","stmt","prepare","known","all","r","path","files","filter","f","relPath","segmenting","warnings","rows","file","doc","fileWarnings","push","length","runBatch"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,SAAS,EAAEC,SAAS,QAAQ,sBAAsB;AAC3D,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,UAAU,EAAEC,kBAAkB,QAAQ,iBAAiB;AAEhE,gGAAgG;AAChG,6FAA6F;AAC7F,sBAAsB;AAEtB,0DAA0D;AAC1D,OAAO,SAASC,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGC,IAAI,CAAC,OAAOJ,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMG,QAAQ,CAACC,MAAgB,IAAIC,IAAID,IAAIJ,KAAK,CAAC,KAAKM,GAAG,CAAC,CAACR,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMS,IAAIJ,MAAMR;IAChB,MAAMa,IAAIL,MAAMP;IAChB,MAAMa,UAAU,IAAIC;IACpB,KAAK,MAAM,CAACC,KAAKC,IAAI,IAAIJ,EAAG,IAAID,EAAEM,GAAG,CAACF,SAASC,KAAKH,QAAQK,GAAG,CAACH;IAChE,KAAK,MAAMA,OAAOJ,EAAEQ,IAAI,GAAI,IAAI,CAACP,EAAEQ,GAAG,CAACL,MAAMF,QAAQK,GAAG,CAACH;IACzD,OAAOF;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASQ,qBAAqBtB,MAAc,EAAEC,KAAa;IAChE,MAAMsB,YAAY,CAACd,MAAgBA,IAAIJ,KAAK,CAAC,KAAKmB,IAAI,CAAC,CAACC,IAAMA,EAAErB,UAAU,CAAC;IAC3E,MAAMS,IAAIU,UAAUvB;IACpB,MAAMY,IAAIW,UAAUtB;IACpB,IAAIY,MAAMa,aAAad,MAAMc,WAAW,OAAO;IAC/C,MAAMC,KAAKf,EAAEgB,OAAO,CAAC;IACrB,OAAOf,EAAEe,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKf,EAAEN,KAAK,CAAC,GAAGqB,QAAQd;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASgB,cAAc7B,MAAc,EAAEC,KAAa;IACzD,MAAMa,UAAUf,qBAAqBC,QAAQC;IAC7C,MAAM6B,QAAQ,CAACd,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIZ,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEY,IAAIV,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOQ,QAAQiB,IAAI,KAAK,IAAI,aAAa;WAAIjB;KAAQ,CAACH,GAAG,CAACmB,OAAOvB,IAAI,CAAC;AACxE;AAEA,oGAAoG;AACpG,gIAAgI;AAChI,OAAO,eAAeyB,oBAAoBC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMC,OAAO,MAAMH,KAAKI,OAAO,CAAC;IAChC,MAAMC,QAAQ,IAAIvB,IAAI,AAAE,CAAA,MAAMqB,KAAKG,GAAG,EAAC,EAA+B5B,GAAG,CAAC,CAAC6B,IAAMA,EAAEC,IAAI;IACvF,MAAMC,QAAQjD,UAAUyC,KAAKC,SAASQ,MAAM,CAAC,CAACC,IAAMN,MAAMjB,GAAG,CAACuB,EAAEC,OAAO;IACvE,MAAMC,aAAatD,gBAAgB0C,SAASR;IAC5C,MAAMqB,WAAqB,EAAE;IAC7B,MAAMC,OAAoB,EAAE;IAC5B,KAAK,MAAMC,QAAQP,MAAO;QACxB,MAAM,EAAEQ,GAAG,EAAEH,UAAUI,YAAY,EAAE,GAAGzD,UAAUuD;QAClDF,SAASK,IAAI,IAAID;QACjBH,KAAKI,IAAI,CAACvD,WAAWqD,KAAKJ;IAC5B;IACA,MAAMlD,gBACJqC,MACA;QACE,IAAIe,KAAKK,MAAM,GAAG,GAAG,MAAMpB,KAAKqB,QAAQ,CAACxD,oBAAoBkD;IAC/D,GACArD;IAEF,OAAOoD;AACT"}