sensemaking 0.9.5 → 0.11.0

package/dist/cjs/features/embed.js

@@ -9,14 +9,38 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get MODEL_FILENAMES () {
+        return MODEL_FILENAMES;
+    },
+    get downloadModel () {
+        return downloadModel;
+    },
     get embed () {
         return embed;
     },
     get embedPending () {
         return embedPending;
     },
+    get hasEmbedding () {
+        return hasEmbedding;
+    },
+    get hasModelFiles () {
+        return hasModelFiles;
+    },
+    get isDownloadable () {
+        return isDownloadable;
+    },
+    get modelDir () {
+        return modelDir;
+    },
+    get modelPresent () {
+        return modelPresent;
+    },
     get semanticCandidates () {
         return semanticCandidates;
+    },
+    get similarNotes () {
+        return similarNotes;
     }
 });
 var _nodefs = require("node:fs");
@@ -267,20 +291,25 @@ function _ts_values(o) {
     };
     throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
 }
-// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,
-// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.
-// Reconcile stores dirty rows inside its transaction; embedding tops up on the next
-// semantic query (embedPending), so staleness costs recall, never correctness.
+// Heading-based chunks, int8 vectors with a per-vector scale, NULL vector = not yet embedded.
+// Reconcile writes dirty rows; embedding tops up on the next search, so staleness costs recall.
 // Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is
 // quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
 var STORE_DIMS = 256;
+// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.
+var TARGET_CHUNK_CAP = 16;
 var BATCH = 64;
-// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text
-// re-derived from the file -- chunk text is never stored). Heading-delimited with the
-// preamble kept; whole file when no headings. The title/summary prefix mirrors the
-// bm25 column weighting.
-function chunksOf(raw, search) {
-    var lines = raw.split('\n');
+// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,
+// preamble kept, whole body when no headings; the title/summary prefix mirrors bm25 weighting.
+//
+// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw
+// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw
+// put the frontmatter block in its own leading chunk on every note with a heading, which made
+// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also
+// disagreed with FTS, which indexes the body alone.
+function chunksOf(body, search) {
+    var offset = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 0;
+    var lines = body.split('\n');
     var starts = [];
     var inFence = false;
     for(var i = 0; i < lines.length; i++){
@@ -303,9 +332,11 @@ function chunksOf(raw, search) {
     bounds.forEach(function(start, i) {
         var end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;
         var text = lines.slice(start - 1, end).join('\n').trim();
+        // A body with nothing in it yields no chunks at all, so a frontmatter-only note has no
+        // vectors rather than a vector of its own YAML.
         if (text.length > 0) chunks.push({
-            startLine: start,
-            endLine: end,
+            startLine: start + offset,
+            endLine: end + offset,
             text: prefix ? "".concat(prefix, "\n").concat(text) : text
         });
     });
@@ -316,14 +347,15 @@ var embed = {
     schema: function schema(db) {
         db.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))');
     },
-    extract: function extract(raw, _body, search) {
-        return chunksOf(raw, search);
+    extract: function extract(raw, body, search) {
+        // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.
+        return chunksOf(body, search, raw.split('\n').length - body.split('\n').length);
     },
     remove: function remove(db, path) {
         db.prepare('DELETE FROM embeddings WHERE "path" = ?').run(path);
     },
-    // A doc covered only by semantic-false presets never had extract() run for it (db.ts's
-    // per-file filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.
+    // A tree with no embedding model never had extract() run for the doc (db.ts's per-file
+    // filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.
     store: function store(db, path, extracted) {
         if (!extracted) return;
         var insert = db.prepare('INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');
@@ -338,6 +370,39 @@ var embed = {
 // --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---
 // Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,
 // L2-normalize.
+var MODEL_FILES = [
+    'model.safetensors',
+    'tokenizer.json'
+];
+var MODEL_FILENAMES = MODEL_FILES.join(' and ');
+// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather
+// than flattened into a cache key Windows would reject.
+var HF_MODEL_ID = /^[A-Za-z0-9][\w.-]*\/[A-Za-z0-9][\w.-]*$/;
+function isDownloadable(model) {
+    return HF_MODEL_ID.test(model);
+}
+// Machine-wide, not per-tree: `.sense/` is the index a rebuild throws away, while a 124 MB
+// model is shared by every tree on the machine. Honors XDG_CACHE_HOME where it is set;
+// elsewhere ~/.cache, which is also where Hugging Face's own libraries keep theirs.
+function cacheRoot() {
+    var xdg = process.env.XDG_CACHE_HOME;
+    return (0, _nodepath.join)(xdg && xdg.length > 0 ? xdg : (0, _nodepath.join)((0, _nodeos.homedir)(), '.cache'), 'sensemaking', 'models');
+}
+function modelDir(model) {
+    if (!HF_MODEL_ID.test(model)) return model;
+    return (0, _nodepath.join)(cacheRoot(), model.replace(/\//g, '--'));
+}
+function hasModelFiles(model) {
+    var dir = modelDir(model);
+    return MODEL_FILES.every(function(file) {
+        return (0, _nodefs.existsSync)((0, _nodepath.join)(dir, file));
+    });
+}
+function modelPresent(cfg) {
+    var e = (0, _configts.embedConfig)(cfg);
+    if (!e) return false;
+    return e.type === 'api' || hasModelFiles(e.model);
+}
 function fetchToFile(url, dest) {
     return fetch(url).then(function(res) {
         return _async_to_generator(function() {
@@ -369,9 +434,92 @@ function fetchToFile(url, dest) {
         })();
     });
 }
+function downloadModel(model, onFile) {
+    return _async_to_generator(function() {
+        var dir, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (!isDownloadable(model)) {
+                        throw new _errorsts.SenseError('EMBED_MODEL', 'embed.model "'.concat(model, '" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like "minishlab/potion-retrieval-32M"'));
+                    }
+                    dir = modelDir(model);
+                    (0, _nodefs.mkdirSync)(dir, {
+                        recursive: true
+                    });
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        6,
+                        7,
+                        8
+                    ]);
+                    _iterator = MODEL_FILES[Symbol.iterator]();
+                    _state.label = 2;
+                case 2:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        5
+                    ];
+                    file = _step.value;
+                    if ((0, _nodefs.existsSync)((0, _nodepath.join)(dir, file))) return [
+                        3,
+                        4
+                    ];
+                    onFile === null || onFile === void 0 ? void 0 : onFile(file, dir);
+                    return [
+                        4,
+                        fetchToFile("https://huggingface.co/".concat(model, "/resolve/main/").concat(file), (0, _nodepath.join)(dir, file))
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        2
+                    ];
+                case 5:
+                    return [
+                        3,
+                        8
+                    ];
+                case 6:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        8
+                    ];
+                case 7:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 8:
+                    return [
+                        2,
+                        dir
+                    ];
+            }
+        });
+    })();
+}
 function staticProvider(model) {
     return _async_to_generator(function() {
-        var _ref, _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1, dir, _i, _iter, file, raw, headerLen, header, entry, spec, dims, dataStart, matrix, tokenizerJson, Tokenizer, tok, unkId;
+        var _ref, _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1, dir, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, fix, raw, headerLen, header, entry, spec, dims, dataStart, matrix, tokenizerJson, Tokenizer, tok, unkId;
         function one(text) {
             // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an
             // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the
@@ -416,45 +564,32 @@ function staticProvider(model) {
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    dir = model;
-                    if (!!(0, _nodefs.existsSync)((0, _nodepath.join)(dir, 'model.safetensors'))) return [
-                        3,
-                        4
-                    ];
-                    dir = (0, _nodepath.join)((0, _nodeos.homedir)(), '.cache', 'sensemaking', 'models', model.replace(/\//g, '--'));
-                    (0, _nodefs.mkdirSync)(dir, {
-                        recursive: true
-                    });
-                    _i = 0, _iter = [
-                        'model.safetensors',
-                        'tokenizer.json'
-                    ];
-                    _state.label = 1;
-                case 1:
-                    if (!(_i < _iter.length)) return [
-                        3,
-                        4
-                    ];
-                    file = _iter[_i];
-                    if (!!(0, _nodefs.existsSync)((0, _nodepath.join)(dir, file))) return [
-                        3,
-                        3
-                    ];
-                    console.error("fetching ".concat(model, "/").concat(file, " into ").concat(dir, " (once; delete to refetch)"));
-                    return [
-                        4,
-                        fetchToFile("https://huggingface.co/".concat(model, "/resolve/main/").concat(file), (0, _nodepath.join)(dir, file))
-                    ];
-                case 2:
-                    _state.sent();
-                    _state.label = 3;
-                case 3:
-                    _i++;
-                    return [
-                        3,
-                        1
-                    ];
-                case 4:
+                    dir = modelDir(model);
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        for(_iterator = MODEL_FILES[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                            file = _step.value;
+                            if (!(0, _nodefs.existsSync)((0, _nodepath.join)(dir, file))) {
+                                // A local path the caller controls gets told what is missing where; only a repo id can
+                                // be fixed by downloading.
+                                fix = isDownloadable(model) ? 'run `sense download`' : "expected ".concat(MODEL_FILENAMES, " in that directory");
+                                throw new _errorsts.SenseError('EMBED_MODEL_MISSING', "embed model ".concat(model, " is not available (looked in ").concat(dir, "); ").concat(fix));
+                            }
+                        }
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
+                    }
                     raw = (0, _nodefs.readFileSync)((0, _nodepath.join)(dir, 'model.safetensors'));
                     headerLen = Number(raw.readBigUInt64LE(0));
                     header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8'));
@@ -474,7 +609,7 @@ function staticProvider(model) {
                             return /*#__PURE__*/ _interop_require_wildcard(require("@huggingface/tokenizers"));
                         })
                     ];
-                case 5:
+                case 1:
                     Tokenizer = _state.sent().Tokenizer;
                     tok = new Tokenizer(tokenizerJson, {});
                     unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
@@ -573,7 +708,7 @@ var providers = new Map();
 function getProvider(cfg) {
     var _e_url;
     var e = (0, _configts.embedConfig)(cfg);
-    if (!e) throw new _errorsts.SenseError('EMBED_DISABLED', 'semantic search needs at least one preset with vectors on (semantic !== false); every declared preset in sense.config.json has "semantic": false');
+    if (!e) throw new _errorsts.SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an "embed" block naming one to sense.config.json, then run `sense download`');
     var sig = "".concat(e.type, ":").concat(e.model, ":").concat((_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : '');
     var p = providers.get(sig);
     if (!p) {
@@ -683,7 +818,7 @@ function embedPending(db, cfg, baseDir) {
                             chunks = void 0;
                             try {
                                 // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already
-                                // has embeddings rows means it was covered by a semantic-on preset at reconcile time.
+                                // has embeddings rows means the tree had an embedding model at reconcile time.
                                 chunks = (0, _scants.parseFile)({
                                     relPath: path,
                                     absPath: (0, _nodepath.join)(baseDir, path),
@@ -737,7 +872,7 @@ function embedPending(db, cfg, baseDir) {
                         }
                     }
                     update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE "path" = ? AND chunk = ?');
-                    // The lazy build is the one long silence a first --semantic query hits (measured 23s at
+                    // The lazy build is the one long silence a first search hits (measured 23s at
                     // 26k notes); progress makes it distinguishable from a hang.
                     report = (0, _progressts.progress)('embedding chunks', jobs.length);
                     i = 0;
@@ -769,6 +904,11 @@ function embedPending(db, cfg, baseDir) {
         });
     })();
 }
+// Dequantised int8 dot products land a little either side of a true cosine, so an identical
+// pair prints 1.001 and undermines a column whose whole job is being a bounded number.
+function asCosine(score) {
+    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
+}
 function semanticCandidates(db, cfg, terms, fetch1) {
     return _async_to_generator(function() {
         var _terms_match, baseDir, provider, storeDims, text, _toStore, qv, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, q, dot, d, score, existing;
@@ -849,7 +989,7 @@ function semanticCandidates(db, cfg, terms, fetch1) {
                             return {
                                 path: path,
                                 lines: b.lines,
-                                similarity: Math.round(b.score * 1000) / 1000
+                                similarity: asCosine(b.score)
                             };
                         })
                     ];
@@ -857,4 +997,80 @@ function semanticCandidates(db, cfg, terms, fetch1) {
         });
     })();
 }
+function hasEmbedding(db, path) {
+    var row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE "path" = ? AND vector IS NOT NULL LIMIT 1').get(path);
+    return row !== undefined;
+}
+function similarNotes(db, _cfg, path, opts) {
+    // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus
+    // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.
+    var targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk').all(path);
+    if (targetRows.length === 0) return [];
+    var step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
+    var target = targetRows.filter(function(_, i) {
+        return i % step === 0;
+    }).map(function(row) {
+        return {
+            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
+            scale: row.scale
+        };
+    });
+    var rows = db.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
+    var best = new Map();
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = rows[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var row = _step.value;
+            if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
+            var other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+            var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+            try {
+                for(var _iterator1 = target[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                    var t = _step1.value;
+                    var dot = 0;
+                    var len = Math.min(t.v.length, other.length);
+                    for(var d = 0; d < len; d++)dot += t.v[d] * other[d];
+                    var score = dot * t.scale * row.scale;
+                    var existing = best.get(row.path);
+                    if (existing === undefined || score > existing) best.set(row.path, score);
+                }
+            } catch (err) {
+                _didIteratorError1 = true;
+                _iteratorError1 = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                        _iterator1.return();
+                    }
+                } finally{
+                    if (_didIteratorError1) {
+                        throw _iteratorError1;
+                    }
+                }
+            }
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return _to_consumable_array(best.entries()).sort(function(a, b) {
+        return b[1] - a[1];
+    }).slice(0, opts.k).map(function(param) {
+        var _param = _sliced_to_array(param, 2), p = _param[0], score = _param[1];
+        return {
+            path: p,
+            similarity: asCosine(score)
+        };
+    });
+}
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  // A doc covered only by semantic-false presets never had extract() run for it (db.ts's\n  // per-file filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.\n  store(db, path, extracted) {\n    if (!extracted) return;\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic search needs at least one preset with vectors on (semantic !== false); every declared preset in sense.config.json has \"semantic\": false');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means it was covered by a semantic-on preset at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first --semantic query hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + BATCH, jobs.length));\n  }\n  report.finish();\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\n// Returns the cosine similarity alongside each candidate: stored and query vectors are both\n// L2-normalised before quantisation, so the dot product is cosine in [-1, 1]. `find` surfaces\n// it because the fused rank score cannot express match quality -- nearest-neighbour search\n// always returns a nearest neighbour, so without a magnitude an agent cannot tell a real hit\n// from the best of a bad lot.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: Math.round(b.score * 1000) / 1000 }));\n}\n"],"names":["embed","embedPending","semanticCandidates","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","join","chunks","forEach","start","end","text","slice","trim","startLine","endLine","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","enabledForFile","_cfg","file","fetchToFile","url","dest","fetch","then","res","ok","SenseError","status","Buffer","from","arrayBuffer","writeFileSync","renameSync","staticProvider","model","tokenizerJson","dir","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","ids","encode","add_special_tokens","id","Number","isInteger","v","Float32Array","off","d","norm","Math","sqrt","existsSync","homedir","replace","mkdirSync","recursive","console","error","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","vocab","unk_token","texts","map","apiProvider","keyEnv","base","headers","key","post","body","method","stringify","input","json","data","embedding","process","env","undefined","authorization","providers","Map","getProvider","cfg","e","embedConfig","sig","type","p","get","set","toStore","full","int8","scale","max","abs","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","chunkIdxs","update","report","batch","vectors","j","job","q","Int8Array","round","chunk","err","tick","min","all","parseFile","relPath","absPath","mtimeMs","size","presets","doc","progress","finish","terms","qv","rows","best","dot","score","existing","match","t","includes","vector","byteLength","start_line","end_line","sort","a","b","similarity"],"mappings":";;;;;;;;;;;QA8DaA;eAAAA;;QAyISC;eAAAA;;QA2DAC;eAAAA;;;sBAlQyD;sBACvD;wBACH;wBAGO;wBACD;0BACF;sBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG1B,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,IAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,IAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,IAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,IAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;KAAa,CAAd,OAAI,qBAAGA,WAAUA;IAC5E,IAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,IAAMC,SAAkB,EAAE;IAC1BP,OAAOQ,OAAO,CAAC,SAACC,OAAOb;QACrB,IAAMc,MAAMd,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,IAAMc,OAAOnB,MACVoB,KAAK,CAACH,QAAQ,GAAGC,KACjBJ,IAAI,CAAC,MACLO,IAAI;QACP,IAAIF,KAAKd,MAAM,GAAG,GAAGU,OAAOR,IAAI,CAAC;YAAEe,WAAWL;YAAOM,SAASL;YAAKC,MAAMV,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAS,OAALU,QAASA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEO,IAAMvB,QAAiB;IAC5BgC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;IACV;IACAC,SAAAA,SAAAA,QAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAAA,SAAAA,OAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACA,uFAAuF;IACvF,4FAA4F;IAC5FG,OAAAA,SAAAA,MAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAI,CAACA,WAAW;QAChB,IAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBnB,OAAO,CAAC,SAACqB,GAAGC;mBAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEf,SAAS,EAAEe,EAAEd,OAAO;;IACzF;IACAgB,gBAAAA,SAAAA,eAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKjD,KAAK;IACnB;AACF;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASkD,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,SAAOC;;;;;;wBAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,0BAAwCF,OAAfJ,KAAI,aAAsB,OAAXI,IAAIG,MAAM;;4BACrF,GAAO,OAALN,MAAK;;4BAAQO,OAAOC,IAAI;wBAAC;;4BAAML,IAAIM,WAAW;;;wBAA/DC,qBAAa;4BAAiBH,QAAAA;gCAAY;;;wBAC1CI,IAAAA,kBAAU,EAAC,AAAC,GAAO,OAALX,MAAK,UAAQA;;;;;;QAC7B;;AACF;AAEA,SAAeY,eAAeC,KAAa;;kBA2BEC,sBAA7BA,4BAAAA,uBA1BVC,SAIiB,OAARlB,MAQP3C,KACA8D,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAR,eAEES,WACFC,KACAC;QAEN,SAASC,IAAInD,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMoD,MAAM,AAACH,IAAII,MAAM,CAACrD,MAAM;gBAAEsD,oBAAoB;YAAM,GAAGF,GAAG,CAAc3D,MAAM,CAAC,SAAC8D;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOL;;YAC5H,IAAMQ,IAAI,IAAIC,aAAad;YAC3B,IAAIO,IAAIlE,MAAM,KAAK,GAAG,OAAOwE;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYN,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMG,KAAN;oBACH,IAAMK,MAAML,KAAKV;oBACjB,IAAK,IAAIgB,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKH,CAAC,CAACG,EAAE,IAAId,MAAM,CAACa,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK;gBAC7BH,CAAC,CAACG,GAAE,IAAIT,IAAIlE,MAAM;gBAClB4E,QAAQJ,CAAC,CAACG,GAAE,GAAGH,CAAC,CAACG,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,IAAIC;YACvC,OAAOJ;QACT;;;;oBA/CIlB,MAAMF;yBACN,CAAC2B,IAAAA,kBAAU,EAACtE,IAAAA,cAAI,EAAC6C,KAAK,uBAAtB;;;;oBACFA,MAAM7C,IAAAA,cAAI,EAACuE,IAAAA,eAAO,KAAI,UAAU,eAAe,UAAU5B,MAAM6B,OAAO,CAAC,OAAO;oBAC9EC,IAAAA,iBAAS,EAAC5B,KAAK;wBAAE6B,WAAW;oBAAK;4BACd;wBAAC;wBAAqB;;;;+BAAtB;;;;oBAAR/C,OAAQ;yBACb,CAAC2C,IAAAA,kBAAU,EAACtE,IAAAA,cAAI,EAAC6C,KAAKlB,QAAtB;;;;oBACFgD,QAAQC,KAAK,CAAC,AAAC,YAAoBjD,OAATgB,OAAM,KAAgBE,OAAblB,MAAK,UAAY,OAAJkB,KAAI;oBACpD;;wBAAMjB,YAAY,AAAC,0BAA+CD,OAAtBgB,OAAM,kBAAqB,OAALhB,OAAQ3B,IAAAA,cAAI,EAAC6C,KAAKlB;;;oBAApF;;;;;;;;;oBAKA3C,MAAM6F,IAAAA,oBAAY,EAAC7E,IAAAA,cAAI,EAAC6C,KAAK;oBAC7BC,YAAYe,OAAO7E,IAAI8F,eAAe,CAAC;oBACvC/B,SAASgC,KAAKC,KAAK,CAAChG,IAAIiG,QAAQ,CAAC,GAAG,IAAInC,WAAWoC,QAAQ,CAAC;oBAC5DlC,QAAQmC,OAAOC,OAAO,CAACrC,QAAQsC,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAACtC,SAASA,KAAK,CAAC,EAAE,CAACuC,KAAK,KAAK,OAAO,MAAM,IAAIpD,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAANQ,OAAM;oBAC/EM,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKuC,KAAK,CAAC,EAAE;oBACpBrC,YAAYnE,IAAIyG,UAAU,GAAG,IAAI3C,YAAYG,KAAKyC,YAAY,CAAC,EAAE;oBACjEtC,SAASD,YAAY,MAAM,IAAI,IAAIa,aAAahF,IAAI2G,MAAM,EAAExC,WAAWF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,QAAQ,IAAIc,aAAahF,IAAI2G,MAAM,CAACrF,KAAK,CAAC6C,WAAWA,YAAYF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,OAAO;oBAE/KN,gBAAgBmC,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAAC7E,IAAAA,cAAI,EAAC6C,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BQ,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUT,eAAe,CAAC;oBACpCW,iBAAQX,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBgD,KAAK,cAA1BhD,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBiD,SAAS,CAAC,uCAAI,CAAC;oBAuB/E;;wBAAO;4BAAEjC,IAAI,AAAC,UAAe,OAANjB;4BAASO,MAAAA;4BAAMxE,OAAO,SAAPA,MAAcoH;;;;;4CAAUA,MAAMC,GAAG,CAACvC;;;;;wBAAK;;;;IAC/E;;AAEA,gFAAgF;AAEhF,SAAewC,YAAYrD,KAAa,EAAEd,GAAuB,EAAEoE,MAA0B;;YAErFC,MACAC,SACAC,KAUAlD;QAPN,SAAemD,KAAKP,KAAe;;oBAC3B7D,KAEAqE;;;;4BAFM;;gCAAMvE,MAAM,AAAC,GAAO,OAALmE,MAAK,gBAAc;oCAAEK,QAAQ;oCAAQJ,SAAAA;oCAASG,MAAMvB,KAAKyB,SAAS,CAAC;wCAAE7D,OAAAA;wCAAO8D,OAAOX;oCAAM;gCAAG;;;4BAAjH7D,MAAM;4BACZ,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BF,OAA3BiE,MAAK,wBAAiC,OAAXjE,IAAIG,MAAM;4BAC3E;;gCAAMH,IAAIyE,IAAI;;;4BAAtBJ,OAAQ;4BACd;;gCAAOA,KAAKK,IAAI,CAACZ,GAAG,CAAC,SAAC7B;2CAAMF,aAAa1B,IAAI,CAAC4B,EAAE0C,SAAS;;;;;YAC3D;;;;;oBAXA,IAAI,CAAC/E,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe;oBACxC+D,OAAOrE,IAAI2C,OAAO,CAAC,QAAQ;oBAC3B2B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAASY,QAAQC,GAAG,CAACb,OAAO,GAAGc;oBAC3C,IAAIX,KAAKD,QAAQa,aAAa,GAAG,AAAC,UAAa,OAAJZ;oBAS7B;;wBAAMC;4BAAM;;;;oBAApBnD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAAC3D,MAAM;oBACxD;;wBAAO;4BAAEqE,IAAI,AAAC,OAAcjB,OAARuD,MAAK,KAAS,OAANvD;4BAASO,MAAAA;4BAAMxE,OAAO2H;wBAAK;;;;IACzD;;AAEA,IAAMY,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,IAAMA,IAAIC,IAAAA,qBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIlF,oBAAU,CAAC,kBAAkB;IAC/C,IAAMoF,MAAM,AA/Kd,AA+Ke,GAAYF,OAAVA,EAAEG,IAAI,EAAC,YAAGH,EAAE1E,KAAK,EAAC,KAAe,QAAZ0E,SAAAA,EAAExF,GAAG,cAALwF,oBAAAA,SAAS;IAC7C,IAAII,IAAIR,UAAUS,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIJ,EAAEG,IAAI,KAAK,QAAQxB,YAAYqB,EAAE1E,KAAK,EAAE0E,EAAExF,GAAG,EAAEwF,EAAEjB,GAAG,IAAI1D,eAAe2E,EAAE1E,KAAK;QAClFsE,UAAUU,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAE3E,IAAY,EAAE4E,IAAa;IAC9D,IAAM/D,IAAI,IAAIC,aAAad;IAC3B,IAAIiB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKC,QAAQ0D,IAAI,CAAC3D,EAAE,GAAG2D,IAAI,CAAC3D,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,GAAG2D,IAAI,CAAC3D,GAAE,GAAGC;IAChD,IAAI,CAAC2D,MAAM,OAAO;QAAE/D,GAAAA;QAAGgE,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAI9D,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK8D,MAAM5D,KAAK4D,GAAG,CAACA,KAAK5D,KAAK6D,GAAG,CAAClE,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGgE,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAerJ,aAAaiC,EAAgB,EAAEwG,GAAW,EAAEc,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOxH,MAAMyH,WACZzI,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMuB,KAGPmH,QAGAC,QACGtJ;;;;;4BACDuJ,OACAC;;;;oCADAD,QAAQJ,KAAKnI,KAAK,CAAChB,GAAGA,IAAIR;oCAChB;;wCAAMqJ,SAASzJ,KAAK,CAACmK,MAAM9C,GAAG,CAAC,SAACgD;mDAAMA,EAAE1I,IAAI;;;;oCAAtDyI,UAAU;oCAChBlI,GAAGC,IAAI,CAAC;oCACR,IAAI;wCACFgI,MAAM3I,OAAO,CAAC,SAAC8I,KAAKD;4CAClB,IAAqBnB,WAAAA,QAAQkB,OAAO,CAACC,EAAE,EAAEV,WAAW,OAA5CtE,IAAa6D,SAAb7D,GAAGgE,QAAUH,SAAVG;4CACX,IAAMkB,IAAI,IAAIC,UAAUb;4CACxB,IAAK,IAAInE,IAAI,GAAGA,IAAImE,WAAWnE,IAAK+E,CAAC,CAAC/E,EAAE,GAAGE,KAAK+E,KAAK,CAACpF,CAAC,CAACG,EAAE,GAAG6D;4CAC7DY,OAAOxH,GAAG,CAAC4G,OAAO1F,OAAOC,IAAI,CAAC2G,EAAEtD,MAAM,GAAGqD,IAAI/H,IAAI,EAAE+H,IAAII,KAAK;wCAC9D;wCACAxI,GAAGC,IAAI,CAAC;oCACV,EAAE,OAAOwI,KAAK;wCACZzI,GAAGC,IAAI,CAAC;wCACR,MAAMwI;oCACR;oCACAT,OAAOU,IAAI,CAAClF,KAAKmF,GAAG,CAACjK,IAAIR,OAAO2J,KAAKlJ,MAAM;;;;;;oBAC7C;oBA9CiB;;wBAAM4H,YAAYC;;;oBAA7Be,WAAW;oBACXC,QAAQxH,GAAGM,OAAO,CAAC,oFAAoFsI,GAAG;oBAChH,IAAIpB,MAAM7I,MAAM,KAAK,GAAG;;;oBAClB8I,YAAYjE,KAAKmF,GAAG,CAAC1K,YAAYsJ,SAASjF,IAAI;oBAE9CoF,SAAS,IAAIpB;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAakB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOZ,GAAG,CAACa,IAAItH,IAAI,eAAnBqH,yBAAAA;4BACbE,KAAK/I,IAAI,CAAC8I,IAAIa,KAAK;4BACnBd,OAAOX,GAAG,CAACY,IAAItH,IAAI,EAAEuH;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOrH,uBAAMyH;4BACZzI,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,sFAAsF;gCACtFA,SAASwJ,IAAAA,iBAAS,EAAC;oCAAEC,SAASzI;oCAAM0I,SAAS3J,IAAAA,cAAI,EAACkI,SAASjH;oCAAO2I,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMpL,OAAO;gCAAK;oCAAIA;mCAAQqL,GAAG,CAAC1I,SAAS,CAAC3C,KAAK;4BACjJ,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAagK,gCAAb,8BAAA,SAAA,0BAAA;oCAAMlH,MAAN;oCAAwB,IAAIvB,MAAM,CAACuB,IAAI,EAAEiH,KAAKhJ,IAAI,CAAC;wCAAEwB,MAAAA;wCAAMmI,OAAO5H;wCAAKnB,MAAMJ,MAAM,CAACuB,IAAI,CAACnB,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYCsI,SAAS/H,GAAGM,OAAO,CAAC;oBAC1B,wFAAwF;oBACxF,6DAA6D;oBACvD0H,SAASoB,IAAAA,oBAAQ,EAAC,oBAAoBvB,KAAKlJ,MAAM;oBAC9CD,IAAI;;;yBAAGA,CAAAA,IAAImJ,KAAKlJ,MAAM,AAAD;;;;;;;;;;;;oBAAGD,KAAKR;;;;;;oBAkBtC8J,OAAOqB,MAAM;;;;;;IACf;;AAUO,SAAerL,mBAAmBgC,EAAgB,EAAEwG,GAAW,EAAE8C,KAAa,EAAEnI,MAAa;;YAOpFmI,cANRhC,SAIAC,UACAE,WACAhI,MACYuH,UAAPuC,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM9B,KACHU,GACFqB,KACKpG,GACHqG,OACAC;;;;oBAvBFtC,UAAU,AAACd,IAAgCc,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAI/F,oBAAU,CAAC,eAAe;oBAClD;;wBAAMxD,aAAaiC,IAAIwG,KAAKc;;;oBAA5B;oBAEiB;;wBAAMf,YAAYC;;;oBAA7Be,WAAW;oBACXE,YAAYjE,KAAKmF,GAAG,CAAC1K,YAAYsJ,SAASjF,IAAI;oBAC9C7C,OAAO,EAAC6J,eAAAA,MAAMO,KAAK,CAAC,2naAAZP,0BAAAA,mBAAsCpK,MAAM,CAAC,SAAC4K;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAI1K,IAAI,CAAC;oBACvF;;wBAAMmI,SAASzJ,KAAK;4BAAE2B;;;;oBAA/BuH,WAAAA;wBAAS,aAA6B,CAAC,EAAE;wBAAES;wBAAW;wBAA7D8B,KAAOvC,SAAV7D;oBAEFqG,OAAOxJ,GAAGM,OAAO,CAAC,+FAA+FsI,GAAG;oBAQpHa,OAAO,IAAInD;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAakD,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAb7B,MAAN;4BACGU,IAAI,IAAIC,UAAUX,IAAIqC,MAAM,CAACjF,MAAM,EAAE4C,IAAIqC,MAAM,CAACnF,UAAU,EAAErB,KAAKmF,GAAG,CAAClB,WAAWE,IAAIqC,MAAM,CAACC,UAAU;4BACvGP,MAAM;4BACV,IAASpG,IAAI,GAAGA,IAAI+E,EAAE1J,MAAM,EAAE2E,IAAKoG,OAAOrB,CAAC,CAAC/E,EAAE,GAAGiG,EAAE,CAACjG,EAAE;4BAChDqG,QAAQD,MAAM/B,IAAIR,KAAK;4BACvByC,WAAWH,KAAK3C,GAAG,CAACa,IAAItH,IAAI;4BAClC,IAAI,CAACuJ,YAAYD,QAAQC,SAASD,KAAK,EAAEF,KAAK1C,GAAG,CAACY,IAAItH,IAAI,EAAE;gCAAEsJ,OAAAA;gCAAOrL,OAAO,AAAC,IAAqBqJ,OAAlBA,IAAIuC,UAAU,EAAC,KAAgB,OAAbvC,IAAIwC,QAAQ;4BAAG;wBACnH;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAQL;;wBAAQ,qBAAGV,KAAKjF,OAAO,IACpB4F,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACX,KAAK,GAAGU,CAAC,CAAC,EAAE,CAACV,KAAK;2BACtCjK,KAAK,CAAC,GAAGyB,QACTgE,GAAG,CAAC;qEAAE9E,kBAAMiK;mCAAQ;gCAAEjK,MAAAA;gCAAM/B,OAAOgM,EAAEhM,KAAK;gCAAEiM,YAAY/G,KAAK+E,KAAK,CAAC+B,EAAEX,KAAK,GAAG,QAAQ;4BAAK;;;;;IAC/F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// Heading-based chunks, int8 vectors with a per-vector scale, NULL vector = not yet embedded.\n// Reconcile writes dirty rows; embedding tops up on the next search, so staleness costs recall.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,\n// preamble kept, whole body when no headings; the title/summary prefix mirrors bm25 weighting.\n//\n// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw\n// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw\n// put the frontmatter block in its own leading chunk on every note with a heading, which made\n// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also\n// disagreed with FTS, which indexes the body alone.\nfunction chunksOf(body: string, search?: { title: string; summary: string }, offset = 0): Chunk[] {\n  const lines = body.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    // A body with nothing in it yields no chunks at all, so a frontmatter-only note has no\n    // vectors rather than a vector of its own YAML.\n    if (text.length > 0) chunks.push({ startLine: start + offset, endLine: end + offset, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    return chunksOf(body, search, raw.split('\\n').length - body.split('\\n').length);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  // A tree with no embedding model never had extract() run for the doc (db.ts's per-file\n  // filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.\n  store(db, path, extracted) {\n    if (!extracted) return;\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nconst MODEL_FILES = ['model.safetensors', 'tokenizer.json'];\n// The pair, for messages that name what a local model directory must contain.\nexport const MODEL_FILENAMES = MODEL_FILES.join(' and ');\n\n// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather\n// than flattened into a cache key Windows would reject.\nconst HF_MODEL_ID = /^[A-Za-z0-9][\\w.-]*\\/[A-Za-z0-9][\\w.-]*$/;\n\n// Downloadable iff it names a Hugging Face repo; a local path is the caller's to populate.\nexport function isDownloadable(model: string): boolean {\n  return HF_MODEL_ID.test(model);\n}\n\n// Machine-wide, not per-tree: `.sense/` is the index a rebuild throws away, while a 124 MB\n// model is shared by every tree on the machine. Honors XDG_CACHE_HOME where it is set;\n// elsewhere ~/.cache, which is also where Hugging Face's own libraries keep theirs.\nfunction cacheRoot(): string {\n  const xdg = process.env.XDG_CACHE_HOME;\n  return join(xdg && xdg.length > 0 ? xdg : join(homedir(), '.cache'), 'sensemaking', 'models');\n}\n\n// A model is either a local directory the caller pointed at, or the cache `sense download`\n// fills. Nothing here fetches: an absent model degrades search to its other signals rather\n// than pulling 124 MB out of a command that reads like a query.\nexport function modelDir(model: string): string {\n  if (!HF_MODEL_ID.test(model)) return model;\n  return join(cacheRoot(), model.replace(/\\//g, '--'));\n}\n\n// Both files, so an interrupted download reads as absent and the next one resumes it.\nexport function hasModelFiles(model: string): boolean {\n  const dir = modelDir(model);\n  return MODEL_FILES.every((file) => existsSync(join(dir, file)));\n}\n\n// api providers have nothing to download, so they are never \"missing\" here; an unreachable\n// endpoint surfaces as an EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.type === 'api' || hasModelFiles(e.model);\n}\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\n// The only code path that touches the network for weights. `sense download` calls it; no\n// query ever does. Idempotent: a file already on disk is left alone.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const dir = modelDir(model);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  const dir = modelDir(model);\n  for (const file of MODEL_FILES) {\n    if (!existsSync(join(dir, file))) {\n      // A local path the caller controls gets told what is missing where; only a repo id can\n      // be fixed by downloading.\n      const fix = isDownloadable(model) ? 'run `sense download`' : `expected ${MODEL_FILENAMES} in that directory`;\n      throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); ${fix}`);\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json, then run `sense download`');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + BATCH, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["MODEL_FILENAMES","downloadModel","embed","embedPending","hasEmbedding","hasModelFiles","isDownloadable","modelDir","modelPresent","semanticCandidates","similarNotes","STORE_DIMS","TARGET_CHUNK_CAP","BATCH","chunksOf","body","search","offset","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","join","chunks","forEach","start","end","text","slice","trim","startLine","endLine","name","schema","db","exec","extract","raw","remove","path","prepare","run","store","extracted","insert","c","idx","enabledForFile","_cfg","file","MODEL_FILES","HF_MODEL_ID","model","cacheRoot","xdg","process","env","XDG_CACHE_HOME","homedir","replace","dir","every","existsSync","cfg","e","embedConfig","type","fetchToFile","url","dest","fetch","then","res","ok","SenseError","status","Buffer","from","arrayBuffer","writeFileSync","renameSync","onFile","mkdirSync","recursive","staticProvider","tokenizerJson","fix","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","ids","encode","add_special_tokens","id","Number","isInteger","v","Float32Array","off","d","norm","Math","sqrt","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","vocab","unk_token","texts","map","apiProvider","keyEnv","base","headers","key","post","method","stringify","input","json","data","embedding","undefined","authorization","providers","Map","getProvider","sig","p","get","set","toStore","full","int8","scale","max","abs","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","chunkIdxs","update","report","batch","vectors","j","job","q","Int8Array","round","chunk","err","tick","min","all","parseFile","relPath","absPath","mtimeMs","size","presets","doc","progress","finish","asCosine","score","terms","qv","rows","best","dot","existing","match","t","includes","vector","byteLength","start_line","end_line","sort","a","b","similarity","opts","targetRows","step","ceil","target","_","exclude","has","allowed","other","len"],"mappings":";;;;;;;;;;;QAkGaA;eAAAA;;QAmDSC;eAAAA;;QAjFTC;eAAAA;;QAmMSC;eAAAA;;QA+FNC;eAAAA;;QAxOAC;eAAAA;;QArBAC;eAAAA;;QAeAC;eAAAA;;QAaAC;eAAAA;;QA8LMC;eAAAA;;QA0CNC;eAAAA;;;sBA7W+D;sBACvD;wBACH;wBAGO;wBACD;0BACF;sBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG1B,8FAA8F;AAC9F,gGAAgG;AAEhG,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,0FAA0F;AAC1F,IAAMC,mBAAmB;AACzB,IAAMC,QAAQ;AAcd,8FAA8F;AAC9F,+FAA+F;AAC/F,EAAE;AACF,2FAA2F;AAC3F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,oDAAoD;AACpD,SAASC,SAASC,IAAY,EAAEC,MAA2C;QAAEC,SAAAA,iEAAS;IACpF,IAAMC,QAAQH,KAAKI,KAAK,CAAC;IACzB,IAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,IAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;KAAa,CAAd,OAAI,qBAAGA,WAAUA;IAC5E,IAAMO,SAAS;QAACX,mBAAAA,6BAAAA,OAAQY,KAAK;QAAEZ,mBAAAA,6BAAAA,OAAQa,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,IAAMC,SAAkB,EAAE;IAC1BP,OAAOQ,OAAO,CAAC,SAACC,OAAOb;QACrB,IAAMc,MAAMd,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,IAAMc,OAAOnB,MACVoB,KAAK,CAACH,QAAQ,GAAGC,KACjBJ,IAAI,CAAC,MACLO,IAAI;QACP,uFAAuF;QACvF,gDAAgD;QAChD,IAAIF,KAAKd,MAAM,GAAG,GAAGU,OAAOR,IAAI,CAAC;YAAEe,WAAWL,QAAQlB;YAAQwB,SAASL,MAAMnB;YAAQoB,MAAMV,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAS,OAALU,QAASA;QAAK;IAClI;IACA,OAAOJ;AACT;AAEO,IAAM/B,QAAiB;IAC5BwC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;IACV;IACAC,SAAAA,SAAAA,QAAQC,GAAG,EAAEhC,IAAI,EAAEC,MAAM;QACvB,mFAAmF;QACnF,OAAOF,SAASC,MAAMC,QAAQ+B,IAAI5B,KAAK,CAAC,MAAMI,MAAM,GAAGR,KAAKI,KAAK,CAAC,MAAMI,MAAM;IAChF;IACAyB,QAAAA,SAAAA,OAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACA,uFAAuF;IACvF,mFAAmF;IACnFG,OAAAA,SAAAA,MAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAI,CAACA,WAAW;QAChB,IAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBnB,OAAO,CAAC,SAACqB,GAAGC;mBAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEf,SAAS,EAAEe,EAAEd,OAAO;;IACzF;IACAgB,gBAAAA,SAAAA,eAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKzD,KAAK;IACnB;AACF;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,IAAM0D,cAAc;IAAC;IAAqB;CAAiB;AAEpD,IAAM5D,kBAAkB4D,YAAY5B,IAAI,CAAC;AAEhD,+FAA+F;AAC/F,wDAAwD;AACxD,IAAM6B,cAAc;AAGb,SAASvD,eAAewD,KAAa;IAC1C,OAAOD,YAAYrC,IAAI,CAACsC;AAC1B;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,oFAAoF;AACpF,SAASC;IACP,IAAMC,MAAMC,QAAQC,GAAG,CAACC,cAAc;IACtC,OAAOnC,IAAAA,cAAI,EAACgC,OAAOA,IAAIzC,MAAM,GAAG,IAAIyC,MAAMhC,IAAAA,cAAI,EAACoC,IAAAA,eAAO,KAAI,WAAW,eAAe;AACtF;AAKO,SAAS7D,SAASuD,KAAa;IACpC,IAAI,CAACD,YAAYrC,IAAI,CAACsC,QAAQ,OAAOA;IACrC,OAAO9B,IAAAA,cAAI,EAAC+B,aAAaD,MAAMO,OAAO,CAAC,OAAO;AAChD;AAGO,SAAShE,cAAcyD,KAAa;IACzC,IAAMQ,MAAM/D,SAASuD;IACrB,OAAOF,YAAYW,KAAK,CAAC,SAACZ;eAASa,IAAAA,kBAAU,EAACxC,IAAAA,cAAI,EAACsC,KAAKX;;AAC1D;AAIO,SAASnD,aAAaiE,GAAW;IACtC,IAAMC,IAAIC,IAAAA,qBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEE,IAAI,KAAK,SAASvE,cAAcqE,EAAEZ,KAAK;AAClD;AAEA,SAASe,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,SAAOC;;;;;;wBAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,0BAAwCF,OAAfJ,KAAI,aAAsB,OAAXI,IAAIG,MAAM;;4BACrF,GAAO,OAALN,MAAK;;4BAAQO,OAAOC,IAAI;wBAAC;;4BAAML,IAAIM,WAAW;;;wBAA/DC,qBAAa;4BAAiBH,QAAAA;gCAAY;;;wBAC1CI,IAAAA,kBAAU,EAAC,AAAC,GAAO,OAALX,MAAK,UAAQA;;;;;;QAC7B;;AACF;AAIO,SAAe9E,cAAc6D,KAAa,EAAE6B,MAA4C;;YAIvFrB,KAED,2BAAA,mBAAA,gBAAA,WAAA,OAAMX;;;;oBALX,IAAI,CAACrD,eAAewD,QAAQ;wBAC1B,MAAM,IAAIsB,oBAAU,CAAC,eAAe,AAAC,gBAAqB,OAANtB,OAAM;oBAC5D;oBACMQ,MAAM/D,SAASuD;oBACrB8B,IAAAA,iBAAS,EAACtB,KAAK;wBAAEuB,WAAW;oBAAK;oBAC5B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcjC;;;2BAAd,6BAAA,QAAA;;;;oBAAMD,OAAN;oBACH,IAAIa,IAAAA,kBAAU,EAACxC,IAAAA,cAAI,EAACsC,KAAKX,QAAQ;;;;oBACjCgC,mBAAAA,6BAAAA,OAAShC,MAAMW;oBACf;;wBAAMO,YAAY,AAAC,0BAA+ClB,OAAtBG,OAAM,kBAAqB,OAALH,OAAQ3B,IAAAA,cAAI,EAACsC,KAAKX;;;oBAApF;;;oBAHG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAKL;;wBAAOW;;;;IACT;;AAEA,SAAewB,eAAehC,KAAa;;kBAyBEiC,sBAA7BA,4BAAAA,uBAxBRzB,KACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMX,MAIDqC,KAKJjD,KACAkD,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAR,eAEES,WACFC,KACAC;QAEN,SAASC,IAAItE,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMuE,MAAM,AAACH,IAAII,MAAM,CAACxE,MAAM;gBAAEyE,oBAAoB;YAAM,GAAGF,GAAG,CAAc9E,MAAM,CAAC,SAACiF;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOL;;YAC5H,IAAMQ,IAAI,IAAIC,aAAad;YAC3B,IAAIO,IAAIrF,MAAM,KAAK,GAAG,OAAO2F;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYN,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMG,KAAN;oBACH,IAAMK,MAAML,KAAKV;oBACjB,IAAK,IAAIgB,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKH,CAAC,CAACG,EAAE,IAAId,MAAM,CAACa,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK;gBAC7BH,CAAC,CAACG,GAAE,IAAIT,IAAIrF,MAAM;gBAClB+F,QAAQJ,CAAC,CAACG,GAAE,GAAGH,CAAC,CAACG,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,IAAIC;YACvC,OAAOJ;QACT;;;;oBA7CM5C,MAAM/D,SAASuD;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAcF,kCAAd,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBD,OAAN;4BACH,IAAI,CAACa,IAAAA,kBAAU,EAACxC,IAAAA,cAAI,EAACsC,KAAKX,QAAQ;gCAChC,uFAAuF;gCACvF,2BAA2B;gCACrBqC,MAAM1F,eAAewD,SAAS,yBAAyB,AAAC,YAA2B,OAAhB9D,iBAAgB;gCACzF,MAAM,IAAIoF,oBAAU,CAAC,uBAAuB,AAAC,eAAmDd,OAArCR,OAAM,iCAAwCkC,OAAT1B,KAAI,OAAS,OAAJ0B;4BAC3G;wBACF;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBASCjD,MAAM0E,IAAAA,oBAAY,EAACzF,IAAAA,cAAI,EAACsC,KAAK;oBAC7B2B,YAAYe,OAAOjE,IAAI2E,eAAe,CAAC;oBACvCxB,SAASyB,KAAKC,KAAK,CAAC7E,IAAI8E,QAAQ,CAAC,GAAG,IAAI5B,WAAW6B,QAAQ,CAAC;oBAC5D3B,QAAQ4B,OAAOC,OAAO,CAAC9B,QAAQ+B,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC/B,SAASA,KAAK,CAAC,EAAE,CAACgC,KAAK,KAAK,OAAO,MAAM,IAAI/C,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAANtB,OAAM;oBAC/EsC,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKgC,KAAK,CAAC,EAAE;oBACpB9B,YAAYvD,IAAIsF,UAAU,GAAG,IAAIpC,YAAYG,KAAKkC,YAAY,CAAC,EAAE;oBACjE/B,SAASD,YAAY,MAAM,IAAI,IAAIa,aAAapE,IAAIwF,MAAM,EAAEjC,WAAWF,KAAKgC,KAAK,CAAC,EAAE,GAAG/B,QAAQ,IAAIc,aAAapE,IAAIwF,MAAM,CAACjG,KAAK,CAACgE,WAAWA,YAAYF,KAAKgC,KAAK,CAAC,EAAE,GAAG/B,OAAO;oBAE/KN,gBAAgB4B,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACzF,IAAAA,cAAI,EAACsC,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BkC,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUT,eAAe,CAAC;oBACpCW,iBAAQX,wBAAAA,cAAcjC,KAAK,cAAnBiC,6CAAAA,6BAAAA,sBAAqByC,KAAK,cAA1BzC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcjC,KAAK,cAAnBiC,2CAAAA,qBAAqB0C,SAAS,CAAC,uCAAI,CAAC;oBAuB/E;;wBAAO;4BAAE1B,IAAI,AAAC,UAAe,OAANjD;4BAASuC,MAAAA;4BAAMnG,OAAO,SAAPA,MAAcwI;;;;;4CAAUA,MAAMC,GAAG,CAAChC;;;;;wBAAK;;;;IAC/E;;AAEA,gFAAgF;AAEhF,SAAeiC,YAAY9E,KAAa,EAAEgB,GAAuB,EAAE+D,MAA0B;;YAErFC,MACAC,SACAC,KAUA3C;QAPN,SAAe4C,KAAKP,KAAe;;oBAC3BxD,KAEAnE;;;;4BAFM;;gCAAMiE,MAAM,AAAC,GAAO,OAAL8D,MAAK,gBAAc;oCAAEI,QAAQ;oCAAQH,SAAAA;oCAAShI,MAAM4G,KAAKwB,SAAS,CAAC;wCAAErF,OAAAA;wCAAOsF,OAAOV;oCAAM;gCAAG;;;4BAAjHxD,MAAM;4BACZ,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BF,OAA3B4D,MAAK,wBAAiC,OAAX5D,IAAIG,MAAM;4BAC3E;;gCAAMH,IAAImE,IAAI;;;4BAAtBtI,OAAQ;4BACd;;gCAAOA,KAAKuI,IAAI,CAACX,GAAG,CAAC,SAACtB;2CAAMF,aAAa5B,IAAI,CAAC8B,EAAEkC,SAAS;;;;;YAC3D;;;;;oBAXA,IAAI,CAACzE,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe;oBACxC0D,OAAOhE,IAAIT,OAAO,CAAC,QAAQ;oBAC3B0E,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAAS5E,QAAQC,GAAG,CAAC2E,OAAO,GAAGW;oBAC3C,IAAIR,KAAKD,QAAQU,aAAa,GAAG,AAAC,UAAa,OAAJT;oBAS7B;;wBAAMC;4BAAM;;;;oBAApB5C,OAAO,AAAC,aAAgC,CAAC,EAAE,CAAC9E,MAAM;oBACxD;;wBAAO;4BAAEwF,IAAI,AAAC,OAAcjD,OAARgF,MAAK,KAAS,OAANhF;4BAASuC,MAAAA;4BAAMnG,OAAO+I;wBAAK;;;;IACzD;;AAEA,IAAMS,YAAY,IAAIC;AAEtB,SAASC,YAAYnF,GAAW;QAGMC;IAFpC,IAAMA,IAAIC,IAAAA,qBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIU,oBAAU,CAAC,kBAAkB;IAC/C,IAAMyE,MAAM,AA/Od,AA+Oe,GAAYnF,OAAVA,EAAEE,IAAI,EAAC,YAAGF,EAAEZ,KAAK,EAAC,KAAe,QAAZY,SAAAA,EAAEI,GAAG,cAALJ,oBAAAA,SAAS;IAC7C,IAAIoF,IAAIJ,UAAUK,GAAG,CAACF;IACtB,IAAI,CAACC,GAAG;QACNA,IAAIpF,EAAEE,IAAI,KAAK,QAAQgE,YAAYlE,EAAEZ,KAAK,EAAEY,EAAEI,GAAG,EAAEJ,EAAEsE,GAAG,IAAIlD,eAAepB,EAAEZ,KAAK;QAClF4F,UAAUM,GAAG,CAACH,KAAKC;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAE7D,IAAY,EAAE8D,IAAa;IAC9D,IAAMjD,IAAI,IAAIC,aAAad;IAC3B,IAAIiB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKC,QAAQ4C,IAAI,CAAC7C,EAAE,GAAG6C,IAAI,CAAC7C,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,GAAG6C,IAAI,CAAC7C,GAAE,GAAGC;IAChD,IAAI,CAAC6C,MAAM,OAAO;QAAEjD,GAAAA;QAAGkD,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIhD,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKgD,MAAM9C,KAAK8C,GAAG,CAACA,KAAK9C,KAAK+C,GAAG,CAACpD,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGkD,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAelK,aAAayC,EAAgB,EAAE6B,GAAW,EAAE8F,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAO7H,MAAM8H,WACZ9I,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMuB,KAGPwH,QAGAC,QACG3J;;;;;4BACD4J,OACAC;;;;oCADAD,QAAQJ,KAAKxI,KAAK,CAAChB,GAAGA,IAAIT;oCAChB;;wCAAM2J,SAAStK,KAAK,CAACgL,MAAMvC,GAAG,CAAC,SAACyC;mDAAMA,EAAE/I,IAAI;;;;oCAAtD8I,UAAU;oCAChBvI,GAAGC,IAAI,CAAC;oCACR,IAAI;wCACFqI,MAAMhJ,OAAO,CAAC,SAACmJ,KAAKD;4CAClB,IAAqBnB,WAAAA,QAAQkB,OAAO,CAACC,EAAE,EAAEV,WAAW,OAA5CxD,IAAa+C,SAAb/C,GAAGkD,QAAUH,SAAVG;4CACX,IAAMkB,IAAI,IAAIC,UAAUb;4CACxB,IAAK,IAAIrD,IAAI,GAAGA,IAAIqD,WAAWrD,IAAKiE,CAAC,CAACjE,EAAE,GAAGE,KAAKiE,KAAK,CAACtE,CAAC,CAACG,EAAE,GAAG+C;4CAC7DY,OAAO7H,GAAG,CAACiH,OAAO9E,OAAOC,IAAI,CAAC+F,EAAE/C,MAAM,GAAG8C,IAAIpI,IAAI,EAAEoI,IAAII,KAAK;wCAC9D;wCACA7I,GAAGC,IAAI,CAAC;oCACV,EAAE,OAAO6I,KAAK;wCACZ9I,GAAGC,IAAI,CAAC;wCACR,MAAM6I;oCACR;oCACAT,OAAOU,IAAI,CAACpE,KAAKqE,GAAG,CAACtK,IAAIT,OAAOiK,KAAKvJ,MAAM;;;;;;oBAC7C;oBA9CiB;;wBAAMqI,YAAYnF;;;oBAA7B+F,WAAW;oBACXC,QAAQ7H,GAAGM,OAAO,CAAC,oFAAoF2I,GAAG;oBAChH,IAAIpB,MAAMlJ,MAAM,KAAK,GAAG;;;oBAClBmJ,YAAYnD,KAAKqE,GAAG,CAACjL,YAAY6J,SAASnE,IAAI;oBAE9CsE,SAAS,IAAIhB;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAac,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOZ,GAAG,CAACa,IAAI3H,IAAI,eAAnB0H,yBAAAA;4BACbE,KAAKpJ,IAAI,CAACmJ,IAAIa,KAAK;4BACnBd,OAAOX,GAAG,CAACY,IAAI3H,IAAI,EAAE4H;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAO1H,uBAAM8H;4BACZ9I,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,+EAA+E;gCAC/EA,SAAS6J,IAAAA,iBAAS,EAAC;oCAAEC,SAAS9I;oCAAM+I,SAAShK,IAAAA,cAAI,EAACuI,SAAStH;oCAAOgJ,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMjM,OAAO;gCAAK;oCAAIA;mCAAQkM,GAAG,CAAC/I,SAAS,CAACnD,KAAK;4BACjJ,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa6K,gCAAb,8BAAA,SAAA,0BAAA;oCAAMvH,MAAN;oCAAwB,IAAIvB,MAAM,CAACuB,IAAI,EAAEsH,KAAKrJ,IAAI,CAAC;wCAAEwB,MAAAA;wCAAMwI,OAAOjI;wCAAKnB,MAAMJ,MAAM,CAACuB,IAAI,CAACnB,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYC2I,SAASpI,GAAGM,OAAO,CAAC;oBAC1B,8EAA8E;oBAC9E,6DAA6D;oBACvD+H,SAASoB,IAAAA,oBAAQ,EAAC,oBAAoBvB,KAAKvJ,MAAM;oBAC9CD,IAAI;;;yBAAGA,CAAAA,IAAIwJ,KAAKvJ,MAAM,AAAD;;;;;;;;;;;;oBAAGD,KAAKT;;;;;;oBAkBtCoK,OAAOqB,MAAM;;;;;;IACf;;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAOjF,KAAKiE,KAAK,CAACjE,KAAKqE,GAAG,CAAC,GAAGrE,KAAK8C,GAAG,CAAC,CAAC,GAAGmC,UAAU,QAAQ;AAC/D;AAKO,SAAe/L,mBAAmBmC,EAAgB,EAAE6B,GAAW,EAAEgI,KAAa,EAAEzH,MAAa;;YAOpFyH,cANRlC,SAIAC,UACAE,WACArI,MACY4H,UAAPyC,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMhC,KACHU,GACFuB,KACKxF,GACHmF,OACAM;;;;oBAvBFvC,UAAU,AAAC9F,IAAgC8F,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAInF,oBAAU,CAAC,eAAe;oBAClD;;wBAAMjF,aAAayC,IAAI6B,KAAK8F;;;oBAA5B;oBAEiB;;wBAAMX,YAAYnF;;;oBAA7B+F,WAAW;oBACXE,YAAYnD,KAAKqE,GAAG,CAACjL,YAAY6J,SAASnE,IAAI;oBAC9ChE,OAAO,EAACoK,eAAAA,MAAMM,KAAK,CAAC,2naAAZN,0BAAAA,mBAAsC3K,MAAM,CAAC,SAACkL;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIhL,IAAI,CAAC;oBACvF;;wBAAMwI,SAAStK,KAAK;4BAAEmC;;;;oBAA/B4H,WAAAA;wBAAS,aAA6B,CAAC,EAAE;wBAAES;wBAAW;wBAA7DgC,KAAOzC,SAAV/C;oBAEFyF,OAAO/J,GAAGM,OAAO,CAAC,+FAA+F2I,GAAG;oBAQpHe,OAAO,IAAIjD;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAagD,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAb/B,MAAN;4BACGU,IAAI,IAAIC,UAAUX,IAAIsC,MAAM,CAAC3E,MAAM,EAAEqC,IAAIsC,MAAM,CAAC7E,UAAU,EAAEd,KAAKqE,GAAG,CAAClB,WAAWE,IAAIsC,MAAM,CAACC,UAAU;4BACvGN,MAAM;4BACV,IAASxF,IAAI,GAAGA,IAAIiE,EAAE/J,MAAM,EAAE8F,IAAKwF,OAAOvB,CAAC,CAACjE,EAAE,GAAGqF,EAAE,CAACrF,EAAE;4BAChDmF,QAAQK,MAAMjC,IAAIR,KAAK;4BACvB0C,WAAWF,KAAK7C,GAAG,CAACa,IAAI3H,IAAI;4BAClC,IAAI,CAAC6J,YAAYN,QAAQM,SAASN,KAAK,EAAEI,KAAK5C,GAAG,CAACY,IAAI3H,IAAI,EAAE;gCAAEuJ,OAAAA;gCAAOtL,OAAO,AAAC,IAAqB0J,OAAlBA,IAAIwC,UAAU,EAAC,KAAgB,OAAbxC,IAAIyC,QAAQ;4BAAG;wBACnH;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAQL;;wBAAQ,qBAAGT,KAAK5E,OAAO,IACpBsF,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAAChB,KAAK,GAAGe,CAAC,CAAC,EAAE,CAACf,KAAK;2BACtClK,KAAK,CAAC,GAAG0C,QACT2D,GAAG,CAAC;qEAAE1F,kBAAMuK;mCAAQ;gCAAEvK,MAAAA;gCAAM/B,OAAOsM,EAAEtM,KAAK;gCAAEuM,YAAYlB,SAASiB,EAAEhB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAASpM,aAAawC,EAAgB,EAAEK,IAAY;IACzD,IAAM2H,MAAMhI,GAAGM,OAAO,CAAC,kFAAkF6G,GAAG,CAAC9G;IAC7G,OAAO2H,QAAQpB;AACjB;AAIO,SAAS9I,aAAakC,EAAgB,EAAEc,IAAY,EAAET,IAAY,EAAEyK,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,IAAMC,aAAa/K,GAAGM,OAAO,CAAC,+FAA+F2I,GAAG,CAAC5I;IACjI,IAAI0K,WAAWpM,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,IAAMqM,OAAOrG,KAAK8C,GAAG,CAAC,GAAG9C,KAAKsG,IAAI,CAACF,WAAWpM,MAAM,GAAGX;IACvD,IAAMkN,SAASH,WAAW7L,MAAM,CAAC,SAACiM,GAAGzM;eAAMA,IAAIsM,SAAS;OAAGjF,GAAG,CAAC,SAACiC;eAAS;YAAE1D,GAAG,IAAIqE,UAAUX,IAAIsC,MAAM,CAAC3E,MAAM,EAAEqC,IAAIsC,MAAM,CAAC7E,UAAU,EAAEuC,IAAIsC,MAAM,CAACC,UAAU;YAAG/C,OAAOQ,IAAIR,KAAK;QAAC;;IAE/K,IAAMuC,OAAO/J,GAAGM,OAAO,CAAC,yEAAyE2I,GAAG;IACpG,IAAMe,OAAO,IAAIjD;QACZ,kCAAA,2BAAA;;QAAL,QAAK,YAAagD,yBAAb,SAAA,6BAAA,QAAA,yBAAA,iCAAmB;YAAnB,IAAM/B,MAAN;YACH,IAAIA,IAAI3H,IAAI,KAAKA,QAAQyK,KAAKM,OAAO,CAACC,GAAG,CAACrD,IAAI3H,IAAI,KAAMyK,KAAKQ,OAAO,IAAI,CAACR,KAAKQ,OAAO,CAACD,GAAG,CAACrD,IAAI3H,IAAI,GAAI;YACtG,IAAMkL,QAAQ,IAAI5C,UAAUX,IAAIsC,MAAM,CAAC3E,MAAM,EAAEqC,IAAIsC,MAAM,CAAC7E,UAAU,EAAEuC,IAAIsC,MAAM,CAACC,UAAU;gBACtF,mCAAA,4BAAA;;gBAAL,QAAK,aAAWW,2BAAX,UAAA,8BAAA,SAAA,0BAAA,kCAAmB;oBAAnB,IAAMd,IAAN;oBACH,IAAIH,MAAM;oBACV,IAAMuB,MAAM7G,KAAKqE,GAAG,CAACoB,EAAE9F,CAAC,CAAC3F,MAAM,EAAE4M,MAAM5M,MAAM;oBAC7C,IAAK,IAAI8F,IAAI,GAAGA,IAAI+G,KAAK/G,IAAKwF,OAAOG,EAAE9F,CAAC,CAACG,EAAE,GAAG8G,KAAK,CAAC9G,EAAE;oBACtD,IAAMmF,QAAQK,MAAMG,EAAE5C,KAAK,GAAGQ,IAAIR,KAAK;oBACvC,IAAM0C,WAAWF,KAAK7C,GAAG,CAACa,IAAI3H,IAAI;oBAClC,IAAI6J,aAAatD,aAAagD,QAAQM,UAAUF,KAAK5C,GAAG,CAACY,IAAI3H,IAAI,EAAEuJ;gBACrE;;gBAPK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAQP;;QAXK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAaL,OAAO,AAAC,qBAAGI,KAAK5E,OAAO,IACpBsF,IAAI,CAAC,SAACC,GAAGC;eAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;OAC1BjL,KAAK,CAAC,GAAGoL,KAAKxF,CAAC,EACfS,GAAG,CAAC;iDAAEmB,eAAG0C;eAAY;YAAEvJ,MAAM6G;YAAG2D,YAAYlB,SAASC;QAAO;;AACjE"}