sensemaking 0.16.0 → 0.17.0

package/dist/cjs/embed/langfit.js

@@ -16,6 +16,7 @@ _export(exports, {
         return languageDistribution;
     }
 });
+var _francmin = require("franc-min");
 var _sharedts = require("../db/shared.js");
 var _errorsts = require("../errors.js");
 var _languagests = require("./languages.js");
@@ -64,36 +65,6 @@ function _define_property(obj, key, value) {
     } else obj[key] = value;
     return obj;
 }
-function _getRequireWildcardCache(nodeInterop) {
-    if (typeof WeakMap !== "function") return null;
-    var cacheBabelInterop = new WeakMap();
-    var cacheNodeInterop = new WeakMap();
-    return (_getRequireWildcardCache = function(nodeInterop) {
-        return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
-    })(nodeInterop);
-}
-function _interop_require_wildcard(obj, nodeInterop) {
-    if (!nodeInterop && obj && obj.__esModule) return obj;
-    if (obj === null || typeof obj !== "object" && typeof obj !== "function") return {
-        default: obj
-    };
-    var cache = _getRequireWildcardCache(nodeInterop);
-    if (cache && cache.has(obj)) return cache.get(obj);
-    var newObj = {
-        __proto__: null
-    };
-    var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
-    for(var key in obj){
-        if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
-            var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
-            if (desc && (desc.get || desc.set)) Object.defineProperty(newObj, key, desc);
-            else newObj[key] = obj[key];
-        }
-    }
-    newObj.default = obj;
-    if (cache) cache.set(obj, newObj);
-    return newObj;
-}
 function _iterable_to_array_limit(arr, i) {
     var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
     if (_i == null) return;
@@ -257,69 +228,58 @@ function languageDistribution(db) {
     var raw = (0, _sharedts.getMeta)(db, META_KEY);
     return raw ? JSON.parse(raw) : undefined;
 }
-function classify(franc, text) {
-    var code = franc(text.slice(0, SAMPLE_CHARS));
+function classify(text) {
+    var code = (0, _francmin.franc)(text.slice(0, SAMPLE_CHARS));
     return code === 'und' ? undefined : code;
 }
 function checkLanguageFit(db, provider, texts) {
     return _async_to_generator(function() {
-        var _languageDistribution, franc, persisted, merged, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, text, _merged_code, code, total, declared, _Object_entries_sort_, majorityLang, majorityCount, pct;
+        var _languageDistribution, persisted, merged, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, text, _merged_code, code, total, declared, _Object_entries_sort_, majorityLang, majorityCount, pct;
         return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    if (texts.length === 0) return [
-                        2
-                    ];
-                    return [
-                        4,
-                        Promise.resolve().then(function() {
-                            return /*#__PURE__*/ _interop_require_wildcard(require("franc-min"));
-                        })
-                    ];
-                case 1:
-                    franc = _state.sent().franc;
-                    persisted = (_languageDistribution = languageDistribution(db)) !== null && _languageDistribution !== void 0 ? _languageDistribution : {};
-                    merged = _object_spread({}, persisted);
-                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                    try {
-                        for(_iterator = texts[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                            text = _step.value;
-                            ;
-                            code = classify(franc, text);
-                            if (code) merged[code] = ((_merged_code = merged[code]) !== null && _merged_code !== void 0 ? _merged_code : 0) + 1;
-                        }
-                    } catch (err) {
-                        _didIteratorError = true;
-                        _iteratorError = err;
-                    } finally{
-                        try {
-                            if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                _iterator.return();
-                            }
-                        } finally{
-                            if (_didIteratorError) {
-                                throw _iteratorError;
-                            }
-                        }
+            if (texts.length === 0) return [
+                2
+            ];
+            persisted = (_languageDistribution = languageDistribution(db)) !== null && _languageDistribution !== void 0 ? _languageDistribution : {};
+            merged = _object_spread({}, persisted);
+            _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+            try {
+                for(_iterator = texts[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                    text = _step.value;
+                    ;
+                    code = classify(text);
+                    if (code) merged[code] = ((_merged_code = merged[code]) !== null && _merged_code !== void 0 ? _merged_code : 0) + 1;
+                }
+            } catch (err) {
+                _didIteratorError = true;
+                _iteratorError = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion && _iterator.return != null) {
+                        _iterator.return();
                     }
-                    total = Object.values(merged).reduce(function(a, b) {
-                        return a + b;
-                    }, 0);
-                    if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {
-                        declared = new Set(provider.languages.map(_languagests.toIso3));
-                        _Object_entries_sort_ = _sliced_to_array(Object.entries(merged).sort(function(a, b) {
-                            return b[1] - a[1];
-                        })[0], 2), majorityLang = _Object_entries_sort_[0], majorityCount = _Object_entries_sort_[1];
-                        if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {
-                            pct = Math.round(majorityCount / total * 100);
-                            throw new _errorsts.SenseError('EMBED_MODEL_MISMATCH', "embed model ".concat(provider.id, " declares languages [").concat(provider.languages.join(', '), "], but ").concat(pct, "% of this tree's classified text is \"").concat(majorityLang, "\" (ISO 639-3), which is not among them; choose a model for this tree's languages: `sense init --model ...`, see the sense-setup skill or INTEGRATIONS.md"));
-                        }
+                } finally{
+                    if (_didIteratorError) {
+                        throw _iteratorError;
                     }
-                    (0, _sharedts.setMeta)(db, META_KEY, JSON.stringify(merged));
-                    return [
-                        2
-                    ];
+                }
+            }
+            total = Object.values(merged).reduce(function(a, b) {
+                return a + b;
+            }, 0);
+            if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {
+                declared = new Set(provider.languages.map(_languagests.toIso3));
+                _Object_entries_sort_ = _sliced_to_array(Object.entries(merged).sort(function(a, b) {
+                    return b[1] - a[1];
+                })[0], 2), majorityLang = _Object_entries_sort_[0], majorityCount = _Object_entries_sort_[1];
+                if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {
+                    pct = Math.round(majorityCount / total * 100);
+                    throw new _errorsts.SenseError('EMBED_MODEL_MISMATCH', "embed model ".concat(provider.id, " declares languages [").concat(provider.languages.join(', '), "], but ").concat(pct, "% of this tree's classified text is \"").concat(majorityLang, "\" (ISO 639-3), which is not among them; choose a model for this tree's languages: `sense init --model ...`, see the sense-setup skill or INTEGRATIONS.md"));
+                }
             }
+            (0, _sharedts.setMeta)(db, META_KEY, JSON.stringify(merged));
+            return [
+                2
+            ];
         });
     })();
 }

package/dist/cjs/embed/langfit.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getMeta, setMeta } from '../db/shared.ts';\nimport { SenseError } from '../errors.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst META_KEY = 'embed_languages';\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\ntype LangCounts = Record<string, number>;\n\nexport function languageDistribution(db: DatabaseSync): LangCounts | undefined {\n  const raw = getMeta(db, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nfunction classify(franc: (text: string) => string, text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const { franc } = await import('franc-min');\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(franc, text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  setMeta(db, META_KEY, JSON.stringify(merged));\n}\n"],"names":["checkLanguageFit","languageDistribution","META_KEY","SAMPLE_CHARS","MIN_CLASSIFIED","db","raw","getMeta","JSON","parse","undefined","classify","franc","text","code","slice","provider","texts","persisted","merged","total","declared","Object","majorityLang","majorityCount","pct","length","values","reduce","a","b","languages","Set","map","toIso3","entries","sort","has","Math","round","SenseError","id","join","setMeta","stringify"],"mappings":";;;;;;;;;;;QA4BsBA;eAAAA;;QAZNC;eAAAA;;;wBAfiB;wBACN;2BACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvB,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,8FAA8F;AAC9F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,IAAMC,iBAAiB;AAIhB,SAASH,qBAAqBI,EAAgB;IACnD,IAAMC,MAAMC,IAAAA,iBAAO,EAACF,IAAIH;IACxB,OAAOI,MAAME,KAAKC,KAAK,CAACH,OAAOI;AACjC;AAEA,SAASC,SAASC,KAA+B,EAAEC,IAAY;IAC7D,IAAMC,OAAOF,MAAMC,KAAKE,KAAK,CAAC,GAAGZ;IACjC,OAAOW,SAAS,QAAQJ,YAAYI;AACtC;AAIO,SAAed,iBAAiBK,EAAgB,EAAEW,QAAuB,EAAEC,KAAe;;YAG7EhB,uBADVW,OACFM,WACAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMN,MAEiBM,cADpBL,MAIFM,OAEEC,UACgCC,uBAA/BC,cAAcC,eAEbC;;;;oBAdV,IAAIR,MAAMS,MAAM,KAAK,GAAG;;;oBACN;;wBAAM;2EAAA,QAAO;;;;oBAAvBd,QAAU,cAAVA;oBACFM,aAAYjB,wBAAAA,qBAAqBI,iBAArBJ,mCAAAA,wBAA4B,CAAC;oBACzCkB,SAAS,mBAAKD;oBACf,kCAAA,2BAAA;;wBAAL,IAAK,YAAcD,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAfJ,OAAN;;4BACGC,OAAOH,SAASC,OAAOC;4BAC7B,IAAIC,MAAMK,MAAM,CAACL,KAAK,GAAG,EAACK,eAAAA,MAAM,CAACL,KAAK,cAAZK,0BAAAA,eAAgB,KAAK;wBACjD;;wBAHK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKCC,QAAQE,OAAOK,MAAM,CAACR,QAAQS,MAAM,CAAC,SAACC,GAAGC;+BAAMD,IAAIC;uBAAG;oBAC5D,IAAId,SAASe,SAAS,IAAIf,SAASe,SAAS,CAACL,MAAM,GAAG,KAAKN,SAAShB,gBAAgB;wBAC5EiB,WAAW,IAAIW,IAAIhB,SAASe,SAAS,CAACE,GAAG,CAACC,mBAAM;wBAChBZ,yCAAAA,OAAOa,OAAO,CAAChB,QAAQiB,IAAI,CAAC,SAACP,GAAGC;mCAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;0BAAC,CAAC,EAAE,MAApFN,eAA+BD,0BAAjBE,gBAAiBF;wBACtC,IAAIE,gBAAgBJ,SAAS,OAAO,CAACC,SAASgB,GAAG,CAACd,eAAe;4BACzDE,MAAMa,KAAKC,KAAK,CAAC,AAACf,gBAAgBJ,QAAS;4BACjD,MAAM,IAAIoB,oBAAU,CAClB,wBACA,AAAC,eAAiDxB,OAAnCA,SAASyB,EAAE,EAAC,yBAA8DhB,OAAvCT,SAASe,SAAS,CAACW,IAAI,CAAC,OAAM,WAAoDnB,OAA3CE,KAAI,0CAAoD,OAAbF,cAAa;wBAErJ;oBACF;oBACAoB,IAAAA,iBAAO,EAACtC,IAAIH,UAAUM,KAAKoC,SAAS,CAACzB;;;;;;IACvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { franc } from 'franc-min';\nimport { getMeta, setMeta } from '../db/shared.ts';\nimport { SenseError } from '../errors.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst META_KEY = 'embed_languages';\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\ntype LangCounts = Record<string, number>;\n\nexport function languageDistribution(db: DatabaseSync): LangCounts | undefined {\n  const raw = getMeta(db, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nfunction classify(text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  setMeta(db, META_KEY, JSON.stringify(merged));\n}\n"],"names":["checkLanguageFit","languageDistribution","META_KEY","SAMPLE_CHARS","MIN_CLASSIFIED","db","raw","getMeta","JSON","parse","undefined","classify","text","code","franc","slice","provider","texts","persisted","merged","total","declared","Object","majorityLang","majorityCount","pct","length","values","reduce","a","b","languages","Set","map","toIso3","entries","sort","has","Math","round","SenseError","id","join","setMeta","stringify"],"mappings":";;;;;;;;;;;QA6BsBA;eAAAA;;QAZNC;eAAAA;;;wBAhBM;wBACW;wBACN;2BACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvB,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,8FAA8F;AAC9F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,IAAMC,iBAAiB;AAIhB,SAASH,qBAAqBI,EAAgB;IACnD,IAAMC,MAAMC,IAAAA,iBAAO,EAACF,IAAIH;IACxB,OAAOI,MAAME,KAAKC,KAAK,CAACH,OAAOI;AACjC;AAEA,SAASC,SAASC,IAAY;IAC5B,IAAMC,OAAOC,IAAAA,eAAK,EAACF,KAAKG,KAAK,CAAC,GAAGZ;IACjC,OAAOU,SAAS,QAAQH,YAAYG;AACtC;AAIO,SAAeb,iBAAiBK,EAAgB,EAAEW,QAAuB,EAAEC,KAAe;;YAE7EhB,uBAAZiB,WACAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMP,MAEiBO,cADpBN,MAIFO,OAEEC,UACgCC,uBAA/BC,cAAcC,eAEbC;;YAbV,IAAIR,MAAMS,MAAM,KAAK,GAAG;;;YAClBR,aAAYjB,wBAAAA,qBAAqBI,iBAArBJ,mCAAAA,wBAA4B,CAAC;YACzCkB,SAAS,mBAAKD;YACf,kCAAA,2BAAA;;gBAAL,IAAK,YAAcD,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;oBAAfL,OAAN;;oBACGC,OAAOF,SAASC;oBACtB,IAAIC,MAAMM,MAAM,CAACN,KAAK,GAAG,EAACM,eAAAA,MAAM,CAACN,KAAK,cAAZM,0BAAAA,eAAgB,KAAK;gBACjD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAKCC,QAAQE,OAAOK,MAAM,CAACR,QAAQS,MAAM,CAAC,SAACC,GAAGC;uBAAMD,IAAIC;eAAG;YAC5D,IAAId,SAASe,SAAS,IAAIf,SAASe,SAAS,CAACL,MAAM,GAAG,KAAKN,SAAShB,gBAAgB;gBAC5EiB,WAAW,IAAIW,IAAIhB,SAASe,SAAS,CAACE,GAAG,CAACC,mBAAM;gBAChBZ,yCAAAA,OAAOa,OAAO,CAAChB,QAAQiB,IAAI,CAAC,SAACP,GAAGC;2BAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;kBAAC,CAAC,EAAE,MAApFN,eAA+BD,0BAAjBE,gBAAiBF;gBACtC,IAAIE,gBAAgBJ,SAAS,OAAO,CAACC,SAASgB,GAAG,CAACd,eAAe;oBACzDE,MAAMa,KAAKC,KAAK,CAAC,AAACf,gBAAgBJ,QAAS;oBACjD,MAAM,IAAIoB,oBAAU,CAClB,wBACA,AAAC,eAAiDxB,OAAnCA,SAASyB,EAAE,EAAC,yBAA8DhB,OAAvCT,SAASe,SAAS,CAACW,IAAI,CAAC,OAAM,WAAoDnB,OAA3CE,KAAI,0CAAoD,OAAbF,cAAa;gBAErJ;YACF;YACAoB,IAAAA,iBAAO,EAACtC,IAAIH,UAAUM,KAAKoC,SAAS,CAACzB;;;;;IACvC"}

package/dist/cjs/embed/languages.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/languages.ts"],"sourcesContent":["// HF model cards declare languages as ISO 639-1 tags (e.g. \"zh\"); franc-min reports ISO 639-3\n// (e.g. \"cmn\"). This bridges the two for comparison. Limited to franc-min's own 82-language\n// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min\n// can actually emit; a declared tag with no entry here is compared as-is.\nconst ISO_639_1_TO_3: Record<string, string> = {\n  ar: 'arb',\n  am: 'amh',\n  az: 'azj',\n  be: 'bel',\n  bg: 'bul',\n  bn: 'ben',\n  bs: 'bos',\n  cs: 'ces',\n  de: 'deu',\n  el: 'ell',\n  en: 'eng',\n  es: 'spa',\n  fa: 'pes',\n  fr: 'fra',\n  gu: 'guj',\n  ha: 'hau',\n  hi: 'hin',\n  hr: 'hrv',\n  hu: 'hun',\n  id: 'ind',\n  ig: 'ibo',\n  it: 'ita',\n  ja: 'jpn',\n  jv: 'jav',\n  kk: 'kaz',\n  kn: 'kan',\n  ko: 'kor',\n  ml: 'mal',\n  mr: 'mar',\n  ms: 'zlm',\n  my: 'mya',\n  ne: 'npi',\n  nl: 'nld',\n  pa: 'pan',\n  pl: 'pol',\n  pt: 'por',\n  ro: 'ron',\n  ru: 'rus',\n  rw: 'kin',\n  si: 'sin',\n  so: 'som',\n  sr: 'srp',\n  su: 'sun',\n  sv: 'swe',\n  sw: 'swh',\n  ta: 'tam',\n  te: 'tel',\n  th: 'tha',\n  tl: 'tgl',\n  tr: 'tur',\n  uk: 'ukr',\n  ur: 'urd',\n  uz: 'uzn',\n  vi: 'vie',\n  yo: 'yor',\n  zh: 'cmn',\n  zu: 'zul',\n};\n\nexport function isKnownLanguageTag(tag: string): boolean {\n  return tag in ISO_639_1_TO_3;\n}\n\n// Normalizes a declared or detected language code to ISO 639-3 for comparison; strips a\n// region/script subtag (\"zh-cn\" -> \"zh\") first. Codes already in 639-3 form pass through.\nexport function toIso3(code: string): string {\n  const base = code.toLowerCase().split('-')[0];\n  return ISO_639_1_TO_3[base] ?? base;\n}\n\n// Languages this project measured for models whose cards declare none, so the fit check is\n// not silently off for the shipped default; asserted here because it was measured, not attested.\nexport const MEASURED_MODEL_LANGUAGES: Record<string, string[]> = {\n  'minishlab/potion-retrieval-32M': ['en'], // BENCHMARKING.md bake-off; token-loss measurements\n};\n"],"names":["MEASURED_MODEL_LANGUAGES","isKnownLanguageTag","toIso3","ISO_639_1_TO_3","ar","am","az","be","bg","bn","bs","cs","de","el","en","es","fa","fr","gu","ha","hi","hr","hu","id","ig","it","ja","jv","kk","kn","ko","ml","mr","ms","my","ne","nl","pa","pl","pt","ro","ru","rw","si","so","sr","su","sv","sw","ta","te","th","tl","tr","uk","ur","uz","vi","yo","zh","zu","tag","code","base","toLowerCase","split"],"mappings":";;;;;;;;;;;QA6EaA;eAAAA;;QAbGC;eAAAA;;QAMAC;eAAAA;;;AAtEhB,8FAA8F;AAC9F,4FAA4F;AAC5F,gGAAgG;AAChG,0EAA0E;AAC1E,IAAMC,iBAAyC;IAC7CC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;AACN;AAEO,SAAS3D,mBAAmB4D,GAAW;IAC5C,OAAOA,OAAO1D;AAChB;AAIO,SAASD,OAAO4D,IAAY;QAE1B3D;IADP,IAAM4D,OAAOD,KAAKE,WAAW,GAAGC,KAAK,CAAC,IAAI,CAAC,EAAE;IAC7C,QAAO9D,uBAAAA,cAAc,CAAC4D,KAAK,cAApB5D,kCAAAA,uBAAwB4D;AACjC;AAIO,IAAM/D,2BAAqD;IAChE,kCAAkC;QAAC;KAAK;AAC1C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/languages.ts"],"sourcesContent":["// HF model cards declare languages as ISO 639-1 tags (e.g. \"zh\"); franc-min reports ISO 639-3\n// (e.g. \"cmn\"). This bridges the two for comparison. Limited to franc-min's own 82-language\n// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min\n// can actually emit; a declared tag with no entry here is compared as-is.\nconst ISO_639_1_TO_3: Record<string, string> = {\n  ar: 'arb',\n  am: 'amh',\n  az: 'azj',\n  be: 'bel',\n  bg: 'bul',\n  bn: 'ben',\n  bs: 'bos',\n  cs: 'ces',\n  de: 'deu',\n  el: 'ell',\n  en: 'eng',\n  es: 'spa',\n  fa: 'pes',\n  fr: 'fra',\n  gu: 'guj',\n  ha: 'hau',\n  hi: 'hin',\n  hr: 'hrv',\n  hu: 'hun',\n  id: 'ind',\n  ig: 'ibo',\n  it: 'ita',\n  ja: 'jpn',\n  jv: 'jav',\n  kk: 'kaz',\n  kn: 'kan',\n  ko: 'kor',\n  ml: 'mal',\n  mr: 'mar',\n  ms: 'zlm',\n  my: 'mya',\n  ne: 'npi',\n  nl: 'nld',\n  pa: 'pan',\n  pl: 'pol',\n  pt: 'por',\n  ro: 'ron',\n  ru: 'rus',\n  rw: 'kin',\n  si: 'sin',\n  so: 'som',\n  sr: 'srp',\n  su: 'sun',\n  sv: 'swe',\n  sw: 'swh',\n  ta: 'tam',\n  te: 'tel',\n  th: 'tha',\n  tl: 'tgl',\n  tr: 'tur',\n  uk: 'ukr',\n  ur: 'urd',\n  uz: 'uzn',\n  vi: 'vie',\n  yo: 'yor',\n  zh: 'cmn',\n  zu: 'zul',\n};\n\nexport function isKnownLanguageTag(tag: string): boolean {\n  return tag in ISO_639_1_TO_3;\n}\n\n// Normalizes a declared or detected language code to ISO 639-3 for comparison; strips a\n// region/script subtag (\"zh-cn\" -> \"zh\") first. Codes already in 639-3 form pass through.\nexport function toIso3(code: string): string {\n  const base = code.toLowerCase().split('-')[0];\n  return ISO_639_1_TO_3[base] ?? base;\n}\n\n// Languages this project measured for models whose cards declare none, so the fit check is\n// not silently off for the shipped default; asserted here because it was measured, not attested.\nexport const MEASURED_MODEL_LANGUAGES: Record<string, string[]> = {\n  'minishlab/potion-retrieval-32M': ['en'], // benchmark/reports/2026-08-27-embedding-model-selection.md\n};\n"],"names":["MEASURED_MODEL_LANGUAGES","isKnownLanguageTag","toIso3","ISO_639_1_TO_3","ar","am","az","be","bg","bn","bs","cs","de","el","en","es","fa","fr","gu","ha","hi","hr","hu","id","ig","it","ja","jv","kk","kn","ko","ml","mr","ms","my","ne","nl","pa","pl","pt","ro","ru","rw","si","so","sr","su","sv","sw","ta","te","th","tl","tr","uk","ur","uz","vi","yo","zh","zu","tag","code","base","toLowerCase","split"],"mappings":";;;;;;;;;;;QA6EaA;eAAAA;;QAbGC;eAAAA;;QAMAC;eAAAA;;;AAtEhB,8FAA8F;AAC9F,4FAA4F;AAC5F,gGAAgG;AAChG,0EAA0E;AAC1E,IAAMC,iBAAyC;IAC7CC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;AACN;AAEO,SAAS3D,mBAAmB4D,GAAW;IAC5C,OAAOA,OAAO1D;AAChB;AAIO,SAASD,OAAO4D,IAAY;QAE1B3D;IADP,IAAM4D,OAAOD,KAAKE,WAAW,GAAGC,KAAK,CAAC,IAAI,CAAC,EAAE;IAC7C,QAAO9D,uBAAAA,cAAc,CAAC4D,KAAK,cAApB5D,kCAAAA,uBAAwB4D;AACjC;AAIO,IAAM/D,2BAAqD;IAChE,kCAAkC;QAAC;KAAK;AAC1C"}

package/dist/cjs/embed/query.js

@@ -28,8 +28,8 @@ _export(exports, {
 var _nodepath = require("node:path");
 var _errorsts = require("../errors.js");
 var _embedts = require("../features/embed.js");
-var _progressts = require("../progress.js");
-var _scants = require("../scan.js");
+var _progressts = require("../output/progress.js");
+var _indexts = require("../scan/index.js");
 var _langfitts = require("./langfit.js");
 var _registryts = require("./registry.js");
 function _array_like_to_array(arr, len) {
@@ -237,8 +237,8 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is
-// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
+// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):
+// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
 var STORE_DIMS = 256;
 // Seed chunks that participate in a `related` scan; see similarNotes for the measurement.
 var TARGET_CHUNK_CAP = 16;
@@ -343,7 +343,7 @@ function embedPending(db, cfg, baseDir) {
                             try {
                                 // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already
                                 // has embeddings rows means the tree had an embedding model at reconcile time.
-                                chunks = (0, _scants.parseFile)({
+                                chunks = (0, _indexts.parseFile)({
                                     relPath: path,
                                     absPath: (0, _nodepath.join)(baseDir, path),
                                     mtimeMs: 0,
@@ -353,7 +353,7 @@ function embedPending(db, cfg, baseDir) {
                                     embed: true
                                 }, [
                                     _embedts.embed
-                                ]).doc.extracted.embed;
+                                ], cfg).doc.extracted.embed;
                             } catch (unused) {
                                 continue; // vanished since reconcile; the next reconcile removes its rows
                             }

package/dist/cjs/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["embedPending","hasEmbedding","semanticCandidates","similarNotes","toStore","STORE_DIMS","TARGET_CHUNK_CAP","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","db","cfg","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","path","chunkIdxs","chunks","idx","update","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","chunk","err","tick","min","length","getProvider","prepare","all","Map","get","push","set","parseFile","relPath","absPath","join","mtimeMs","ctimeMs","size","presets","embed","doc","extracted","checkLanguageFit","progress","finish","asCosine","score","terms","fetch","allowed","qv","rows","best","dot","existing","SenseError","match","filter","t","includes","embedQuery","has","vector","byteOffset","byteLength","lines","start_line","end_line","entries","sort","a","b","similarity","undefined","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":";;;;;;;;;;;QAiCsBA;eAAAA;;QAwGNC;eAAAA;;QApCMC;eAAAA;;QA2CNC;eAAAA;;QA7HAC;eAAAA;;;wBAnBK;wBAGM;uBAEL;0BACG;sBACC;yBACO;0BACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,0FAA0F;AAC1F,IAAMC,mBAAmB;AAIlB,SAASF,QAAQG,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAejB,aAAamB,EAAgB,EAAEC,GAAW,EAAEC,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOC,MAAMC,WACZC,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,KAWPC,QAGAC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQR,KAAKU,KAAK,CAACH,GAAGA,IAAIb,SAASiB,QAAQ;oCACjC;;wCAAMjB,SAASkB,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChBlB,GAAGyB,IAAI,CAAC;oCACR,IAAI;wCACFR,MAAMS,OAAO,CAAC,SAACC,KAAKJ;4CAClB,IAAqBtC,WAAAA,QAAQiC,OAAO,CAACK,EAAE,EAAElB,WAAW,OAA5Cd,IAAaN,SAAbM,GAAGM,QAAUZ,SAAVY;4CACX,IAAM+B,IAAI,IAAIC,UAAUxB;4CACxB,IAAK,IAAIX,IAAI,GAAGA,IAAIW,WAAWX,IAAKkC,CAAC,CAAClC,EAAE,GAAGC,KAAKmC,KAAK,CAACvC,CAAC,CAACG,EAAE,GAAGG;4CAC7DiB,OAAOiB,GAAG,CAAClC,OAAOmC,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAIjB,IAAI,EAAEiB,IAAIQ,KAAK;wCAC9D;wCACAnC,GAAGyB,IAAI,CAAC;oCACV,EAAE,OAAOW,KAAK;wCACZpC,GAAGyB,IAAI,CAAC;wCACR,MAAMW;oCACR;oCACArB,OAAOsB,IAAI,CAAC1C,KAAK2C,GAAG,CAACtB,IAAIb,SAASiB,QAAQ,EAAEX,KAAK8B,MAAM;;;;;;oBACzD;oBAtDiB;;wBAAMC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXC,QAAQJ,GAAGyC,OAAO,CAAC,oFAAoFC,GAAG;oBAChH,IAAItC,MAAMmC,MAAM,KAAK,GAAG;;;oBAClBlC,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAE9CiB,SAAS,IAAIqC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAavC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOsC,GAAG,CAACrC,IAAIG,IAAI,eAAnBJ,yBAAAA;4BACbE,KAAKqC,IAAI,CAACtC,IAAI4B,KAAK;4BACnB7B,OAAOwC,GAAG,CAACvC,IAAIG,IAAI,EAAEF;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOI,uBAAMC;4BACZC,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,+EAA+E;gCAC/EA,SAASmC,IAAAA,iBAAS,EAAC;oCAAEC,SAAStC;oCAAMuC,SAASC,IAAAA,cAAI,EAAChD,SAASQ;oCAAOyC,SAAS;oCAAGC,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMC,OAAO;gCAAK;oCAAIA,cAAK;mCAAGC,GAAG,CAACC,SAAS,CAACF,KAAK;4BAC7J,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa5C,gCAAb,8BAAA,SAAA,0BAAA;oCAAME,MAAN;oCAAwB,IAAID,MAAM,CAACC,IAAI,EAAEJ,KAAKoC,IAAI,CAAC;wCAAEnC,MAAAA;wCAAMyB,OAAOtB;wCAAKW,MAAMZ,MAAM,CAACC,IAAI,CAACW,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAMkC,IAAAA,2BAAgB,EACpB1D,IACAG,UACAM,KAAKa,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMMV,SAASd,GAAGyC,OAAO,CAAC;oBAC1B,8EAA8E;oBAC9E,6DAA6D;oBACvD1B,SAAS4C,IAAAA,oBAAQ,EAAC,oBAAoBlD,KAAK8B,MAAM;oBAC9CvB,IAAI;;;yBAAGA,CAAAA,IAAIP,KAAK8B,MAAM,AAAD;;;;;;;;;;;;oBAAGvB,KAAKb,SAASiB,QAAQ;;;;;;oBAkBvDL,OAAO6C,MAAM;;;;;;IACf;;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAOnE,KAAKmC,KAAK,CAACnC,KAAK2C,GAAG,CAAC,GAAG3C,KAAKG,GAAG,CAAC,CAAC,GAAGgE,UAAU,QAAQ;AAC/D;AAKO,SAAe/E,mBAAmBiB,EAAgB,EAAEC,GAAW,EAAE8D,KAAa,EAAEC,KAAa,EAAEC,OAAqB;;YAO3GF,cANR7D,SAIAC,UACAE,WACAmB,MACYvC,UAAPiF,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM7D,KAEHqB,GACFyC,KACK3E,GACHoE,OACAQ;;;;oBAxBFpE,UAAU,AAACD,IAAgCC,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIqE,oBAAU,CAAC,eAAe;oBAClD;;wBAAM1F,aAAamB,IAAIC,KAAKC;;;oBAA5B;oBAEiB;;wBAAMsC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXE,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAC9CmC,OAAO,EAACuC,eAAAA,MAAMS,KAAK,CAAC,2naAAZT,0BAAAA,mBAAsCU,MAAM,CAAC,SAACC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIxB,IAAI,CAAC;oBACxF;;wBAAM/C,SAASyE,UAAU,CAACpD;;;oBAAlCvC,WAAAA;wBAAQ;wBAAiCoB;wBAAW;wBAA3D6D,KAAOjF,SAAVM;oBAEF4E,OAAOnE,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG;oBAQpH0B,OAAO,IAAIzB;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAawB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAb5D,MAAN;4BACH,IAAI0D,WAAW,CAACA,QAAQY,GAAG,CAACtE,IAAIG,IAAI,GAAG;4BACjCkB,IAAI,IAAIC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAEpF,KAAK2C,GAAG,CAACjC,WAAWE,IAAIuE,MAAM,CAACE,UAAU;4BACvGX,MAAM;4BACV,IAAS3E,IAAI,GAAGA,IAAIkC,EAAEW,MAAM,EAAE7C,IAAK2E,OAAOzC,CAAC,CAAClC,EAAE,GAAGwE,EAAE,CAACxE,EAAE;4BAChDoE,QAAQO,MAAM9D,IAAIV,KAAK;4BACvByE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;4BAClC,IAAI,CAAC4D,YAAYR,QAAQQ,SAASR,KAAK,EAAEM,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAE;gCAAEoD,OAAAA;gCAAOmB,OAAO,AAAC,IAAqB1E,OAAlBA,IAAI2E,UAAU,EAAC,KAAgB,OAAb3E,IAAI4E,QAAQ;4BAAG;wBACnH;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBASL;;wBAAQ,qBAAGf,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK;2BACtC3C,KAAK,CAAC,GAAG6C,OACT1C,GAAG,CAAC;qEAAEZ,kBAAM6E;mCAAQ;gCAAE7E,MAAAA;gCAAMuE,OAAOM,EAAEN,KAAK;gCAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAAShF,aAAakB,EAAgB,EAAEU,IAAY;IACzD,IAAMH,MAAMP,GAAGyC,OAAO,CAAC,kFAAkFG,GAAG,CAAClC;IAC7G,OAAOH,QAAQkF;AACjB;AAIO,SAASzG,aAAagB,EAAgB,EAAE0F,IAAY,EAAEhF,IAAY,EAAEiF,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,IAAMC,aAAa5F,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG,CAAChC;IACjI,IAAIkF,WAAWrD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,IAAMsD,OAAOlG,KAAKG,GAAG,CAAC,GAAGH,KAAKmG,IAAI,CAACF,WAAWrD,MAAM,GAAGpD;IACvD,IAAM4G,SAASH,WAAWnB,MAAM,CAAC,SAACuB,GAAGhF;eAAMA,IAAI6E,SAAS;OAAGvE,GAAG,CAAC,SAACf;eAAS;YAAEhB,GAAG,IAAIsC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;YAAGnF,OAAOU,IAAIV,KAAK;QAAC;;IAE/K,IAAMsE,OAAOnE,GAAGyC,OAAO,CAAC,yEAAyEC,GAAG;IACpG,IAAM0B,OAAO,IAAIzB;QACZ,kCAAA,2BAAA;;QAAL,QAAK,YAAawB,yBAAb,SAAA,6BAAA,QAAA,yBAAA,iCAAmB;YAAnB,IAAM5D,MAAN;YACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQiF,KAAKM,OAAO,CAACpB,GAAG,CAACtE,IAAIG,IAAI,KAAMiF,KAAK1B,OAAO,IAAI,CAAC0B,KAAK1B,OAAO,CAACY,GAAG,CAACtE,IAAIG,IAAI,GAAI;YACtG,IAAMwF,QAAQ,IAAIrE,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;gBACtF,mCAAA,4BAAA;;gBAAL,QAAK,aAAWe,2BAAX,UAAA,8BAAA,SAAA,0BAAA,kCAAmB;oBAAnB,IAAMrB,IAAN;oBACH,IAAIL,MAAM;oBACV,IAAM8B,MAAMxG,KAAK2C,GAAG,CAACoC,EAAEnF,CAAC,CAACgD,MAAM,EAAE2D,MAAM3D,MAAM;oBAC7C,IAAK,IAAI7C,IAAI,GAAGA,IAAIyG,KAAKzG,IAAK2E,OAAOK,EAAEnF,CAAC,CAACG,EAAE,GAAGwG,KAAK,CAACxG,EAAE;oBACtD,IAAMoE,QAAQO,MAAMK,EAAE7E,KAAK,GAAGU,IAAIV,KAAK;oBACvC,IAAMyE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;oBAClC,IAAI4D,aAAamB,aAAa3B,QAAQQ,UAAUF,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAEoD;gBACrE;;gBAPK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAQP;;QAXK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAaL,OAAO,AAAC,qBAAGM,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;eAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;OAC1BnE,KAAK,CAAC,GAAGwE,KAAKS,CAAC,EACf9E,GAAG,CAAC;iDAAE+E,eAAGvC;eAAY;YAAEpD,MAAM2F;YAAGb,YAAY3B,SAASC;QAAO;;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["embedPending","hasEmbedding","semanticCandidates","similarNotes","toStore","STORE_DIMS","TARGET_CHUNK_CAP","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","db","cfg","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","path","chunkIdxs","chunks","idx","update","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","chunk","err","tick","min","length","getProvider","prepare","all","Map","get","push","set","parseFile","relPath","absPath","join","mtimeMs","ctimeMs","size","presets","embed","doc","extracted","checkLanguageFit","progress","finish","asCosine","score","terms","fetch","allowed","qv","rows","best","dot","existing","SenseError","match","filter","t","includes","embedQuery","has","vector","byteOffset","byteLength","lines","start_line","end_line","entries","sort","a","b","similarity","undefined","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":";;;;;;;;;;;QAiCsBA;eAAAA;;QAwGNC;eAAAA;;QApCMC;eAAAA;;QA2CNC;eAAAA;;QA7HAC;eAAAA;;;wBAnBK;wBAGM;uBAEL;0BACG;uBACC;yBACO;0BACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8FAA8F;AAC9F,6FAA6F;AAC7F,IAAMC,aAAa;AACnB,0FAA0F;AAC1F,IAAMC,mBAAmB;AAIlB,SAASF,QAAQG,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAejB,aAAamB,EAAgB,EAAEC,GAAW,EAAEC,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOC,MAAMC,WACZC,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,KAWPC,QAGAC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQR,KAAKU,KAAK,CAACH,GAAGA,IAAIb,SAASiB,QAAQ;oCACjC;;wCAAMjB,SAASkB,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChBlB,GAAGyB,IAAI,CAAC;oCACR,IAAI;wCACFR,MAAMS,OAAO,CAAC,SAACC,KAAKJ;4CAClB,IAAqBtC,WAAAA,QAAQiC,OAAO,CAACK,EAAE,EAAElB,WAAW,OAA5Cd,IAAaN,SAAbM,GAAGM,QAAUZ,SAAVY;4CACX,IAAM+B,IAAI,IAAIC,UAAUxB;4CACxB,IAAK,IAAIX,IAAI,GAAGA,IAAIW,WAAWX,IAAKkC,CAAC,CAAClC,EAAE,GAAGC,KAAKmC,KAAK,CAACvC,CAAC,CAACG,EAAE,GAAGG;4CAC7DiB,OAAOiB,GAAG,CAAClC,OAAOmC,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAIjB,IAAI,EAAEiB,IAAIQ,KAAK;wCAC9D;wCACAnC,GAAGyB,IAAI,CAAC;oCACV,EAAE,OAAOW,KAAK;wCACZpC,GAAGyB,IAAI,CAAC;wCACR,MAAMW;oCACR;oCACArB,OAAOsB,IAAI,CAAC1C,KAAK2C,GAAG,CAACtB,IAAIb,SAASiB,QAAQ,EAAEX,KAAK8B,MAAM;;;;;;oBACzD;oBAtDiB;;wBAAMC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXC,QAAQJ,GAAGyC,OAAO,CAAC,oFAAoFC,GAAG;oBAChH,IAAItC,MAAMmC,MAAM,KAAK,GAAG;;;oBAClBlC,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAE9CiB,SAAS,IAAIqC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAavC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOsC,GAAG,CAACrC,IAAIG,IAAI,eAAnBJ,yBAAAA;4BACbE,KAAKqC,IAAI,CAACtC,IAAI4B,KAAK;4BACnB7B,OAAOwC,GAAG,CAACvC,IAAIG,IAAI,EAAEF;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOI,uBAAMC;4BACZC,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,+EAA+E;gCAC/EA,SAASmC,IAAAA,kBAAS,EAAC;oCAAEC,SAAStC;oCAAMuC,SAASC,IAAAA,cAAI,EAAChD,SAASQ;oCAAOyC,SAAS;oCAAGC,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMC,OAAO;gCAAK;oCAAIA,cAAK;mCAAGtD,KAAKuD,GAAG,CAACC,SAAS,CAACF,KAAK;4BAClK,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa5C,gCAAb,8BAAA,SAAA,0BAAA;oCAAME,MAAN;oCAAwB,IAAID,MAAM,CAACC,IAAI,EAAEJ,KAAKoC,IAAI,CAAC;wCAAEnC,MAAAA;wCAAMyB,OAAOtB;wCAAKW,MAAMZ,MAAM,CAACC,IAAI,CAACW,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAMkC,IAAAA,2BAAgB,EACpB1D,IACAG,UACAM,KAAKa,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMMV,SAASd,GAAGyC,OAAO,CAAC;oBAC1B,8EAA8E;oBAC9E,6DAA6D;oBACvD1B,SAAS4C,IAAAA,oBAAQ,EAAC,oBAAoBlD,KAAK8B,MAAM;oBAC9CvB,IAAI;;;yBAAGA,CAAAA,IAAIP,KAAK8B,MAAM,AAAD;;;;;;;;;;;;oBAAGvB,KAAKb,SAASiB,QAAQ;;;;;;oBAkBvDL,OAAO6C,MAAM;;;;;;IACf;;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAOnE,KAAKmC,KAAK,CAACnC,KAAK2C,GAAG,CAAC,GAAG3C,KAAKG,GAAG,CAAC,CAAC,GAAGgE,UAAU,QAAQ;AAC/D;AAKO,SAAe/E,mBAAmBiB,EAAgB,EAAEC,GAAW,EAAE8D,KAAa,EAAEC,KAAa,EAAEC,OAAqB;;YAO3GF,cANR7D,SAIAC,UACAE,WACAmB,MACYvC,UAAPiF,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM7D,KAEHqB,GACFyC,KACK3E,GACHoE,OACAQ;;;;oBAxBFpE,UAAU,AAACD,IAAgCC,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIqE,oBAAU,CAAC,eAAe;oBAClD;;wBAAM1F,aAAamB,IAAIC,KAAKC;;;oBAA5B;oBAEiB;;wBAAMsC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXE,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAC9CmC,OAAO,EAACuC,eAAAA,MAAMS,KAAK,CAAC,2naAAZT,0BAAAA,mBAAsCU,MAAM,CAAC,SAACC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIxB,IAAI,CAAC;oBACxF;;wBAAM/C,SAASyE,UAAU,CAACpD;;;oBAAlCvC,WAAAA;wBAAQ;wBAAiCoB;wBAAW;wBAA3D6D,KAAOjF,SAAVM;oBAEF4E,OAAOnE,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG;oBAQpH0B,OAAO,IAAIzB;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAawB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAb5D,MAAN;4BACH,IAAI0D,WAAW,CAACA,QAAQY,GAAG,CAACtE,IAAIG,IAAI,GAAG;4BACjCkB,IAAI,IAAIC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAEpF,KAAK2C,GAAG,CAACjC,WAAWE,IAAIuE,MAAM,CAACE,UAAU;4BACvGX,MAAM;4BACV,IAAS3E,IAAI,GAAGA,IAAIkC,EAAEW,MAAM,EAAE7C,IAAK2E,OAAOzC,CAAC,CAAClC,EAAE,GAAGwE,EAAE,CAACxE,EAAE;4BAChDoE,QAAQO,MAAM9D,IAAIV,KAAK;4BACvByE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;4BAClC,IAAI,CAAC4D,YAAYR,QAAQQ,SAASR,KAAK,EAAEM,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAE;gCAAEoD,OAAAA;gCAAOmB,OAAO,AAAC,IAAqB1E,OAAlBA,IAAI2E,UAAU,EAAC,KAAgB,OAAb3E,IAAI4E,QAAQ;4BAAG;wBACnH;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBASL;;wBAAQ,qBAAGf,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK;2BACtC3C,KAAK,CAAC,GAAG6C,OACT1C,GAAG,CAAC;qEAAEZ,kBAAM6E;mCAAQ;gCAAE7E,MAAAA;gCAAMuE,OAAOM,EAAEN,KAAK;gCAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAAShF,aAAakB,EAAgB,EAAEU,IAAY;IACzD,IAAMH,MAAMP,GAAGyC,OAAO,CAAC,kFAAkFG,GAAG,CAAClC;IAC7G,OAAOH,QAAQkF;AACjB;AAIO,SAASzG,aAAagB,EAAgB,EAAE0F,IAAY,EAAEhF,IAAY,EAAEiF,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,IAAMC,aAAa5F,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG,CAAChC;IACjI,IAAIkF,WAAWrD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,IAAMsD,OAAOlG,KAAKG,GAAG,CAAC,GAAGH,KAAKmG,IAAI,CAACF,WAAWrD,MAAM,GAAGpD;IACvD,IAAM4G,SAASH,WAAWnB,MAAM,CAAC,SAACuB,GAAGhF;eAAMA,IAAI6E,SAAS;OAAGvE,GAAG,CAAC,SAACf;eAAS;YAAEhB,GAAG,IAAIsC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;YAAGnF,OAAOU,IAAIV,KAAK;QAAC;;IAE/K,IAAMsE,OAAOnE,GAAGyC,OAAO,CAAC,yEAAyEC,GAAG;IACpG,IAAM0B,OAAO,IAAIzB;QACZ,kCAAA,2BAAA;;QAAL,QAAK,YAAawB,yBAAb,SAAA,6BAAA,QAAA,yBAAA,iCAAmB;YAAnB,IAAM5D,MAAN;YACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQiF,KAAKM,OAAO,CAACpB,GAAG,CAACtE,IAAIG,IAAI,KAAMiF,KAAK1B,OAAO,IAAI,CAAC0B,KAAK1B,OAAO,CAACY,GAAG,CAACtE,IAAIG,IAAI,GAAI;YACtG,IAAMwF,QAAQ,IAAIrE,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;gBACtF,mCAAA,4BAAA;;gBAAL,QAAK,aAAWe,2BAAX,UAAA,8BAAA,SAAA,0BAAA,kCAAmB;oBAAnB,IAAMrB,IAAN;oBACH,IAAIL,MAAM;oBACV,IAAM8B,MAAMxG,KAAK2C,GAAG,CAACoC,EAAEnF,CAAC,CAACgD,MAAM,EAAE2D,MAAM3D,MAAM;oBAC7C,IAAK,IAAI7C,IAAI,GAAGA,IAAIyG,KAAKzG,IAAK2E,OAAOK,EAAEnF,CAAC,CAACG,EAAE,GAAGwG,KAAK,CAACxG,EAAE;oBACtD,IAAMoE,QAAQO,MAAMK,EAAE7E,KAAK,GAAGU,IAAIV,KAAK;oBACvC,IAAMyE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;oBAClC,IAAI4D,aAAamB,aAAa3B,QAAQQ,UAAUF,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAEoD;gBACrE;;gBAPK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAQP;;QAXK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAaL,OAAO,AAAC,qBAAGM,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;eAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;OAC1BnE,KAAK,CAAC,GAAGwE,KAAKS,CAAC,EACf9E,GAAG,CAAC;iDAAE+E,eAAGvC;eAAY;YAAEpD,MAAM2F;YAAGb,YAAY3B,SAASC;QAAO;;AACjE"}

package/dist/cjs/embed/static.js

@@ -10,6 +10,7 @@ Object.defineProperty(exports, "staticProvider", {
 });
 var _nodefs = require("node:fs");
 var _nodepath = require("node:path");
+var _tokenizers = require("@huggingface/tokenizers");
 var _errorsts = require("../errors.js");
 var _storets = require("./store.js");
 function _array_like_to_array(arr, len) {
@@ -46,36 +47,6 @@ function _async_to_generator(fn) {
         });
     };
 }
-function _getRequireWildcardCache(nodeInterop) {
-    if (typeof WeakMap !== "function") return null;
-    var cacheBabelInterop = new WeakMap();
-    var cacheNodeInterop = new WeakMap();
-    return (_getRequireWildcardCache = function(nodeInterop) {
-        return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
-    })(nodeInterop);
-}
-function _interop_require_wildcard(obj, nodeInterop) {
-    if (!nodeInterop && obj && obj.__esModule) return obj;
-    if (obj === null || typeof obj !== "object" && typeof obj !== "function") return {
-        default: obj
-    };
-    var cache = _getRequireWildcardCache(nodeInterop);
-    if (cache && cache.has(obj)) return cache.get(obj);
-    var newObj = {
-        __proto__: null
-    };
-    var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
-    for(var key in obj){
-        if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
-            var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
-            if (desc && (desc.get || desc.set)) Object.defineProperty(newObj, key, desc);
-            else newObj[key] = obj[key];
-        }
-    }
-    newObj.default = obj;
-    if (cache) cache.set(obj, newObj);
-    return newObj;
-}
 function _iterable_to_array_limit(arr, i) {
     var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
     if (_i == null) return;
@@ -216,7 +187,7 @@ function _unsupported_iterable_to_array(o, minLen) {
 var BATCH_CAP = 64;
 function staticProvider(model) {
     return _async_to_generator(function() {
-        var _ref, _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1, dir, languages, cached, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, raw, headerLen, header, entry, spec, dims, dataStart, matrix, tokenizerJson, Tokenizer, tok, unkId;
+        var _ref, _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1, dir, languages, cached, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, raw, headerLen, header, entry, spec, dims, dataStart, matrix, tokenizerJson, tok, unkId;
         function one(text) {
             // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an
             // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the
@@ -320,15 +291,7 @@ function staticProvider(model) {
                     dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];
                     matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));
                     tokenizerJson = JSON.parse((0, _nodefs.readFileSync)((0, _nodepath.join)(dir, 'tokenizer.json'), 'utf8'));
-                    return [
-                        4,
-                        Promise.resolve().then(function() {
-                            return /*#__PURE__*/ _interop_require_wildcard(require("@huggingface/tokenizers"));
-                        })
-                    ];
-                case 4:
-                    Tokenizer = _state.sent().Tokenizer;
-                    tok = new Tokenizer(tokenizerJson, {});
+                    tok = new _tokenizers.Tokenizer(tokenizerJson, {});
                     unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
                     // Symmetric model: document and query embedding are the same call.
                     return [

package/dist/cjs/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["staticProvider","BATCH_CAP","model","tokenizerJson","dir","languages","cached","file","raw","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","text","ids","encode","add_special_tokens","filter","id","Number","isInteger","v","Float32Array","length","off","d","norm","Math","sqrt","isDownloadable","downloadModel","into","console","error","readLanguages","undefined","modelDir","MODEL_FILES","existsSync","join","SenseError","MODEL_FILENAMES","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","slice","vocab","unk_token","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":";;;;+BAUsBA;;;eAAAA;;;sBAVmB;wBACpB;wBACM;uBAC0E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGrG,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa;;kBAgCLC,sBAA7BA,4BAAAA,uBA/BVC,KACAC,WAKIC,QAID,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAOPC,KACAC,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAZ,eAEEa,WACFC,KACAC;QAEN,SAASC,IAAIC,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMC,MAAM,AAACJ,IAAIK,MAAM,CAACF,MAAM;gBAAEG,oBAAoB;YAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,SAACC;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOP;;YAC5H,IAAMU,IAAI,IAAIC,aAAahB;YAC3B,IAAIQ,IAAIS,MAAM,KAAK,GAAG,OAAOF;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYP,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMI,KAAN;oBACH,IAAMM,MAAMN,KAAKZ;oBACjB,IAAK,IAAImB,IAAI,GAAGA,IAAInB,MAAMmB,IAAKJ,CAAC,CAACI,EAAE,IAAIjB,MAAM,CAACgB,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAK;gBAC7BJ,CAAC,CAACI,GAAE,IAAIX,IAAIS,MAAM;gBAClBG,QAAQL,CAAC,CAACI,GAAE,GAAGJ,CAAC,CAACI,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAKJ,CAAC,CAACI,GAAE,IAAIC;YACvC,OAAOL;QACT;;;;yBAlDIQ,IAAAA,uBAAc,EAAClC,QAAfkC;;;;oBAEI;;wBAAMC,IAAAA,sBAAa,EAACnC,OAAO,SAACK,MAAM+B;mCAASC,QAAQC,KAAK,CAAC,AAAC,mBAA2BjC,OAATL,OAAM,KAAgBoC,OAAb/B,MAAK,UAAa,OAAL+B;;;;oBADxG,qFAAqF;oBACrFlC,MAAM;oBACN,uFAAuF;oBACjFE,SAASmC,IAAAA,sBAAa,EAACvC;oBAC7BG,YAAYC,UAAUA,OAAOwB,MAAM,GAAG,IAAIxB,SAASoC;;;;;;oBAEnDtC,MAAMuC,IAAAA,iBAAQ,EAACzC;oBACV,kCAAA,2BAAA;;wBAAL,IAAK,YAAc0C,oBAAW,uBAAzB,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBrC,OAAN;4BACH,IAAI,CAACsC,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAAC1C,KAAKG,QAAQ;gCAChC,MAAM,IAAIwC,oBAAU,CAAC,uBAAuB,AAAC,eAAmD3C,OAArCF,OAAM,iCAAiD8C,OAAlB5C,KAAI,gBAA8B,OAAhB4C,wBAAe,EAAC;4BACpI;wBACF;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAODxC,MAAMyC,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK;oBAC7BK,YAAYiB,OAAOlB,IAAI0C,eAAe,CAAC;oBACvCxC,SAASyC,KAAKC,KAAK,CAAC5C,IAAI6C,QAAQ,CAAC,GAAG,IAAI5C,WAAW6C,QAAQ,CAAC;oBAC5D3C,QAAQ4C,OAAOC,OAAO,CAAC9C,QAAQ+C,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC/C,SAASA,KAAK,CAAC,EAAE,CAACgD,KAAK,KAAK,OAAO,MAAM,IAAIZ,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAAN7C,OAAM;oBAC/EU,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKgD,KAAK,CAAC,EAAE;oBACpB9C,YAAYN,IAAIqD,UAAU,GAAG,IAAIpD,YAAYG,KAAKkD,YAAY,CAAC,EAAE;oBACjE/C,SAASD,YAAY,MAAM,IAAI,IAAIe,aAAarB,IAAIuD,MAAM,EAAEjD,WAAWF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,QAAQ,IAAIgB,aAAarB,IAAIuD,MAAM,CAACC,KAAK,CAAClD,WAAWA,YAAYF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,OAAO;oBAE/KV,gBAAgBgD,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BY,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUb,eAAe,CAAC;oBACpCe,iBAAQf,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqB8D,KAAK,cAA1B9D,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqB+D,SAAS,CAAC,uCAAI,CAAC;oBAuB/E,mEAAmE;oBACnE;;wBAAO;4BAAEzC,IAAI,AAAC,UAAe,OAANvB;4BAASW,MAAAA;4BAAMsD,UAAUlE;4BAAWI,WAAAA;4BAAW+D,gBAAgB,SAAhBA,eAAuBC;;;;;4CAAUA,MAAMC,GAAG,CAACnD;;;;;4BAAMoD,YAAY,SAAZA,WAAmBnD;;;;;4CAASD,IAAIC;;;;;wBAAM;;;;IAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Tokenizer } from '@huggingface/tokenizers';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["staticProvider","BATCH_CAP","model","tokenizerJson","dir","languages","cached","file","raw","headerLen","header","entry","spec","dims","dataStart","matrix","tok","unkId","one","text","ids","encode","add_special_tokens","filter","id","Number","isInteger","v","Float32Array","length","off","d","norm","Math","sqrt","isDownloadable","downloadModel","into","console","error","readLanguages","undefined","modelDir","MODEL_FILES","existsSync","join","SenseError","MODEL_FILENAMES","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","slice","Tokenizer","vocab","unk_token","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":";;;;+BAWsBA;;;eAAAA;;;sBAXmB;wBACpB;0BACK;wBACC;uBAC0E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGrG,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa;;kBA8BLC,sBAA7BA,4BAAAA,uBA7BVC,KACAC,WAKIC,QAID,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAOPC,KACAC,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAZ,eACAa,KACAC;QAEN,SAASC,IAAIC,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMC,MAAM,AAACJ,IAAIK,MAAM,CAACF,MAAM;gBAAEG,oBAAoB;YAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,SAACC;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOP;;YAC5H,IAAMU,IAAI,IAAIC,aAAaf;YAC3B,IAAIO,IAAIS,MAAM,KAAK,GAAG,OAAOF;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYP,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMI,KAAN;oBACH,IAAMM,MAAMN,KAAKX;oBACjB,IAAK,IAAIkB,IAAI,GAAGA,IAAIlB,MAAMkB,IAAKJ,CAAC,CAACI,EAAE,IAAIhB,MAAM,CAACe,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAIlB,MAAMkB,KAAK;gBAC7BJ,CAAC,CAACI,GAAE,IAAIX,IAAIS,MAAM;gBAClBG,QAAQL,CAAC,CAACI,GAAE,GAAGJ,CAAC,CAACI,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAIlB,MAAMkB,KAAKJ,CAAC,CAACI,GAAE,IAAIC;YACvC,OAAOL;QACT;;;;yBAhDIQ,IAAAA,uBAAc,EAACjC,QAAfiC;;;;oBAEI;;wBAAMC,IAAAA,sBAAa,EAAClC,OAAO,SAACK,MAAM8B;mCAASC,QAAQC,KAAK,CAAC,AAAC,mBAA2BhC,OAATL,OAAM,KAAgBmC,OAAb9B,MAAK,UAAa,OAAL8B;;;;oBADxG,qFAAqF;oBACrFjC,MAAM;oBACN,uFAAuF;oBACjFE,SAASkC,IAAAA,sBAAa,EAACtC;oBAC7BG,YAAYC,UAAUA,OAAOuB,MAAM,GAAG,IAAIvB,SAASmC;;;;;;oBAEnDrC,MAAMsC,IAAAA,iBAAQ,EAACxC;oBACV,kCAAA,2BAAA;;wBAAL,IAAK,YAAcyC,oBAAW,uBAAzB,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBpC,OAAN;4BACH,IAAI,CAACqC,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAACzC,KAAKG,QAAQ;gCAChC,MAAM,IAAIuC,oBAAU,CAAC,uBAAuB,AAAC,eAAmD1C,OAArCF,OAAM,iCAAiD6C,OAAlB3C,KAAI,gBAA8B,OAAhB2C,wBAAe,EAAC;4BACpI;wBACF;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAODvC,MAAMwC,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAACzC,KAAK;oBAC7BK,YAAYgB,OAAOjB,IAAIyC,eAAe,CAAC;oBACvCvC,SAASwC,KAAKC,KAAK,CAAC3C,IAAI4C,QAAQ,CAAC,GAAG,IAAI3C,WAAW4C,QAAQ,CAAC;oBAC5D1C,QAAQ2C,OAAOC,OAAO,CAAC7C,QAAQ8C,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC9C,SAASA,KAAK,CAAC,EAAE,CAAC+C,KAAK,KAAK,OAAO,MAAM,IAAIZ,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAAN5C,OAAM;oBAC/EU,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAK+C,KAAK,CAAC,EAAE;oBACpB7C,YAAYN,IAAIoD,UAAU,GAAG,IAAInD,YAAYG,KAAKiD,YAAY,CAAC,EAAE;oBACjE9C,SAASD,YAAY,MAAM,IAAI,IAAIc,aAAapB,IAAIsD,MAAM,EAAEhD,WAAWF,KAAK+C,KAAK,CAAC,EAAE,GAAG9C,QAAQ,IAAIe,aAAapB,IAAIsD,MAAM,CAACC,KAAK,CAACjD,WAAWA,YAAYF,KAAK+C,KAAK,CAAC,EAAE,GAAG9C,OAAO;oBAE/KV,gBAAgB+C,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAACzC,KAAK,mBAAmB;oBACrEY,MAAM,IAAIgD,qBAAS,CAAC7D,eAAe,CAAC;oBACpCc,iBAAQd,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqB8D,KAAK,cAA1B9D,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqB+D,SAAS,CAAC,uCAAI,CAAC;oBAuB/E,mEAAmE;oBACnE;;wBAAO;4BAAE1C,IAAI,AAAC,UAAe,OAANtB;4BAASW,MAAAA;4BAAMsD,UAAUlE;4BAAWI,WAAAA;4BAAW+D,gBAAgB,SAAhBA,eAAuBC;;;;;4CAAUA,MAAMC,GAAG,CAACpD;;;;;4BAAMqD,YAAY,SAAZA,WAAmBpD;;;;;4CAASD,IAAIC;;;;;wBAAM;;;;IAC/J"}

package/dist/cjs/features/embed.d.cts

@@ -1,7 +1,4 @@
+import type { Chunk } from '../chunk/index.js';
 import type { Feature } from './types.js';
-export interface Chunk {
-    startLine: number;
-    endLine: number;
-    text: string;
-}
+export type { Chunk };
 export declare const embed: Feature;

package/dist/cjs/features/embed.d.ts

@@ -1,7 +1,4 @@
+import type { Chunk } from '../chunk/index.js';
 import type { Feature } from './types.js';
-export interface Chunk {
-    startLine: number;
-    endLine: number;
-    text: string;
-}
+export type { Chunk };
 export declare const embed: Feature;

package/dist/cjs/features/embed.js

@@ -8,82 +8,46 @@ Object.defineProperty(exports, "embed", {
         return embed;
     }
 });
-var _fencests = require("../fences.js");
-function _array_like_to_array(arr, len) {
-    if (len == null || len > arr.length) len = arr.length;
-    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
-    return arr2;
-}
-function _array_without_holes(arr) {
-    if (Array.isArray(arr)) return _array_like_to_array(arr);
-}
-function _iterable_to_array(iter) {
-    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
-        return Array.from(iter);
-    }
-}
-function _non_iterable_spread() {
-    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
-}
-function _to_consumable_array(arr) {
-    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
-}
-function _unsupported_iterable_to_array(o, minLen) {
-    if (!o) return;
-    if (typeof o === "string") return _array_like_to_array(o, minLen);
-    var n = Object.prototype.toString.call(o).slice(8, -1);
-    if (n === "Object" && o.constructor) n = o.constructor.name;
-    if (n === "Map" || n === "Set") return Array.from(n);
-    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
-}
-// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,
-// preamble kept, whole body when no headings; the title/summary prefix mirrors bm25 weighting.
+var _indexts = require("../chunk/index.js");
+var _indexts1 = require("../config/index.js");
+var _identityts = require("../embed/identity.js");
+// Deterministic, so embed time can re-derive text from stored line ranges; the title/summary
+// prefix mirrors bm25 field weighting, and an empty chunk is dropped before the prefix is added.
 //
 // Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw
 // file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw
 // put the frontmatter block in its own leading chunk on every note with a heading, which made
 // `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also
 // disagreed with FTS, which indexes the body alone.
-function chunksOf(body, search) {
-    var offset = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 0;
-    var lines = body.split('\n');
-    var starts = [];
-    var fence = (0, _fencests.fenceTracker)();
-    for(var i = 0; i < lines.length; i++){
-        if (fence.feed(lines[i])) continue;
-        if (!fence.inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);
-    }
-    var bounds = starts.length === 0 ? [
-        1
-    ] : starts[0] > 1 ? [
-        1
-    ].concat(_to_consumable_array(starts)) : starts;
+function chunksOf(blocks, body, search) {
+    var offset = arguments.length > 3 && arguments[3] !== void 0 ? arguments[3] : 0, chunkTokens = arguments.length > 4 ? arguments[4] : void 0;
     var prefix = [
         search === null || search === void 0 ? void 0 : search.title,
         search === null || search === void 0 ? void 0 : search.summary
     ].filter(Boolean).join('\n');
-    var chunks = [];
-    bounds.forEach(function(start, i) {
-        var end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;
-        var text = lines.slice(start - 1, end).join('\n').trim();
-        // A body with nothing in it yields no chunks at all, so a frontmatter-only note has no
-        // vectors rather than a vector of its own YAML.
-        if (text.length > 0) chunks.push({
-            startLine: start + offset,
-            endLine: end + offset,
-            text: prefix ? "".concat(prefix, "\n").concat(text) : text
-        });
+    // chunk()'s default text mode is 'raw' (D9), the shipped choice. A hardcoded option change
+    // here must bump src/chunk/version.ts's CHUNK_VERSION -- the signature for every chunker
+    // input not already carried in config.
+    return (0, _indexts.chunkFromBlocks)(blocks, body, chunkTokens !== undefined ? {
+        targetTokens: chunkTokens
+    } : undefined).map(function(c) {
+        return {
+            startLine: c.startLine + offset,
+            endLine: c.endLine + offset,
+            text: prefix ? "".concat(prefix, "\n").concat(c.text) : c.text
+        };
     });
-    return chunks;
 }
 var embed = {
     name: 'embed',
     schema: function schema(db) {
         db.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))');
     },
-    extract: function extract(raw, body, search) {
+    extract: function extract(raw, body, search, _data, cfg, blocks) {
+        var _cfg_embed;
         // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.
-        return chunksOf(body, search, raw.split('\n').length - body.split('\n').length);
+        // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.
+        return chunksOf(blocks !== null && blocks !== void 0 ? blocks : (0, _indexts.parse)(body), body, search, raw.split('\n').length - body.split('\n').length, cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
     },
     remove: function remove(db, path) {
         db.prepare('DELETE FROM embeddings WHERE "path" = ?').run(path);
@@ -99,6 +63,20 @@ var embed = {
     },
     enabledForFile: function enabledForFile(_cfg, file) {
         return file.embed;
+    },
+    // Provider/model/chunk-version/identity, keyed so a rebuild notice can name it "embed
+    // settings". Driven by embedConfig(cfg) directly (a model can be configured with no preset
+    // yet using vectors), not by whether this feature is itself active.
+    signature: function signature(cfg) {
+        var e = (0, _indexts1.embedConfig)(cfg);
+        if (!e) return 'embed:off';
+        // Weight identity from identity.ts, no network: a static model's resolved sha or local
+        // size+mtime, appended once known so changed weights re-embed.
+        var identity = e.provider === 'static' ? (0, _identityts.modelIdentity)(e.model) : undefined;
+        // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker
+        // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.
+        var chunkVersion = e.chunkTokens !== undefined ? "".concat(_indexts.CHUNK_VERSION, ":").concat(e.chunkTokens) : _indexts.CHUNK_VERSION;
+        return "embed:".concat(e.provider, ":").concat(e.model, ":").concat(chunkVersion).concat(identity !== undefined ? "@".concat(identity) : '');
     }
 };
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }