sensemaking 0.17.1 → 0.18.0

package/dist/cjs/embed/langfit.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import Module from 'node:module';\nimport type { DatabaseSync } from 'node:sqlite';\nimport { SenseError } from '../errors.ts';\nimport { languageDistribution, saveLanguageDistribution } from './distribution.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\n// franc-min is a 148K static import; a top-level import put it on every command's module graph\n// regardless of whether the semantic path ran, shipped as the 0.17.0 regression. Deferred to a\n// synchronous require inside checkLanguageFit so only the embed path pays for it.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\nfunction classify(franc: typeof import('franc-min')['franc'], text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const { franc } = _require('franc-min') as typeof import('franc-min');\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(franc, text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  saveLanguageDistribution(db, merged);\n}\n"],"names":["checkLanguageFit","SAMPLE_CHARS","MIN_CLASSIFIED","_require","require","Module","createRequire","classify","franc","text","code","slice","undefined","db","provider","texts","languageDistribution","persisted","merged","total","declared","Object","majorityLang","majorityCount","pct","length","values","reduce","a","b","languages","Set","map","toIso3","entries","sort","has","Math","round","SenseError","id","join","saveLanguageDistribution"],"mappings":";;;;+BA0BsBA;;;eAAAA;;;iEA1BH;wBAEQ;8BACoC;2BACxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvB,+FAA+F;AAC/F,8FAA8F;AAC9F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,IAAMC,iBAAiB;AAEvB,+FAA+F;AAC/F,+FAA+F;AAC/F,kFAAkF;AAClF,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAE1F,SAASG,SAASC,KAA0C,EAAEC,IAAY;IACxE,IAAMC,OAAOF,MAAMC,KAAKE,KAAK,CAAC,GAAGV;IACjC,OAAOS,SAAS,QAAQE,YAAYF;AACtC;AAIO,SAAeV,iBAAiBa,EAAgB,EAAEC,QAAuB,EAAEC,KAAe;;YAG7EC,uBADVR,OACFS,WACAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMT,MAEiBS,cADpBR,MAIFS,OAEEC,UACgCC,uBAA/BC,cAAcC,eAEbC;;YAdV,IAAIT,MAAMU,MAAM,KAAK,GAAG;;;YAChBjB,QAAUL,SAAS,aAAnBK;YACFS,aAAYD,wBAAAA,IAAAA,oCAAoB,EAACH,iBAArBG,mCAAAA,wBAA4B,CAAC;YACzCE,SAAS,mBAAKD;YACf,kCAAA,2BAAA;;gBAAL,IAAK,YAAcF,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;oBAAfN,OAAN;;oBACGC,OAAOH,SAASC,OAAOC;oBAC7B,IAAIC,MAAMQ,MAAM,CAACR,KAAK,GAAG,EAACQ,eAAAA,MAAM,CAACR,KAAK,cAAZQ,0BAAAA,eAAgB,KAAK;gBACjD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAKCC,QAAQE,OAAOK,MAAM,CAACR,QAAQS,MAAM,CAAC,SAACC,GAAGC;uBAAMD,IAAIC;eAAG;YAC5D,IAAIf,SAASgB,SAAS,IAAIhB,SAASgB,SAAS,CAACL,MAAM,GAAG,KAAKN,SAASjB,gBAAgB;gBAC5EkB,WAAW,IAAIW,IAAIjB,SAASgB,SAAS,CAACE,GAAG,CAACC,mBAAM;gBAChBZ,yCAAAA,OAAOa,OAAO,CAAChB,QAAQiB,IAAI,CAAC,SAACP,GAAGC;2BAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;kBAAC,CAAC,EAAE,MAApFN,eAA+BD,0BAAjBE,gBAAiBF;gBACtC,IAAIE,gBAAgBJ,SAAS,OAAO,CAACC,SAASgB,GAAG,CAACd,eAAe;oBACzDE,MAAMa,KAAKC,KAAK,CAAC,AAACf,gBAAgBJ,QAAS;oBACjD,MAAM,IAAIoB,oBAAU,CAClB,wBACA,AAAC,eAAiDzB,OAAnCA,SAAS0B,EAAE,EAAC,yBAA8DhB,OAAvCV,SAASgB,SAAS,CAACW,IAAI,CAAC,OAAM,WAAoDnB,OAA3CE,KAAI,0CAAoD,OAAbF,cAAa;gBAErJ;YACF;YACAoB,IAAAA,wCAAwB,EAAC7B,IAAIK;;;;;IAC/B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import Module from 'node:module';\nimport { SenseError } from '../errors.ts';\nimport type { Store } from '../store/types.ts';\nimport { languageDistribution, saveLanguageDistribution } from './distribution.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\n// franc-min is a 148K static import; a top-level import put it on every command's module graph\n// regardless of whether the semantic path ran, shipped as the 0.17.0 regression. Deferred to a\n// synchronous require inside checkLanguageFit so only the embed path pays for it.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\nfunction classify(franc: typeof import('franc-min')['franc'], text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(store: Store, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const { franc } = _require('franc-min') as typeof import('franc-min');\n  const persisted = (await languageDistribution(store)) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(franc, text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  await saveLanguageDistribution(store, merged);\n}\n"],"names":["checkLanguageFit","SAMPLE_CHARS","MIN_CLASSIFIED","_require","require","Module","createRequire","classify","franc","text","code","slice","undefined","store","provider","texts","persisted","merged","total","declared","Object","majorityLang","majorityCount","pct","length","languageDistribution","values","reduce","a","b","languages","Set","map","toIso3","entries","sort","has","Math","round","SenseError","id","join","saveLanguageDistribution"],"mappings":";;;;+BA0BsBA;;;eAAAA;;;iEA1BH;wBACQ;8BAEoC;2BACxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvB,+FAA+F;AAC/F,8FAA8F;AAC9F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,IAAMC,iBAAiB;AAEvB,+FAA+F;AAC/F,+FAA+F;AAC/F,kFAAkF;AAClF,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAE1F,SAASG,SAASC,KAA0C,EAAEC,IAAY;IACxE,IAAMC,OAAOF,MAAMC,KAAKE,KAAK,CAAC,GAAGV;IACjC,OAAOS,SAAS,QAAQE,YAAYF;AACtC;AAIO,SAAeV,iBAAiBa,KAAY,EAAEC,QAAuB,EAAEC,KAAe;;YAGxE,MADXP,OACFQ,WACAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMR,MAEiBQ,cADpBP,MAIFQ,OAEEC,UACgCC,uBAA/BC,cAAcC,eAEbC;;;;oBAdV,IAAIR,MAAMS,MAAM,KAAK,GAAG;;;oBAChBhB,QAAUL,SAAS,aAAnBK;oBACW;;wBAAMiB,IAAAA,oCAAoB,EAACZ;;;oBAAxCG,aAAa,OAAA,2BAAA,kBAAA,OAAsC,CAAC;oBACpDC,SAAS,mBAAKD;oBACf,kCAAA,2BAAA;;wBAAL,IAAK,YAAcD,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAfN,OAAN;;4BACGC,OAAOH,SAASC,OAAOC;4BAC7B,IAAIC,MAAMO,MAAM,CAACP,KAAK,GAAG,EAACO,eAAAA,MAAM,CAACP,KAAK,cAAZO,0BAAAA,eAAgB,KAAK;wBACjD;;wBAHK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKCC,QAAQE,OAAOM,MAAM,CAACT,QAAQU,MAAM,CAAC,SAACC,GAAGC;+BAAMD,IAAIC;uBAAG;oBAC5D,IAAIf,SAASgB,SAAS,IAAIhB,SAASgB,SAAS,CAACN,MAAM,GAAG,KAAKN,SAAShB,gBAAgB;wBAC5EiB,WAAW,IAAIY,IAAIjB,SAASgB,SAAS,CAACE,GAAG,CAACC,mBAAM;wBAChBb,yCAAAA,OAAOc,OAAO,CAACjB,QAAQkB,IAAI,CAAC,SAACP,GAAGC;mCAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;0BAAC,CAAC,EAAE,MAApFP,eAA+BD,0BAAjBE,gBAAiBF;wBACtC,IAAIE,gBAAgBJ,SAAS,OAAO,CAACC,SAASiB,GAAG,CAACf,eAAe;4BACzDE,MAAMc,KAAKC,KAAK,CAAC,AAAChB,gBAAgBJ,QAAS;4BACjD,MAAM,IAAIqB,oBAAU,CAClB,wBACA,AAAC,eAAiDzB,OAAnCA,SAAS0B,EAAE,EAAC,yBAA8DjB,OAAvCT,SAASgB,SAAS,CAACW,IAAI,CAAC,OAAM,WAAoDpB,OAA3CE,KAAI,0CAAoD,OAAbF,cAAa;wBAErJ;oBACF;oBACA;;wBAAMqB,IAAAA,wCAAwB,EAAC7B,OAAOI;;;oBAAtC;;;;;;IACF"}

package/dist/cjs/embed/query.d.cts

@@ -1,21 +1,14 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../config/index.js';
+import type { Store, VectorCandidate, VectorSimilar } from '../store/types.js';
 export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
     v: Float32Array;
     scale: number;
 };
-export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
-export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
-    path: string;
-    lines: string;
-    similarity: number;
-}>>;
-export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
-export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+export declare function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export declare function hasEmbedding(store: Store, path: string): Promise<boolean>;
+export declare function similarNotes(store: Store, _cfg: Config, path: string, opts: {
     exclude: Set<string>;
     allowed?: Set<string>;
     k: number;
-}): Array<{
-    path: string;
-    similarity: number;
-}>;
+}): Promise<VectorSimilar[]>;

package/dist/cjs/embed/query.d.ts

@@ -1,21 +1,14 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../config/index.js';
+import type { Store, VectorCandidate, VectorSimilar } from '../store/types.js';
 export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
     v: Float32Array;
     scale: number;
 };
-export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
-export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
-    path: string;
-    lines: string;
-    similarity: number;
-}>>;
-export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
-export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+export declare function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export declare function hasEmbedding(store: Store, path: string): Promise<boolean>;
+export declare function similarNotes(store: Store, _cfg: Config, path: string, opts: {
     exclude: Set<string>;
     allowed?: Set<string>;
     k: number;
-}): Array<{
-    path: string;
-    similarity: number;
-}>;
+}): Promise<VectorSimilar[]>;

package/dist/cjs/embed/query.js

@@ -32,6 +32,7 @@ var _progressts = require("../output/progress.js");
 var _indexts = require("../scan/index.js");
 var _langfitts = require("./langfit.js");
 var _registryts = require("./registry.js");
+var _typests = require("./types.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -40,9 +41,6 @@ function _array_like_to_array(arr, len) {
 function _array_with_holes(arr) {
     if (Array.isArray(arr)) return arr;
 }
-function _array_without_holes(arr) {
-    if (Array.isArray(arr)) return _array_like_to_array(arr);
-}
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -69,11 +67,6 @@ function _async_to_generator(fn) {
         });
     };
 }
-function _iterable_to_array(iter) {
-    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
-        return Array.from(iter);
-    }
-}
 function _iterable_to_array_limit(arr, i) {
     var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
     if (_i == null) return;
@@ -101,15 +94,9 @@ function _iterable_to_array_limit(arr, i) {
 function _non_iterable_rest() {
     throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
 }
-function _non_iterable_spread() {
-    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
-}
 function _sliced_to_array(arr, i) {
     return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
 }
-function _to_consumable_array(arr) {
-    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
-}
 function _ts_generator(thisArg, body) {
     var f, y, t, _ = {
         label: 0,
@@ -237,11 +224,6 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):
-// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
-var STORE_DIMS = 256;
-// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.
-var TARGET_CHUNK_CAP = 16;
 function toStore(full, dims, int8) {
     var v = new Float32Array(dims);
     var norm = 0;
@@ -259,9 +241,9 @@ function toStore(full, dims, int8) {
         scale: max / 127 || 1
     };
 }
-function embedPending(db, cfg, baseDir) {
+function embedPending(store, cfg, baseDir) {
     return _async_to_generator(function() {
-        var _loop, provider, dirty, storeDims, byPath, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, _byPath_get, list, jobs, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, _step_value, path, chunkIdxs, chunks, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, idx, update, report, i;
+        var _loop, provider, dirty, storeDims, byPath, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, _byPath_get, list, jobs, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, _step_value, path, chunkIdxs, chunks, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, idx, report, i;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -279,19 +261,22 @@ function embedPending(db, cfg, baseDir) {
                                     ];
                                 case 1:
                                     vectors = _state.sent();
-                                    db.exec('BEGIN');
-                                    try {
-                                        batch.forEach(function(job, j) {
+                                    return [
+                                        4,
+                                        store.vectors.writeVectors(batch.map(function(job, j) {
                                             var _toStore = toStore(vectors[j], storeDims, true), v = _toStore.v, scale = _toStore.scale;
                                             var q = new Int8Array(storeDims);
                                             for(var d = 0; d < storeDims; d++)q[d] = Math.round(v[d] / scale);
-                                            update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);
-                                        });
-                                        db.exec('COMMIT');
-                                    } catch (err) {
-                                        db.exec('ROLLBACK');
-                                        throw err;
-                                    }
+                                            return {
+                                                path: job.path,
+                                                chunk: job.chunk,
+                                                scale: scale,
+                                                vector: Buffer.from(q.buffer)
+                                            };
+                                        }))
+                                    ];
+                                case 2:
+                                    _state.sent();
                                     report.tick(Math.min(i + provider.batchCap, jobs.length));
                                     return [
                                         2
@@ -305,11 +290,16 @@ function embedPending(db, cfg, baseDir) {
                     ];
                 case 1:
                     provider = _state.sent();
-                    dirty = db.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk').all();
+                    return [
+                        4,
+                        store.vectors.pending()
+                    ];
+                case 2:
+                    dirty = _state.sent();
                     if (dirty.length === 0) return [
                         2
                     ];
-                    storeDims = Math.min(STORE_DIMS, provider.dims);
+                    storeDims = Math.min(_typests.STORE_DIMS, provider.dims);
                     byPath = new Map();
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
@@ -400,37 +390,36 @@ function embedPending(db, cfg, baseDir) {
                     // this run loudly instead of quietly storing vectors from the wrong model.
                     return [
                         4,
-                        (0, _langfitts.checkLanguageFit)(db, provider, jobs.map(function(j) {
+                        (0, _langfitts.checkLanguageFit)(store, provider, jobs.map(function(j) {
                             return j.text;
                         }))
                     ];
-                case 2:
+                case 3:
                     _state.sent();
-                    update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE "path" = ? AND chunk = ?');
                     // The lazy build is the one long silence a first search hits (measured 23s at
                     // 26k notes); progress makes it distinguishable from a hang.
                     report = (0, _progressts.progress)('embedding chunks', jobs.length);
                     i = 0;
-                    _state.label = 3;
-                case 3:
+                    _state.label = 4;
+                case 4:
                     if (!(i < jobs.length)) return [
                         3,
-                        6
+                        7
                     ];
                     return [
                         5,
                         _ts_values(_loop(i))
                     ];
-                case 4:
-                    _state.sent();
-                    _state.label = 5;
                 case 5:
+                    _state.sent();
+                    _state.label = 6;
+                case 6:
                     i += provider.batchCap;
                     return [
                         3,
-                        3
+                        4
                     ];
-                case 6:
+                case 7:
                     report.finish();
                     return [
                         2
@@ -439,14 +428,9 @@ function embedPending(db, cfg, baseDir) {
         });
     })();
 }
-// Dequantised int8 dot products land a little either side of a true cosine, so an identical
-// pair prints 1.001 and undermines a column whose whole job is being a bounded number.
-function asCosine(score) {
-    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
-}
-function semanticCandidates(db, cfg, terms, fetch, allowed) {
+function semanticCandidates(store, cfg, terms, fetch, allowed) {
     return _async_to_generator(function() {
-        var _terms_match, baseDir, provider, storeDims, text, _toStore, qv, rows, best, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, q, dot, d, score, existing;
+        var _terms_match, baseDir, provider, storeDims, text, _toStore, qv;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -454,7 +438,7 @@ function semanticCandidates(db, cfg, terms, fetch, allowed) {
                     if (!baseDir) throw new _errorsts.SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');
                     return [
                         4,
-                        embedPending(db, cfg, baseDir)
+                        embedPending(store, cfg, baseDir)
                     ];
                 case 1:
                     _state.sent();
@@ -464,7 +448,7 @@ function semanticCandidates(db, cfg, terms, fetch, allowed) {
                     ];
                 case 2:
                     provider = _state.sent();
-                    storeDims = Math.min(STORE_DIMS, provider.dims);
+                    storeDims = Math.min(_typests.STORE_DIMS, provider.dims);
                     text = ((_terms_match = terms.match(RegExp("[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376-\\u0377\\u037A-\\u037D\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u052F\\u0531-\\u0556\\u0559\\u0560-\\u0588\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u064A\\u066E-\\u066F\\u0671-\\u06D3\\u06D5\\u06E5-\\u06E6\\u06EE-\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4-\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u0860-\\u086A\\u0870-\\u0887\\u0889-\\u088F\\u08A0-\\u08C9\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098C\\u098F-\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC-\\u09DD\\u09DF-\\u09E1\\u09F0-\\u09F1\\u09FC\\u0A05-\\u0A0A\\u0A0F-\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32-\\u0A33\\u0A35-\\u0A36\\u0A38-\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2-\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0-\\u0AE1\\u0AF9\\u0B05-\\u0B0C\\u0B0F-\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32-\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C-\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99-\\u0B9A\\u0B9C\\u0B9E-\\u0B9F\\u0BA3-\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C39\\u0C3D\\u0C58-\\u0C5A\\u0C5C-\\u0C5D\\u0C60-\\u0C61\\u0C80\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDC-\\u0CDE\\u0CE0-\\u0CE1\\u0CF1-\\u0CF2\\u0D04-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D54-\\u0D56\\u0D5F-\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32-\\u0E33\\u0E40-\\u0E46\\u0E81-\\u0E82\\u0E84\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB0\\u0EB2-\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065-\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F5\\u13F8-\\u13FD\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16F1-\\u16F8\\u1700-\\u1711\\u171F-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1878\\u1880-\\u1884\\u1887-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191E\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19B0-\\u19C9\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4C\\u1B83-\\u1BA0\\u1BAE-\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1C80-\\u1C8A\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1CE9-\\u1CEC\\u1CEE-\\u1CF3\\u1CF5-\\u1CF6\\u1CFA\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183-\\u2184\\u2C00-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2-\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3006\\u3031-\\u3035\\u303B-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312F\\u3131-\\u318E\\u31A0-\\u31BF\\u31F0-\\u31FF\\u3400-\\u4DBF\\u4E00-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A-\\uA62B\\uA640-\\uA66E\\uA67F-\\uA69D\\uA6A0-\\uA6E5\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA7DC\\uA7F1-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA8FD-\\uA8FE\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uA9E0-\\uA9E4\\uA9E6-\\uA9EF\\uA9FA-\\uA9FE\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA7E-\\uAAAF\\uAAB1\\uAAB5-\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB30-\\uAB5A\\uAB5C-\\uAB69\\uAB70-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40-\\uFB41\\uFB43-\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC\\u{10000}-\\u{1000B}\\u{1000D}-\\u{10026}\\u{10028}-\\u{1003A}\\u{1003C}-\\u{1003D}\\u{1003F}-\\u{1004D}\\u{10050}-\\u{1005D}\\u{10080}-\\u{100FA}\\u{10280}-\\u{1029C}\\u{102A0}-\\u{102D0}\\u{10300}-\\u{1031F}\\u{1032D}-\\u{10340}\\u{10342}-\\u{10349}\\u{10350}-\\u{10375}\\u{10380}-\\u{1039D}\\u{103A0}-\\u{103C3}\\u{103C8}-\\u{103CF}\\u{10400}-\\u{1049D}\\u{104B0}-\\u{104D3}\\u{104D8}-\\u{104FB}\\u{10500}-\\u{10527}\\u{10530}-\\u{10563}\\u{10570}-\\u{1057A}\\u{1057C}-\\u{1058A}\\u{1058C}-\\u{10592}\\u{10594}-\\u{10595}\\u{10597}-\\u{105A1}\\u{105A3}-\\u{105B1}\\u{105B3}-\\u{105B9}\\u{105BB}-\\u{105BC}\\u{105C0}-\\u{105F3}\\u{10600}-\\u{10736}\\u{10740}-\\u{10755}\\u{10760}-\\u{10767}\\u{10780}-\\u{10785}\\u{10787}-\\u{107B0}\\u{107B2}-\\u{107BA}\\u{10800}-\\u{10805}\\u{10808}\\u{1080A}-\\u{10835}\\u{10837}-\\u{10838}\\u{1083C}\\u{1083F}-\\u{10855}\\u{10860}-\\u{10876}\\u{10880}-\\u{1089E}\\u{108E0}-\\u{108F2}\\u{108F4}-\\u{108F5}\\u{10900}-\\u{10915}\\u{10920}-\\u{10939}\\u{10940}-\\u{10959}\\u{10980}-\\u{109B7}\\u{109BE}-\\u{109BF}\\u{10A00}\\u{10A10}-\\u{10A13}\\u{10A15}-\\u{10A17}\\u{10A19}-\\u{10A35}\\u{10A60}-\\u{10A7C}\\u{10A80}-\\u{10A9C}\\u{10AC0}-\\u{10AC7}\\u{10AC9}-\\u{10AE4}\\u{10B00}-\\u{10B35}\\u{10B40}-\\u{10B55}\\u{10B60}-\\u{10B72}\\u{10B80}-\\u{10B91}\\u{10C00}-\\u{10C48}\\u{10C80}-\\u{10CB2}\\u{10CC0}-\\u{10CF2}\\u{10D00}-\\u{10D23}\\u{10D4A}-\\u{10D65}\\u{10D6F}-\\u{10D85}\\u{10E80}-\\u{10EA9}\\u{10EB0}-\\u{10EB1}\\u{10EC2}-\\u{10EC7}\\u{10F00}-\\u{10F1C}\\u{10F27}\\u{10F30}-\\u{10F45}\\u{10F70}-\\u{10F81}\\u{10FB0}-\\u{10FC4}\\u{10FE0}-\\u{10FF6}\\u{11003}-\\u{11037}\\u{11071}-\\u{11072}\\u{11075}\\u{11083}-\\u{110AF}\\u{110D0}-\\u{110E8}\\u{11103}-\\u{11126}\\u{11144}\\u{11147}\\u{11150}-\\u{11172}\\u{11176}\\u{11183}-\\u{111B2}\\u{111C1}-\\u{111C4}\\u{111DA}\\u{111DC}\\u{11200}-\\u{11211}\\u{11213}-\\u{1122B}\\u{1123F}-\\u{11240}\\u{11280}-\\u{11286}\\u{11288}\\u{1128A}-\\u{1128D}\\u{1128F}-\\u{1129D}\\u{1129F}-\\u{112A8}\\u{112B0}-\\u{112DE}\\u{11305}-\\u{1130C}\\u{1130F}-\\u{11310}\\u{11313}-\\u{11328}\\u{1132A}-\\u{11330}\\u{11332}-\\u{11333}\\u{11335}-\\u{11339}\\u{1133D}\\u{11350}\\u{1135D}-\\u{11361}\\u{11380}-\\u{11389}\\u{1138B}\\u{1138E}\\u{11390}-\\u{113B5}\\u{113B7}\\u{113D1}\\u{113D3}\\u{11400}-\\u{11434}\\u{11447}-\\u{1144A}\\u{1145F}-\\u{11461}\\u{11480}-\\u{114AF}\\u{114C4}-\\u{114C5}\\u{114C7}\\u{11580}-\\u{115AE}\\u{115D8}-\\u{115DB}\\u{11600}-\\u{1162F}\\u{11644}\\u{11680}-\\u{116AA}\\u{116B8}\\u{11700}-\\u{1171A}\\u{11740}-\\u{11746}\\u{11800}-\\u{1182B}\\u{118A0}-\\u{118DF}\\u{118FF}-\\u{11906}\\u{11909}\\u{1190C}-\\u{11913}\\u{11915}-\\u{11916}\\u{11918}-\\u{1192F}\\u{1193F}\\u{11941}\\u{119A0}-\\u{119A7}\\u{119AA}-\\u{119D0}\\u{119E1}\\u{119E3}\\u{11A00}\\u{11A0B}-\\u{11A32}\\u{11A3A}\\u{11A50}\\u{11A5C}-\\u{11A89}\\u{11A9D}\\u{11AB0}-\\u{11AF8}\\u{11BC0}-\\u{11BE0}\\u{11C00}-\\u{11C08}\\u{11C0A}-\\u{11C2E}\\u{11C40}\\u{11C72}-\\u{11C8F}\\u{11D00}-\\u{11D06}\\u{11D08}-\\u{11D09}\\u{11D0B}-\\u{11D30}\\u{11D46}\\u{11D60}-\\u{11D65}\\u{11D67}-\\u{11D68}\\u{11D6A}-\\u{11D89}\\u{11D98}\\u{11DB0}-\\u{11DDB}\\u{11EE0}-\\u{11EF2}\\u{11F02}\\u{11F04}-\\u{11F10}\\u{11F12}-\\u{11F33}\\u{11FB0}\\u{12000}-\\u{12399}\\u{12480}-\\u{12543}\\u{12F90}-\\u{12FF0}\\u{13000}-\\u{1342F}\\u{13441}-\\u{13446}\\u{13460}-\\u{143FA}\\u{14400}-\\u{14646}\\u{16100}-\\u{1611D}\\u{16800}-\\u{16A38}\\u{16A40}-\\u{16A5E}\\u{16A70}-\\u{16ABE}\\u{16AD0}-\\u{16AED}\\u{16B00}-\\u{16B2F}\\u{16B40}-\\u{16B43}\\u{16B63}-\\u{16B77}\\u{16B7D}-\\u{16B8F}\\u{16D40}-\\u{16D6C}\\u{16E40}-\\u{16E7F}\\u{16EA0}-\\u{16EB8}\\u{16EBB}-\\u{16ED3}\\u{16F00}-\\u{16F4A}\\u{16F50}\\u{16F93}-\\u{16F9F}\\u{16FE0}-\\u{16FE1}\\u{16FE3}\\u{16FF2}-\\u{16FF3}\\u{17000}-\\u{18CD5}\\u{18CFF}-\\u{18D1E}\\u{18D80}-\\u{18DF2}\\u{1AFF0}-\\u{1AFF3}\\u{1AFF5}-\\u{1AFFB}\\u{1AFFD}-\\u{1AFFE}\\u{1B000}-\\u{1B122}\\u{1B132}\\u{1B150}-\\u{1B152}\\u{1B155}\\u{1B164}-\\u{1B167}\\u{1B170}-\\u{1B2FB}\\u{1BC00}-\\u{1BC6A}\\u{1BC70}-\\u{1BC7C}\\u{1BC80}-\\u{1BC88}\\u{1BC90}-\\u{1BC99}\\u{1D400}-\\u{1D454}\\u{1D456}-\\u{1D49C}\\u{1D49E}-\\u{1D49F}\\u{1D4A2}\\u{1D4A5}-\\u{1D4A6}\\u{1D4A9}-\\u{1D4AC}\\u{1D4AE}-\\u{1D4B9}\\u{1D4BB}\\u{1D4BD}-\\u{1D4C3}\\u{1D4C5}-\\u{1D505}\\u{1D507}-\\u{1D50A}\\u{1D50D}-\\u{1D514}\\u{1D516}-\\u{1D51C}\\u{1D51E}-\\u{1D539}\\u{1D53B}-\\u{1D53E}\\u{1D540}-\\u{1D544}\\u{1D546}\\u{1D54A}-\\u{1D550}\\u{1D552}-\\u{1D6A5}\\u{1D6A8}-\\u{1D6C0}\\u{1D6C2}-\\u{1D6DA}\\u{1D6DC}-\\u{1D6FA}\\u{1D6FC}-\\u{1D714}\\u{1D716}-\\u{1D734}\\u{1D736}-\\u{1D74E}\\u{1D750}-\\u{1D76E}\\u{1D770}-\\u{1D788}\\u{1D78A}-\\u{1D7A8}\\u{1D7AA}-\\u{1D7C2}\\u{1D7C4}-\\u{1D7CB}\\u{1DF00}-\\u{1DF1E}\\u{1DF25}-\\u{1DF2A}\\u{1E030}-\\u{1E06D}\\u{1E100}-\\u{1E12C}\\u{1E137}-\\u{1E13D}\\u{1E14E}\\u{1E290}-\\u{1E2AD}\\u{1E2C0}-\\u{1E2EB}\\u{1E4D0}-\\u{1E4EB}\\u{1E5D0}-\\u{1E5ED}\\u{1E5F0}\\u{1E6C0}-\\u{1E6DE}\\u{1E6E0}-\\u{1E6E2}\\u{1E6E4}-\\u{1E6E5}\\u{1E6E7}-\\u{1E6ED}\\u{1E6F0}-\\u{1E6F4}\\u{1E6FE}-\\u{1E6FF}\\u{1E7E0}-\\u{1E7E6}\\u{1E7E8}-\\u{1E7EB}\\u{1E7ED}-\\u{1E7EE}\\u{1E7F0}-\\u{1E7FE}\\u{1E800}-\\u{1E8C4}\\u{1E900}-\\u{1E943}\\u{1E94B}\\u{1EE00}-\\u{1EE03}\\u{1EE05}-\\u{1EE1F}\\u{1EE21}-\\u{1EE22}\\u{1EE24}\\u{1EE27}\\u{1EE29}-\\u{1EE32}\\u{1EE34}-\\u{1EE37}\\u{1EE39}\\u{1EE3B}\\u{1EE42}\\u{1EE47}\\u{1EE49}\\u{1EE4B}\\u{1EE4D}-\\u{1EE4F}\\u{1EE51}-\\u{1EE52}\\u{1EE54}\\u{1EE57}\\u{1EE59}\\u{1EE5B}\\u{1EE5D}\\u{1EE5F}\\u{1EE61}-\\u{1EE62}\\u{1EE64}\\u{1EE67}-\\u{1EE6A}\\u{1EE6C}-\\u{1EE72}\\u{1EE74}-\\u{1EE77}\\u{1EE79}-\\u{1EE7C}\\u{1EE7E}\\u{1EE80}-\\u{1EE89}\\u{1EE8B}-\\u{1EE9B}\\u{1EEA1}-\\u{1EEA3}\\u{1EEA5}-\\u{1EEA9}\\u{1EEAB}-\\u{1EEBB}\\u{20000}-\\u{2A6DF}\\u{2A700}-\\u{2B81D}\\u{2B820}-\\u{2CEAD}\\u{2CEB0}-\\u{2EBE0}\\u{2EBF0}-\\u{2EE5D}\\u{2F800}-\\u{2FA1D}\\u{30000}-\\u{3134A}\\u{31350}-\\u{33479}\\u0030-\\u0039\\u00B2-\\u00B3\\u00B9\\u00BC-\\u00BE\\u0660-\\u0669\\u06F0-\\u06F9\\u07C0-\\u07C9\\u0966-\\u096F\\u09E6-\\u09EF\\u09F4-\\u09F9\\u0A66-\\u0A6F\\u0AE6-\\u0AEF\\u0B66-\\u0B6F\\u0B72-\\u0B77\\u0BE6-\\u0BF2\\u0C66-\\u0C6F\\u0C78-\\u0C7E\\u0CE6-\\u0CEF\\u0D58-\\u0D5E\\u0D66-\\u0D78\\u0DE6-\\u0DEF\\u0E50-\\u0E59\\u0ED0-\\u0ED9\\u0F20-\\u0F33\\u1040-\\u1049\\u1090-\\u1099\\u1369-\\u137C\\u16EE-\\u16F0\\u17E0-\\u17E9\\u17F0-\\u17F9\\u1810-\\u1819\\u1946-\\u194F\\u19D0-\\u19DA\\u1A80-\\u1A89\\u1A90-\\u1A99\\u1B50-\\u1B59\\u1BB0-\\u1BB9\\u1C40-\\u1C49\\u1C50-\\u1C59\\u2070\\u2074-\\u2079\\u2080-\\u2089\\u2150-\\u2182\\u2185-\\u2189\\u2460-\\u249B\\u24EA-\\u24FF\\u2776-\\u2793\\u2CFD\\u3007\\u3021-\\u3029\\u3038-\\u303A\\u3192-\\u3195\\u3220-\\u3229\\u3248-\\u324F\\u3251-\\u325F\\u3280-\\u3289\\u32B1-\\u32BF\\uA620-\\uA629\\uA6E6-\\uA6EF\\uA830-\\uA835\\uA8D0-\\uA8D9\\uA900-\\uA909\\uA9D0-\\uA9D9\\uA9F0-\\uA9F9\\uAA50-\\uAA59\\uABF0-\\uABF9\\uFF10-\\uFF19\\u{10107}-\\u{10133}\\u{10140}-\\u{10178}\\u{1018A}-\\u{1018B}\\u{102E1}-\\u{102FB}\\u{10320}-\\u{10323}\\u{10341}\\u{1034A}\\u{103D1}-\\u{103D5}\\u{104A0}-\\u{104A9}\\u{10858}-\\u{1085F}\\u{10879}-\\u{1087F}\\u{108A7}-\\u{108AF}\\u{108FB}-\\u{108FF}\\u{10916}-\\u{1091B}\\u{109BC}-\\u{109BD}\\u{109C0}-\\u{109CF}\\u{109D2}-\\u{109FF}\\u{10A40}-\\u{10A48}\\u{10A7D}-\\u{10A7E}\\u{10A9D}-\\u{10A9F}\\u{10AEB}-\\u{10AEF}\\u{10B58}-\\u{10B5F}\\u{10B78}-\\u{10B7F}\\u{10BA9}-\\u{10BAF}\\u{10CFA}-\\u{10CFF}\\u{10D30}-\\u{10D39}\\u{10D40}-\\u{10D49}\\u{10E60}-\\u{10E7E}\\u{10F1D}-\\u{10F26}\\u{10F51}-\\u{10F54}\\u{10FC5}-\\u{10FCB}\\u{11052}-\\u{1106F}\\u{110F0}-\\u{110F9}\\u{11136}-\\u{1113F}\\u{111D0}-\\u{111D9}\\u{111E1}-\\u{111F4}\\u{112F0}-\\u{112F9}\\u{11450}-\\u{11459}\\u{114D0}-\\u{114D9}\\u{11650}-\\u{11659}\\u{116C0}-\\u{116C9}\\u{116D0}-\\u{116E3}\\u{11730}-\\u{1173B}\\u{118E0}-\\u{118F2}\\u{11950}-\\u{11959}\\u{11BF0}-\\u{11BF9}\\u{11C50}-\\u{11C6C}\\u{11D50}-\\u{11D59}\\u{11DA0}-\\u{11DA9}\\u{11DE0}-\\u{11DE9}\\u{11F50}-\\u{11F59}\\u{11FC0}-\\u{11FD4}\\u{12400}-\\u{1246E}\\u{16130}-\\u{16139}\\u{16A60}-\\u{16A69}\\u{16AC0}-\\u{16AC9}\\u{16B50}-\\u{16B59}\\u{16B5B}-\\u{16B61}\\u{16D70}-\\u{16D79}\\u{16E80}-\\u{16E96}\\u{16FF4}-\\u{16FF6}\\u{1CCF0}-\\u{1CCF9}\\u{1D2C0}-\\u{1D2D3}\\u{1D2E0}-\\u{1D2F3}\\u{1D360}-\\u{1D378}\\u{1D7CE}-\\u{1D7FF}\\u{1E140}-\\u{1E149}\\u{1E2F0}-\\u{1E2F9}\\u{1E4F0}-\\u{1E4F9}\\u{1E5F1}-\\u{1E5FA}\\u{1E8C7}-\\u{1E8CF}\\u{1E950}-\\u{1E959}\\u{1EC71}-\\u{1ECAB}\\u{1ECAD}-\\u{1ECAF}\\u{1ECB1}-\\u{1ECB4}\\u{1ED01}-\\u{1ED2D}\\u{1ED2F}-\\u{1ED3D}\\u{1F100}-\\u{1F10C}\\u{1FBF0}-\\u{1FBF9}]+", "gu"))) !== null && _terms_match !== void 0 ? _terms_match : []).filter(function(t) {
                         return ![
                             'AND',
@@ -483,128 +467,32 @@ function semanticCandidates(db, cfg, terms, fetch, allowed) {
                         storeDims,
                         false
                     ]), qv = _toStore.v;
-                    rows = db.prepare('SELECT "path", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
-                    best = new Map();
-                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                    try {
-                        for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                            row = _step.value;
-                            if (allowed && !allowed.has(row.path)) continue;
-                            q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));
-                            dot = 0;
-                            for(d = 0; d < q.length; d++)dot += q[d] * qv[d];
-                            score = dot * row.scale;
-                            existing = best.get(row.path);
-                            if (!existing || score > existing.score) best.set(row.path, {
-                                score: score,
-                                lines: "L".concat(row.start_line, "-").concat(row.end_line)
-                            });
-                        }
-                    } catch (err) {
-                        _didIteratorError = true;
-                        _iteratorError = err;
-                    } finally{
-                        try {
-                            if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                _iterator.return();
-                            }
-                        } finally{
-                            if (_didIteratorError) {
-                                throw _iteratorError;
-                            }
-                        }
-                    }
                     return [
                         2,
-                        _to_consumable_array(best.entries()).sort(function(a, b) {
-                            return b[1].score - a[1].score;
-                        }).slice(0, fetch).map(function(param) {
-                            var _param = _sliced_to_array(param, 2), path = _param[0], b = _param[1];
-                            return {
-                                path: path,
-                                lines: b.lines,
-                                similarity: asCosine(b.score)
-                            };
-                        })
+                        store.vectors.candidates(qv, storeDims, fetch, allowed)
                     ];
             }
         });
     })();
 }
-function hasEmbedding(db, path) {
-    var row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE "path" = ? AND vector IS NOT NULL LIMIT 1').get(path);
-    return row !== undefined;
+function hasEmbedding(store, path) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            return [
+                2,
+                store.vectors.hasVector(path)
+            ];
+        });
+    })();
 }
-function similarNotes(db, _cfg, path, opts) {
-    // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus
-    // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.
-    var targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk').all(path);
-    if (targetRows.length === 0) return [];
-    var step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
-    var target = targetRows.filter(function(_, i) {
-        return i % step === 0;
-    }).map(function(row) {
-        return {
-            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
-            scale: row.scale
-        };
-    });
-    var rows = db.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
-    var best = new Map();
-    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-    try {
-        for(var _iterator = rows[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-            var row = _step.value;
-            if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
-            var other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
-            var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
-            try {
-                for(var _iterator1 = target[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
-                    var t = _step1.value;
-                    var dot = 0;
-                    var len = Math.min(t.v.length, other.length);
-                    for(var d = 0; d < len; d++)dot += t.v[d] * other[d];
-                    var score = dot * t.scale * row.scale;
-                    var existing = best.get(row.path);
-                    if (existing === undefined || score > existing) best.set(row.path, score);
-                }
-            } catch (err) {
-                _didIteratorError1 = true;
-                _iteratorError1 = err;
-            } finally{
-                try {
-                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
-                        _iterator1.return();
-                    }
-                } finally{
-                    if (_didIteratorError1) {
-                        throw _iteratorError1;
-                    }
-                }
-            }
-        }
-    } catch (err) {
-        _didIteratorError = true;
-        _iteratorError = err;
-    } finally{
-        try {
-            if (!_iteratorNormalCompletion && _iterator.return != null) {
-                _iterator.return();
-            }
-        } finally{
-            if (_didIteratorError) {
-                throw _iteratorError;
-            }
-        }
-    }
-    return _to_consumable_array(best.entries()).sort(function(a, b) {
-        return b[1] - a[1];
-    }).slice(0, opts.k).map(function(param) {
-        var _param = _sliced_to_array(param, 2), p = _param[0], score = _param[1];
-        return {
-            path: p,
-            similarity: asCosine(score)
-        };
-    });
+function similarNotes(store, _cfg, path, opts) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            return [
+                2,
+                store.vectors.similar(path, opts)
+            ];
+        });
+    })();
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["embedPending","hasEmbedding","semanticCandidates","similarNotes","toStore","STORE_DIMS","TARGET_CHUNK_CAP","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","db","cfg","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","path","chunkIdxs","chunks","idx","update","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","chunk","err","tick","min","length","getProvider","prepare","all","Map","get","push","set","parseFile","relPath","absPath","join","mtimeMs","ctimeMs","size","presets","embed","doc","extracted","checkLanguageFit","progress","finish","asCosine","score","terms","fetch","allowed","qv","rows","best","dot","existing","SenseError","match","filter","t","includes","embedQuery","has","vector","byteOffset","byteLength","lines","start_line","end_line","entries","sort","a","b","similarity","undefined","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":";;;;;;;;;;;QAiCsBA;eAAAA;;QAwGNC;eAAAA;;QApCMC;eAAAA;;QA2CNC;eAAAA;;QA7HAC;eAAAA;;;wBAnBK;wBAGM;uBAEL;0BACG;uBACC;yBACO;0BACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8FAA8F;AAC9F,6FAA6F;AAC7F,IAAMC,aAAa;AACnB,0FAA0F;AAC1F,IAAMC,mBAAmB;AAIlB,SAASF,QAAQG,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAejB,aAAamB,EAAgB,EAAEC,GAAW,EAAEC,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOC,MAAMC,WACZC,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,KAWPC,QAGAC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQR,KAAKU,KAAK,CAACH,GAAGA,IAAIb,SAASiB,QAAQ;oCACjC;;wCAAMjB,SAASkB,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChBlB,GAAGyB,IAAI,CAAC;oCACR,IAAI;wCACFR,MAAMS,OAAO,CAAC,SAACC,KAAKJ;4CAClB,IAAqBtC,WAAAA,QAAQiC,OAAO,CAACK,EAAE,EAAElB,WAAW,OAA5Cd,IAAaN,SAAbM,GAAGM,QAAUZ,SAAVY;4CACX,IAAM+B,IAAI,IAAIC,UAAUxB;4CACxB,IAAK,IAAIX,IAAI,GAAGA,IAAIW,WAAWX,IAAKkC,CAAC,CAAClC,EAAE,GAAGC,KAAKmC,KAAK,CAACvC,CAAC,CAACG,EAAE,GAAGG;4CAC7DiB,OAAOiB,GAAG,CAAClC,OAAOmC,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAIjB,IAAI,EAAEiB,IAAIQ,KAAK;wCAC9D;wCACAnC,GAAGyB,IAAI,CAAC;oCACV,EAAE,OAAOW,KAAK;wCACZpC,GAAGyB,IAAI,CAAC;wCACR,MAAMW;oCACR;oCACArB,OAAOsB,IAAI,CAAC1C,KAAK2C,GAAG,CAACtB,IAAIb,SAASiB,QAAQ,EAAEX,KAAK8B,MAAM;;;;;;oBACzD;oBAtDiB;;wBAAMC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXC,QAAQJ,GAAGyC,OAAO,CAAC,oFAAoFC,GAAG;oBAChH,IAAItC,MAAMmC,MAAM,KAAK,GAAG;;;oBAClBlC,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAE9CiB,SAAS,IAAIqC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAavC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOsC,GAAG,CAACrC,IAAIG,IAAI,eAAnBJ,yBAAAA;4BACbE,KAAKqC,IAAI,CAACtC,IAAI4B,KAAK;4BACnB7B,OAAOwC,GAAG,CAACvC,IAAIG,IAAI,EAAEF;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOI,uBAAMC;4BACZC,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,+EAA+E;gCAC/EA,SAASmC,IAAAA,kBAAS,EAAC;oCAAEC,SAAStC;oCAAMuC,SAASC,IAAAA,cAAI,EAAChD,SAASQ;oCAAOyC,SAAS;oCAAGC,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMC,OAAO;gCAAK;oCAAIA,cAAK;mCAAGtD,KAAKuD,GAAG,CAACC,SAAS,CAACF,KAAK;4BAClK,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa5C,gCAAb,8BAAA,SAAA,0BAAA;oCAAME,MAAN;oCAAwB,IAAID,MAAM,CAACC,IAAI,EAAEJ,KAAKoC,IAAI,CAAC;wCAAEnC,MAAAA;wCAAMyB,OAAOtB;wCAAKW,MAAMZ,MAAM,CAACC,IAAI,CAACW,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAMkC,IAAAA,2BAAgB,EACpB1D,IACAG,UACAM,KAAKa,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMMV,SAASd,GAAGyC,OAAO,CAAC;oBAC1B,8EAA8E;oBAC9E,6DAA6D;oBACvD1B,SAAS4C,IAAAA,oBAAQ,EAAC,oBAAoBlD,KAAK8B,MAAM;oBAC9CvB,IAAI;;;yBAAGA,CAAAA,IAAIP,KAAK8B,MAAM,AAAD;;;;;;;;;;;;oBAAGvB,KAAKb,SAASiB,QAAQ;;;;;;oBAkBvDL,OAAO6C,MAAM;;;;;;IACf;;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAOnE,KAAKmC,KAAK,CAACnC,KAAK2C,GAAG,CAAC,GAAG3C,KAAKG,GAAG,CAAC,CAAC,GAAGgE,UAAU,QAAQ;AAC/D;AAKO,SAAe/E,mBAAmBiB,EAAgB,EAAEC,GAAW,EAAE8D,KAAa,EAAEC,KAAa,EAAEC,OAAqB;;YAO3GF,cANR7D,SAIAC,UACAE,WACAmB,MACYvC,UAAPiF,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM7D,KAEHqB,GACFyC,KACK3E,GACHoE,OACAQ;;;;oBAxBFpE,UAAU,AAACD,IAAgCC,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIqE,oBAAU,CAAC,eAAe;oBAClD;;wBAAM1F,aAAamB,IAAIC,KAAKC;;;oBAA5B;oBAEiB;;wBAAMsC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXE,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAC9CmC,OAAO,EAACuC,eAAAA,MAAMS,KAAK,CAAC,2naAAZT,0BAAAA,mBAAsCU,MAAM,CAAC,SAACC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIxB,IAAI,CAAC;oBACxF;;wBAAM/C,SAASyE,UAAU,CAACpD;;;oBAAlCvC,WAAAA;wBAAQ;wBAAiCoB;wBAAW;wBAA3D6D,KAAOjF,SAAVM;oBAEF4E,OAAOnE,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG;oBAQpH0B,OAAO,IAAIzB;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAawB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAb5D,MAAN;4BACH,IAAI0D,WAAW,CAACA,QAAQY,GAAG,CAACtE,IAAIG,IAAI,GAAG;4BACjCkB,IAAI,IAAIC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAEpF,KAAK2C,GAAG,CAACjC,WAAWE,IAAIuE,MAAM,CAACE,UAAU;4BACvGX,MAAM;4BACV,IAAS3E,IAAI,GAAGA,IAAIkC,EAAEW,MAAM,EAAE7C,IAAK2E,OAAOzC,CAAC,CAAClC,EAAE,GAAGwE,EAAE,CAACxE,EAAE;4BAChDoE,QAAQO,MAAM9D,IAAIV,KAAK;4BACvByE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;4BAClC,IAAI,CAAC4D,YAAYR,QAAQQ,SAASR,KAAK,EAAEM,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAE;gCAAEoD,OAAAA;gCAAOmB,OAAO,AAAC,IAAqB1E,OAAlBA,IAAI2E,UAAU,EAAC,KAAgB,OAAb3E,IAAI4E,QAAQ;4BAAG;wBACnH;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBASL;;wBAAQ,qBAAGf,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK;2BACtC3C,KAAK,CAAC,GAAG6C,OACT1C,GAAG,CAAC;qEAAEZ,kBAAM6E;mCAAQ;gCAAE7E,MAAAA;gCAAMuE,OAAOM,EAAEN,KAAK;gCAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAAShF,aAAakB,EAAgB,EAAEU,IAAY;IACzD,IAAMH,MAAMP,GAAGyC,OAAO,CAAC,kFAAkFG,GAAG,CAAClC;IAC7G,OAAOH,QAAQkF;AACjB;AAIO,SAASzG,aAAagB,EAAgB,EAAE0F,IAAY,EAAEhF,IAAY,EAAEiF,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,IAAMC,aAAa5F,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG,CAAChC;IACjI,IAAIkF,WAAWrD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,IAAMsD,OAAOlG,KAAKG,GAAG,CAAC,GAAGH,KAAKmG,IAAI,CAACF,WAAWrD,MAAM,GAAGpD;IACvD,IAAM4G,SAASH,WAAWnB,MAAM,CAAC,SAACuB,GAAGhF;eAAMA,IAAI6E,SAAS;OAAGvE,GAAG,CAAC,SAACf;eAAS;YAAEhB,GAAG,IAAIsC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;YAAGnF,OAAOU,IAAIV,KAAK;QAAC;;IAE/K,IAAMsE,OAAOnE,GAAGyC,OAAO,CAAC,yEAAyEC,GAAG;IACpG,IAAM0B,OAAO,IAAIzB;QACZ,kCAAA,2BAAA;;QAAL,QAAK,YAAawB,yBAAb,SAAA,6BAAA,QAAA,yBAAA,iCAAmB;YAAnB,IAAM5D,MAAN;YACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQiF,KAAKM,OAAO,CAACpB,GAAG,CAACtE,IAAIG,IAAI,KAAMiF,KAAK1B,OAAO,IAAI,CAAC0B,KAAK1B,OAAO,CAACY,GAAG,CAACtE,IAAIG,IAAI,GAAI;YACtG,IAAMwF,QAAQ,IAAIrE,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;gBACtF,mCAAA,4BAAA;;gBAAL,QAAK,aAAWe,2BAAX,UAAA,8BAAA,SAAA,0BAAA,kCAAmB;oBAAnB,IAAMrB,IAAN;oBACH,IAAIL,MAAM;oBACV,IAAM8B,MAAMxG,KAAK2C,GAAG,CAACoC,EAAEnF,CAAC,CAACgD,MAAM,EAAE2D,MAAM3D,MAAM;oBAC7C,IAAK,IAAI7C,IAAI,GAAGA,IAAIyG,KAAKzG,IAAK2E,OAAOK,EAAEnF,CAAC,CAACG,EAAE,GAAGwG,KAAK,CAACxG,EAAE;oBACtD,IAAMoE,QAAQO,MAAMK,EAAE7E,KAAK,GAAGU,IAAIV,KAAK;oBACvC,IAAMyE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;oBAClC,IAAI4D,aAAamB,aAAa3B,QAAQQ,UAAUF,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAEoD;gBACrE;;gBAPK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAQP;;QAXK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAaL,OAAO,AAAC,qBAAGM,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;eAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;OAC1BnE,KAAK,CAAC,GAAGwE,KAAKS,CAAC,EACf9E,GAAG,CAAC;iDAAE+E,eAAGvC;eAAY;YAAEpD,MAAM2F;YAAGb,YAAY3B,SAASC;QAAO;;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = await store.vectors.pending();\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(store, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["embedPending","hasEmbedding","semanticCandidates","similarNotes","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","store","cfg","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","path","chunkIdxs","chunks","idx","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","writeVectors","job","q","Int8Array","round","chunk","vector","Buffer","from","buffer","tick","min","length","getProvider","pending","STORE_DIMS","Map","get","push","set","parseFile","relPath","absPath","join","mtimeMs","ctimeMs","size","presets","embed","doc","extracted","checkLanguageFit","progress","finish","terms","fetch","allowed","qv","SenseError","match","filter","t","includes","embedQuery","candidates","hasVector","_cfg","opts","similar"],"mappings":";;;;;;;;;;;QA4BsBA;eAAAA;;QAuEAC;eAAAA;;QAfAC;eAAAA;;QAqBAC;eAAAA;;QA3FNC;eAAAA;;;wBAdK;wBAEM;uBAEL;0BACG;uBACC;yBAEO;0BACL;uBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIpB,SAASA,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAef,aAAaiB,KAAY,EAAEC,GAAW,EAAEC,OAAe;;mBACrEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOC,MAAMC,WACZC,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,KAaPC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQP,KAAKS,KAAK,CAACH,GAAGA,IAAIZ,SAASgB,QAAQ;oCACjC;;wCAAMhB,SAASiB,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChB;;wCAAMjB,MAAMiB,OAAO,CAACO,YAAY,CAC9BR,MAAMK,GAAG,CAAC,SAACI,KAAKH;4CACd,IAAqBnC,WAAAA,QAAQ8B,OAAO,CAACK,EAAE,EAAEjB,WAAW,OAA5Cd,IAAaJ,SAAbI,GAAGM,QAAUV,SAAVU;4CACX,IAAM6B,IAAI,IAAIC,UAAUtB;4CACxB,IAAK,IAAIX,IAAI,GAAGA,IAAIW,WAAWX,IAAKgC,CAAC,CAAChC,EAAE,GAAGC,KAAKiC,KAAK,CAACrC,CAAC,CAACG,EAAE,GAAGG;4CAC7D,OAAO;gDAAEa,MAAMe,IAAIf,IAAI;gDAAEmB,OAAOJ,IAAII,KAAK;gDAAEhC,OAAAA;gDAAOiC,QAAQC,OAAOC,IAAI,CAACN,EAAEO,MAAM;4CAAE;wCAClF;;;oCANF;oCAQAnB,OAAOoB,IAAI,CAACvC,KAAKwC,GAAG,CAACpB,IAAIZ,SAASgB,QAAQ,EAAEV,KAAK2B,MAAM;;;;;;oBACzD;oBAhDiB;;wBAAMC,IAAAA,uBAAW,EAACpC;;;oBAA7BE,WAAW;oBACH;;wBAAMH,MAAMiB,OAAO,CAACqB,OAAO;;;oBAAnClC,QAAQ;oBACd,IAAIA,MAAMgC,MAAM,KAAK,GAAG;;;oBAClB/B,YAAYV,KAAKwC,GAAG,CAACI,mBAAU,EAAEpC,SAASd,IAAI;oBAE9CiB,SAAS,IAAIkC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAapC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOmC,GAAG,CAAClC,IAAIG,IAAI,eAAnBJ,yBAAAA;4BACbE,KAAKkC,IAAI,CAACnC,IAAIsB,KAAK;4BACnBvB,OAAOqC,GAAG,CAACpC,IAAIG,IAAI,EAAEF;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOI,uBAAMC;4BACZC,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,+EAA+E;gCAC/EA,SAASgC,IAAAA,kBAAS,EAAC;oCAAEC,SAASnC;oCAAMoC,SAASC,IAAAA,cAAI,EAAC7C,SAASQ;oCAAOsC,SAAS;oCAAGC,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMC,OAAO;gCAAK;oCAAIA,cAAK;mCAAGnD,KAAKoD,GAAG,CAACC,SAAS,CAACF,KAAK;4BAClK,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAazC,gCAAb,8BAAA,SAAA,0BAAA;oCAAME,MAAN;oCAAwB,IAAID,MAAM,CAACC,IAAI,EAAEJ,KAAKiC,IAAI,CAAC;wCAAEhC,MAAAA;wCAAMmB,OAAOhB;wCAAKU,MAAMX,MAAM,CAACC,IAAI,CAACU,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAMgC,IAAAA,2BAAgB,EACpBvD,OACAG,UACAM,KAAKY,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMA,8EAA8E;oBAC9E,6DAA6D;oBACvDT,SAAS0C,IAAAA,oBAAQ,EAAC,oBAAoB/C,KAAK2B,MAAM;oBAC9CrB,IAAI;;;yBAAGA,CAAAA,IAAIN,KAAK2B,MAAM,AAAD;;;;;;;;;;;;oBAAGrB,KAAKZ,SAASgB,QAAQ;;;;;;oBAavDL,OAAO2C,MAAM;;;;;;IACf;;AAKO,SAAexE,mBAAmBe,KAAY,EAAEC,GAAW,EAAEyD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;;YAOvGF,cANRxD,SAIAC,UACAE,WACAkB,MACYpC,UAAP0E;;;;oBAPL3D,UAAU,AAACD,IAAgCC,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAI4D,oBAAU,CAAC,eAAe;oBAClD;;wBAAM/E,aAAaiB,OAAOC,KAAKC;;;oBAA/B;oBAEiB;;wBAAMmC,IAAAA,uBAAW,EAACpC;;;oBAA7BE,WAAW;oBACXE,YAAYV,KAAKwC,GAAG,CAACI,mBAAU,EAAEpC,SAASd,IAAI;oBAC9CkC,OAAO,EAACmC,eAAAA,MAAMK,KAAK,CAAC,2naAAZL,0BAAAA,mBAAsCM,MAAM,CAAC,SAACC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIlB,IAAI,CAAC;oBACxF;;wBAAM5C,SAASgE,UAAU,CAAC5C;;;oBAAlCpC,WAAAA;wBAAQ;wBAAiCkB;wBAAW;wBAA3DwD,KAAO1E,SAAVI;oBAER;;wBAAOS,MAAMiB,OAAO,CAACmD,UAAU,CAACP,IAAIxD,WAAWsD,OAAOC;;;;IACxD;;AAIO,SAAe5E,aAAagB,KAAY,EAAEU,IAAY;;;YAC3D;;gBAAOV,MAAMiB,OAAO,CAACoD,SAAS,CAAC3D;;;IACjC;;AAIO,SAAexB,aAAac,KAAY,EAAEsE,IAAY,EAAE5D,IAAY,EAAE6D,IAAgE;;;YAC3I;;gBAAOvE,MAAMiB,OAAO,CAACuD,OAAO,CAAC9D,MAAM6D;;;IACrC"}

package/dist/cjs/embed/types.d.cts

@@ -1,3 +1,4 @@
+export declare const STORE_DIMS = 256;
 export interface EmbedProvider {
     id: string;
     dims: number;

package/dist/cjs/embed/types.d.ts

@@ -1,3 +1,4 @@
+export declare const STORE_DIMS = 256;
 export interface EmbedProvider {
     id: string;
     dims: number;

package/dist/cjs/embed/types.js

@@ -1,7 +1,16 @@
-// The provider contract every wire protocol implements: a Liskov-substitutable
-// surface so query/search code never branches on provider identity.
+// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):
+// int8 at 256 dims is quality-free vs f32-512 when fused. Also the duckdb store's FLOAT[N]
+// column width, so both stores share one source of truth for it rather than a schema
+// constant drifting from the quantization one.
 "use strict";
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
+Object.defineProperty(exports, "STORE_DIMS", {
+    enumerable: true,
+    get: function() {
+        return STORE_DIMS;
+    }
+});
+var STORE_DIMS = 256;
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,oEAAoE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused. Also the duckdb store's FLOAT[N]\n// column width, so both stores share one source of truth for it rather than a schema\n// constant drifting from the quantization one.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,2FAA2F;AAC3F,qFAAqF;AACrF,+CAA+C;;;;;+BAClCA;;;eAAAA;;;AAAN,IAAMA,aAAa"}

package/dist/cjs/errors.d.cts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.d.ts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAmBxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED'\n  | 'STORE_UNKNOWN'\n  | 'STORE_DEPENDENCY_MISSING'\n  | 'STORE_CAPABILITY_MISSING';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAsBxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}